diff --git a/README.md b/README.md index aca1791c..45a02498 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,8 @@ Alternatively, open the [`nix develop`](https://nixos.org/manual/nix/stable/comm $ cabal v2-run -- hackage-server run --static-dir=datafiles --state-dir=state \ --base-uri=http://localhost:8080 \ --required-base-host-header=localhost:8080 \ - --user-content-uri=http://127.0.0.1:8080 + --user-content-uri=http://127.0.0.1:8080 \ + --database-path=path/to/hackage.db hackage-server: Ready! Point your browser at http://localhost:8080 #### Populate the local package index @@ -50,6 +51,11 @@ Then $ nix run .#mirror-hackage-server +#### Database + +When hackage-server starts it will migrate data from the acid store to the specified database and +will keep using that going forward. + #### Not working Please note this App *cannot* be run [directly from GitHub](https://determinate.systems/posts/nix-run) like this: @@ -224,7 +230,8 @@ once to initialise the state. After that you can run the server with cabal v2-run -- hackage-server run --static-dir=datafiles --state-dir=state \ --base-uri=http://127.0.0.1:8080 \ --required-base-host-header=localhost:8080 \ - --user-content-uri=http://127.0.0.1:8080 + --user-content-uri=http://127.0.0.1:8080 \ + --database-path=path/to/hackage.db By default the server runs on port `8080` with the following settings: diff --git a/exes/Main.hs b/exes/Main.hs index 2e4723f1..02c52741 100644 --- a/exes/Main.hs +++ b/exes/Main.hs @@ -206,6 +206,7 @@ data RunFlags = RunFlags { flagRunTemp :: Flag Bool, flagRunCacheDelay :: Flag String, flagRunLiveTemplates :: Flag Bool, + flagRunDatabasePath :: Flag FilePath, -- Online backup flags flagRunBackupOutputDir :: Flag FilePath, flagRunBackupLinkBlobs :: Flag Bool, @@ -226,6 +227,7 @@ defaultRunFlags = RunFlags { flagRunTemp = Flag False, flagRunCacheDelay = NoFlag, flagRunLiveTemplates = Flag False, + flagRunDatabasePath = NoFlag, flagRunBackupOutputDir = Flag "backups", flagRunBackupLinkBlobs = Flag False, flagRunBackupScrubbed = Flag False @@ -311,6 +313,11 @@ runCommand = "Do not cache templates, for quicker feedback during development." flagRunLiveTemplates (\v flags -> flags { flagRunLiveTemplates = v }) (noArg (Flag True)) + , option [] ["database-path"] + "Path to the database file" + flagRunDatabasePath (\v flags -> flags { flagRunDatabasePath = v }) + (reqArgFlag "FILE") + -- NOTE: How to make --database-path mandatory? ] runAction :: RunFlags -> IO () @@ -340,12 +347,14 @@ runAction opts = do confTmpDir = tmpDir, confCacheDelay = cacheDelay, confLiveTemplates = liveTemplates, - confVerbosity = verbosity + confVerbosity = verbosity, + confDatabasePath = databasePath } outputDir = fromFlag (flagRunBackupOutputDir opts) linkBlobs = fromFlag (flagRunBackupLinkBlobs opts) scrubbed = fromFlag (flagRunBackupScrubbed opts) liveTemplates = fromFlag (flagRunLiveTemplates opts) + databasePath = fromFlagOrDefault (confDatabasePath defaults) (flagRunDatabasePath opts) checkBlankServerState =<< Server.hasSavedState config checkStaticDir staticDir (flagRunStaticDir opts) diff --git a/hackage-server.cabal b/hackage-server.cabal index 0b3a43bb..fe1bf735 100644 --- a/hackage-server.cabal +++ b/hackage-server.cabal @@ -176,7 +176,14 @@ common defaults , unordered-containers ^>= 0.2.10 , vector ^>= 0.12 || ^>= 0.13.0.0 , zlib ^>= 0.6.2 || ^>= 0.7.0.0 - + -- database dependencies + build-depends: + , beam-core ^>= 0.10.4.0 + , beam-sqlite ^>= 0.5.5.0 + , sqlite-simple ^>= 0.4.19.0 + , direct-sqlite ^>= 2.3.29 + , resource-pool ^>= 0.5.0.0 + , file-embed ^>= 0.0.16.0 library import: defaults @@ -306,6 +313,7 @@ library Distribution.Server.Features.Core Distribution.Server.Features.Core.State Distribution.Server.Features.Core.Backup + Distribution.Server.Features.Database Distribution.Server.Features.Security Distribution.Server.Features.Security.Backup Distribution.Server.Features.Security.FileInfo @@ -393,6 +401,7 @@ library Distribution.Server.Features.AnalyticsPixels Distribution.Server.Features.AnalyticsPixels.State Distribution.Server.Features.UserDetails + Distribution.Server.Features.UserDetails.State Distribution.Server.Features.UserSignup Distribution.Server.Features.StaticFiles Distribution.Server.Features.ServerIntrospect diff --git a/init_db.sql b/init_db.sql new file mode 100644 index 00000000..210cfae8 --- /dev/null +++ b/init_db.sql @@ -0,0 +1,26 @@ +-- Initialize SQLite3 database for hackage-server +-- +-- sqlite3 hackage.db < init_db.sql +-- +CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY, + username TEXT NOT NULL UNIQUE, -- CHECK: should it be unique? what happens with deleted users? + status TEXT NOT NULL, + auth_info TEXT, + admin BOOLEAN NOT NULL +); + +CREATE TABLE IF NOT EXISTS user_tokens ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL, + description TEXT NOT NULL, + token TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS account_details ( + user_id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + contact_email TEXT NOT NULL, + kind TEXT, + admin_notes TEXT NOT NULL +); diff --git a/src/Distribution/Server.hs b/src/Distribution/Server.hs index fe422e20..c60c2e89 100644 --- a/src/Distribution/Server.hs +++ b/src/Distribution/Server.hs @@ -75,7 +75,8 @@ data ServerConfig = ServerConfig { confStaticDir :: FilePath, confTmpDir :: FilePath, confCacheDelay:: Int, - confLiveTemplates :: Bool + confLiveTemplates :: Bool, + confDatabasePath :: String } deriving (Show) confDbStateDir, confBlobStoreDir :: ServerConfig -> FilePath @@ -108,7 +109,8 @@ defaultServerConfig = do confStaticDir = dataDir, confTmpDir = "state" "tmp", confCacheDelay= 0, - confLiveTemplates = False + confLiveTemplates = False, + confDatabasePath = "hackage.db" } data Server = Server { @@ -128,7 +130,7 @@ hasSavedState = doesDirectoryExist . confDbStateDir mkServerEnv :: ServerConfig -> IO ServerEnv mkServerEnv config@(ServerConfig verbosity hostURI userContentURI requiredBaseHostHeader _ stateDir _ tmpDir - cacheDelay liveTemplates) = do + cacheDelay liveTemplates _) = do createDirectoryIfMissing False stateDir let blobStoreDir = confBlobStoreDir config staticDir = confStaticFilesDir config @@ -153,7 +155,8 @@ mkServerEnv config@(ServerConfig verbosity hostURI userContentURI requiredBaseHo serverBaseURI = hostURI, serverUserContentBaseURI = userContentURI, serverRequiredBaseHostHeader = requiredBaseHostHeader, - serverVerbosity = verbosity + serverVerbosity = verbosity, + serverDatabasePath = confDatabasePath config } return env diff --git a/src/Distribution/Server/Features.hs b/src/Distribution/Server/Features.hs index 9755fce2..556e66c9 100644 --- a/src/Distribution/Server/Features.hs +++ b/src/Distribution/Server/Features.hs @@ -19,6 +19,7 @@ import Distribution.Server.Features.Core (initCoreFeature, coreResource, que import Distribution.Server.Features.Security (initSecurityFeature) import Distribution.Server.Features.Upload (initUploadFeature) import Distribution.Server.Features.Mirror (initMirrorFeature) +import Distribution.Server.Features.Database (initDatabaseFeature) #ifndef MINIMAL import Distribution.Server.Features.Browse (initBrowseFeature) @@ -89,6 +90,8 @@ initHackageFeatures env@ServerEnv{serverVerbosity = verbosity} = do loginfo verbosity "Initialising features, part 1" + mkDatabaseFeature <- logStartup "database" $ + initDatabaseFeature env mkStaticFilesFeature <- logStartup "static files" $ initStaticFilesFeature env mkUserFeature <- logStartup "user" $ @@ -172,9 +175,12 @@ initHackageFeatures env@ServerEnv{serverVerbosity = verbosity} = do -- Arguments denote feature dependencies. -- What follows is a topological sort along those lines + databaseFeature <- mkDatabaseFeature + staticFilesFeature <- mkStaticFilesFeature usersFeature <- mkUserFeature + databaseFeature coreFeature <- mkCoreFeature usersFeature @@ -204,11 +210,13 @@ initHackageFeatures env@ServerEnv{serverVerbosity = verbosity} = do coreFeature userDetailsFeature <- mkUserDetailsFeature + databaseFeature usersFeature coreFeature uploadFeature userSignupFeature <- mkUserSignupFeature + databaseFeature usersFeature userDetailsFeature uploadFeature @@ -299,6 +307,7 @@ initHackageFeatures env@ServerEnv{serverVerbosity = verbosity} = do platformFeature <- mkPlatformFeature htmlFeature <- mkHtmlFeature + databaseFeature usersFeature coreFeature packageContentsFeature @@ -328,6 +337,7 @@ initHackageFeatures env@ServerEnv{serverVerbosity = verbosity} = do uploadFeature adminFrontendFeature <- mkAdminFrontendFeature + databaseFeature usersFeature userDetailsFeature userSignupFeature @@ -352,6 +362,7 @@ initHackageFeatures env@ServerEnv{serverVerbosity = verbosity} = do uploadFeature userNotifyFeature <- mkUserNotifyFeature + databaseFeature usersFeature coreFeature uploadFeature @@ -392,6 +403,7 @@ initHackageFeatures env@ServerEnv{serverVerbosity = verbosity} = do , getFeatureInterface securityFeature , getFeatureInterface mirrorFeature , getFeatureInterface uploadFeature + , getFeatureInterface databaseFeature #ifndef MINIMAL , getFeatureInterface tarIndexCacheFeature , getFeatureInterface packageContentsFeature diff --git a/src/Distribution/Server/Features/AdminFrontend.hs b/src/Distribution/Server/Features/AdminFrontend.hs index 9261f3b5..0f0a0eff 100644 --- a/src/Distribution/Server/Features/AdminFrontend.hs +++ b/src/Distribution/Server/Features/AdminFrontend.hs @@ -12,6 +12,7 @@ import Distribution.Server.Features.Users import Distribution.Server.Features.UserDetails import Distribution.Server.Features.UserSignup import Distribution.Server.Features.LegacyPasswds +import Distribution.Server.Features.Database (DatabaseFeature(..)) import Distribution.Server.Users.Types import qualified Distribution.Server.Users.Users as Users @@ -27,7 +28,8 @@ import Data.Maybe (isJust) -- | An HTML UI for various server admin tasks, mostly user accounts -- initAdminFrontendFeature :: ServerEnv - -> IO (UserFeature + -> IO (DatabaseFeature + -> UserFeature -> UserDetailsFeature -> UserSignupFeature -> LegacyPasswdsFeature @@ -40,8 +42,9 @@ initAdminFrontendFeature env@ServerEnv{ serverTemplatesDir, [ "admin.html", "accounts.html", "account.html" , "signups.html", "resets.html", "legacy.html" ] - return $ \user userdetails usersignup legacypasswds -> do + return $ \database user userdetails usersignup legacypasswds -> do let feature = adminFrontendFeature env templates + database user userdetails usersignup legacypasswds @@ -49,12 +52,13 @@ initAdminFrontendFeature env@ServerEnv{ serverTemplatesDir, adminFrontendFeature :: ServerEnv -> Templates + -> DatabaseFeature -> UserFeature -> UserDetailsFeature -> UserSignupFeature -> LegacyPasswdsFeature -> HackageFeature -adminFrontendFeature _env templates +adminFrontendFeature _env templates DatabaseFeature{..} UserFeature{..} UserDetailsFeature{..} UserSignupFeature{..} LegacyPasswdsFeature{..} = (emptyHackageFeature "admin-frontend") { @@ -224,7 +228,7 @@ adminFrontendFeature _env templates template <- getTemplate templates "account.html" uid <- maybe mzero return (simpleParse =<< lookup "uid" dpath) uinfo <- lookupUserInfo uid - mudetails <- queryUserDetails uid + mudetails <- withTransaction $ queryUserDetails uid resetInfo <- lookupPasswordReset uid <$> queryAllSignupResetInfo mlegacy <- lookupUserLegacyPasswd uid <$> queryLegacyPasswds diff --git a/src/Distribution/Server/Features/Database.hs b/src/Distribution/Server/Features/Database.hs new file mode 100644 index 00000000..f417cdf4 --- /dev/null +++ b/src/Distribution/Server/Features/Database.hs @@ -0,0 +1,164 @@ +{-# LANGUAGE DeriveFunctor #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TypeFamilies #-} + +module Distribution.Server.Features.Database + ( DatabaseFeature (..), + Transaction, + initDatabaseFeature, + runSelectReturningOne, + runInsert, + HackageDb (..), + hackageDb, + -- for tests + testDatabaseFeature, + testDatabaseFeatureIO, + ) +where + +import Control.Monad.Reader +import Data.FileEmbed (embedStringFile) +import Data.Kind +import Data.Pool +import Database.Beam hiding (runInsert, runSelectReturningOne) +import qualified Database.Beam +import Database.Beam.Sqlite +import qualified Database.SQLite.Simple +import qualified Database.SQLite3 +import Distribution.Server.Features.UserDetails.State +import Distribution.Server.Framework +import Distribution.Server.Users.State + +newtype Connection = SqlLiteConnection Database.SQLite.Simple.Connection + +runSelectReturningOne :: forall a. (FromBackendRow Sqlite a) => SqlSelect Sqlite a -> Transaction (Maybe a) +runSelectReturningOne q = + Transaction $ ReaderT $ \(SqlLiteConnection conn) -> runBeamSqlite conn $ Database.Beam.runSelectReturningOne q + +runInsert :: forall (table :: (Type -> Type) -> Type). SqlInsert Sqlite table -> Transaction () +runInsert q = + Transaction $ ReaderT $ \(SqlLiteConnection conn) -> runBeamSqlite conn $ Database.Beam.runInsert q + +newtype Transaction a = Transaction {unTransaction :: ReaderT Connection IO a} + deriving (Functor, Applicative, Monad) + +runTransaction :: Transaction a -> Connection -> IO a +runTransaction t = runReaderT (unTransaction t) + +-- | A feature to store extra information about users like email addresses. +data DatabaseFeature = DatabaseFeature + { databaseFeatureInterface :: HackageFeature, + withTransaction :: forall a m. (MonadIO m) => Transaction a -> m a, + -- | whether the database is fresh and feature should migrate acid data + fresh :: Bool + } + +instance IsHackageFeature DatabaseFeature where + getFeatureInterface = databaseFeatureInterface + +-- | Ensures the database schema is initialized. Returns 'True' if the database was just created, 'False' if it already existed. +initSchema :: Database.SQLite.Simple.Connection -> IO Bool +initSchema conn = do + [Database.SQLite.Simple.Only tableExists] <- + Database.SQLite.Simple.query_ + conn + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='account_details');" + Database.SQLite3.exec (Database.SQLite.Simple.connectionHandle conn) $(embedStringFile "init_db.sql") + pure (not tableExists) + +initDatabaseFeature :: ServerEnv -> IO (IO DatabaseFeature) +initDatabaseFeature env = pure $ do + dbpool <- + newPool $ + defaultPoolConfig + (Database.SQLite.Simple.open (serverDatabasePath env)) + Database.SQLite.Simple.close + (5 {- time in seconds before unused connection is closed -}) + (20 {- number of connections -}) + + -- Initialize the database schema. + -- Script produce no changes if database is already initialized. + -- TODO: implement migrations + -- CHECK: Should this be done in featurePostInit instead? + fresh <- withResource dbpool initSchema + + pure $ mkDatabaseFeature dbpool fresh + where + mkDatabaseFeature :: Pool Database.SQLite.Simple.Connection -> Bool -> DatabaseFeature + mkDatabaseFeature dbpool fresh = DatabaseFeature {..} + where + databaseFeatureInterface = + (emptyHackageFeature "database") + { featureDesc = "A feature to store information in a SQL database.", + featurePostInit = pure (), + featureState = [] -- CHECK: should probably do a dump of the database here and perform an import somewhere else? + } + + withTransaction :: forall a m. (MonadIO m) => Transaction a -> m a + withTransaction action = + liftIO $ withResource dbpool $ \conn -> + Database.SQLite.Simple.withTransaction conn $ + runTransaction + action + (SqlLiteConnection conn) + +data HackageDb f = HackageDb + { _tblAccountDetails :: f (TableEntity AccountDetailsT), + _tblUsers :: f (TableEntity UsersT), + _tblUserTokens :: f (TableEntity UserTokensT) + } + deriving stock (Generic) + +instance Database be HackageDb + +hackageDb :: DatabaseSettings be HackageDb +hackageDb = defaultDbSettings + +-- | For testing purposes, in memory single connection database. +testDatabaseFeature :: + (forall r. IO r -> (r -> IO ()) -> (r -> b) -> b) -> + (DatabaseFeature -> b) -> + b +testDatabaseFeature withResourceFn action = + withResourceFn + setupTestDatabase + (\(conn, _) -> Database.SQLite.Simple.close conn) + (\(_, database) -> action database) + +testDatabaseFeatureIO :: + (forall r. IO r -> (r -> IO ()) -> (IO r -> b) -> b) -> + (IO DatabaseFeature -> b) -> + b +testDatabaseFeatureIO withResourceFn action = + withResourceFn + setupTestDatabase + (\(conn, _) -> Database.SQLite.Simple.close conn) + (\ioResource -> action (snd <$> ioResource)) + +setupTestDatabase :: IO (Database.SQLite.Simple.Connection, DatabaseFeature) +setupTestDatabase = do + conn <- Database.SQLite.Simple.open ":memory:" + fresh <- initSchema conn -- NOTE: Always fresh since it's an in-memory database + pure + ( conn, + DatabaseFeature + { databaseFeatureInterface = undefined, -- not needed for these tests + withTransaction = \transaction -> + liftIO $ + Database.SQLite.Simple.withTransaction conn $ + runTransaction + transaction + (SqlLiteConnection conn), + fresh = fresh + } + ) diff --git a/src/Distribution/Server/Features/Html.hs b/src/Distribution/Server/Features/Html.hs index bea91542..4073ae7a 100644 --- a/src/Distribution/Server/Features/Html.hs +++ b/src/Distribution/Server/Features/Html.hs @@ -14,6 +14,7 @@ import qualified Distribution.Server.Framework.ResponseContentTypes as Resource import Distribution.Server.Framework.Templating import Distribution.Server.Features.Core +import Distribution.Server.Features.Database (DatabaseFeature(..)) import Distribution.Server.Features.Upload import Distribution.Server.Features.BuildReports import Distribution.Server.Features.BuildReports.Render @@ -103,7 +104,8 @@ instance IsHackageFeature HtmlFeature where -- This means of generating HTML is somewhat temporary, in that a more advanced -- (and better-looking) HTML ajaxy scheme should come about later on. initHtmlFeature :: ServerEnv - -> IO (UserFeature + -> IO (DatabaseFeature + -> UserFeature -> CoreFeature -> PackageContentsFeature -> UploadFeature -> PackageCandidatesFeature @@ -147,7 +149,7 @@ initHtmlFeature env@ServerEnv{serverTemplatesDir, serverTemplatesMode, ] - return $ \user core@CoreFeature{packageChangeHook} + return $ \database user core@CoreFeature{packageChangeHook} packages upload candidates versions reversef @@ -163,7 +165,7 @@ initHtmlFeature env@ServerEnv{serverTemplatesDir, serverTemplatesMode, recentPackagesFeature -> do -- do rec, tie the knot rec let (feature, packageIndex, packagesPage) = - htmlFeature env user core + htmlFeature env database user core packages upload candidates versions reversef @@ -207,6 +209,7 @@ initHtmlFeature env@ServerEnv{serverTemplatesDir, serverTemplatesMode, return feature htmlFeature :: ServerEnv + -> DatabaseFeature -> UserFeature -> CoreFeature -> PackageContentsFeature @@ -236,6 +239,7 @@ htmlFeature :: ServerEnv -> (HtmlFeature, IO Response, IO Response) htmlFeature env@ServerEnv{..} + database user core@CoreFeature{queryGetPackageIndex} packages upload @@ -299,7 +303,7 @@ htmlFeature env@ServerEnv{..} names candidates recentPackagesFeature - htmlUsers = mkHtmlUsers user usersdetails + htmlUsers = mkHtmlUsers database user usersdetails htmlUploads = mkHtmlUploads utilities upload htmlDocUploads = mkHtmlDocUploads utilities core docsCore templates htmlDownloads = mkHtmlDownloads utilities download @@ -864,8 +868,8 @@ data HtmlUsers = HtmlUsers { htmlUsersResources :: [Resource] } -mkHtmlUsers :: UserFeature -> UserDetailsFeature -> HtmlUsers -mkHtmlUsers UserFeature{..} UserDetailsFeature{..} = HtmlUsers{..} +mkHtmlUsers :: DatabaseFeature -> UserFeature -> UserDetailsFeature -> HtmlUsers +mkHtmlUsers DatabaseFeature{..} UserFeature{..} UserDetailsFeature{..} = HtmlUsers{..} where users = userResource @@ -911,7 +915,7 @@ mkHtmlUsers UserFeature{..} UserDetailsFeature{..} = HtmlUsers{..} serveUserPage dpath = do uname <- userNameInPath dpath uid <- lookupUserName uname - udetails <- queryUserDetails uid + udetails <- withTransaction $ queryUserDetails uid let realname = maybe (display uname) (T.unpack . accountName) udetails uris <- getGroupIndex uid uriPairs <- forM uris $ \uri -> do diff --git a/src/Distribution/Server/Features/UserDetails.hs b/src/Distribution/Server/Features/UserDetails.hs index 0156cfcb..1c83fca7 100644 --- a/src/Distribution/Server/Features/UserDetails.hs +++ b/src/Distribution/Server/Features/UserDetails.hs @@ -5,7 +5,10 @@ module Distribution.Server.Features.UserDetails ( UserDetailsFeature(..), AccountDetails(..), - AccountKind(..) + AccountKind(..), + + -- Exposed to setup features in test context that use the database + dbQueryUserDetails ) where import Distribution.Server.Framework @@ -14,8 +17,11 @@ import Distribution.Server.Framework.BackupRestore import Distribution.Server.Framework.Templating import Distribution.Server.Features.Users +import Distribution.Server.Features.UserDetails.State import Distribution.Server.Features.Upload import Distribution.Server.Features.Core +import Distribution.Server.Features.Database (DatabaseFeature (..), HackageDb (..)) +import qualified Distribution.Server.Features.Database as Database import Distribution.Server.Users.Types import Distribution.Server.Util.Validators (guardValidLookingEmail, guardValidLookingName) @@ -25,6 +31,7 @@ import Data.SafeCopy (base, deriveSafeCopy) import Data.IntMap (IntMap) import qualified Data.IntMap as IntMap import Data.Text (Text) +import Data.Maybe (fromMaybe) import qualified Data.Text as T import qualified Data.Aeson as Aeson import Data.Aeson.TH @@ -35,6 +42,8 @@ import Control.Monad.State (get, put) import Distribution.Text (display) import Data.Version import Text.CSV (CSV, Record) +import Database.Beam hiding (update) +import Database.Beam.Backend.SQL.BeamExtensions -- | A feature to store extra information about users like email addresses. @@ -42,8 +51,8 @@ import Text.CSV (CSV, Record) data UserDetailsFeature = UserDetailsFeature { userDetailsFeatureInterface :: HackageFeature, - queryUserDetails :: forall m. MonadIO m => UserId -> m (Maybe AccountDetails), - updateUserDetails :: forall m. MonadIO m => UserId -> AccountDetails -> m () + queryUserDetails :: UserId -> Database.Transaction (Maybe AccountDetails), + updateUserDetails :: UserId -> AccountDetails -> Database.Transaction () } instance IsHackageFeature UserDetailsFeature where @@ -250,7 +259,8 @@ userDetailsToCSV backuptype (UserDetailsTable tbl) -- initUserDetailsFeature :: ServerEnv - -> IO (UserFeature + -> IO (DatabaseFeature + -> UserFeature -> CoreFeature -> UploadFeature -> IO UserDetailsFeature) @@ -265,18 +275,35 @@ initUserDetailsFeature ServerEnv{serverStateDir, serverTemplatesDir, serverTempl [serverTemplatesDir, serverTemplatesDir "UserDetails"] [ "user-details-form.html" ] - return $ \users core upload -> do - let feature = userDetailsFeature templates usersDetailsState users core upload + return $ \database users core upload -> do + when (fresh database) + (migrateStateToDatabase usersDetailsState database) + let feature = userDetailsFeature templates usersDetailsState database users core upload return feature +migrateStateToDatabase :: StateComponent AcidState UserDetailsTable + -> DatabaseFeature + -> IO () +migrateStateToDatabase userDetailsState DatabaseFeature{..} = do + (UserDetailsTable tbl) <- queryState userDetailsState GetUserDetailsTable + withTransaction $ do + forM_ (IntMap.toList tbl) $ \(uid, details) -> do + accountDetailsUpsert AccountDetailsRow { + _adUserId = UserId uid, + _adName = accountName details, + _adContactEmail = accountContactEmail details, + _adKind = fmap fromAccountKind (accountKind details), + _adAdminNotes = accountAdminNotes details + } userDetailsFeature :: Templates -> StateComponent AcidState UserDetailsTable + -> DatabaseFeature -> UserFeature -> CoreFeature -> UploadFeature -> UserDetailsFeature -userDetailsFeature templates userDetailsState UserFeature{..} CoreFeature{..} UploadFeature{uploadersGroup} +userDetailsFeature templates userDetailsState DatabaseFeature{..} UserFeature{..} CoreFeature{..} UploadFeature{uploadersGroup} = UserDetailsFeature {..} where @@ -317,12 +344,11 @@ userDetailsFeature templates userDetailsState UserFeature{..} CoreFeature{..} Up -- Queries and updates -- - queryUserDetails :: MonadIO m => UserId -> m (Maybe AccountDetails) - queryUserDetails uid = queryState userDetailsState (LookupUserDetails uid) + queryUserDetails :: UserId -> Database.Transaction (Maybe AccountDetails) + queryUserDetails = dbQueryUserDetails - updateUserDetails :: MonadIO m => UserId -> AccountDetails -> m () - updateUserDetails uid udetails = do - updateState userDetailsState (SetUserDetails uid udetails) + updateUserDetails :: UserId -> AccountDetails -> Database.Transaction () + updateUserDetails = dbUpdateUserDetails -- Request handlers -- @@ -331,7 +357,7 @@ userDetailsFeature templates userDetailsState UserFeature{..} CoreFeature{..} Up (uid, uinfo) <- lookupUserNameFull =<< userNameInPath dpath guardAuthorised_ [IsUserId uid, InGroup adminGroup] template <- getTemplate templates "user-details-form.html" - udetails <- queryUserDetails uid + udetails <- withTransaction $ queryUserDetails uid showConfirmationOfSave <- not . null <$> queryString (lookBSs "showConfirmationOfSave") let emailTxt = maybe "" accountContactEmail udetails @@ -356,7 +382,7 @@ userDetailsFeature templates userDetailsState UserFeature{..} CoreFeature{..} Up handlerGetUserNameContact dpath = do uid <- lookupUserName =<< userNameInPath dpath guardAuthorised_ [IsUserId uid, InGroup adminGroup] - udetails <- queryUserDetails uid + udetails <- withTransaction $ queryUserDetails uid return $ toResponse (Aeson.toJSON (render udetails)) where render Nothing = NameAndContact T.empty T.empty @@ -374,21 +400,21 @@ userDetailsFeature templates userDetailsState UserFeature{..} CoreFeature{..} Up NameAndContact name email <- expectAesonContent guardValidLookingName name guardValidLookingEmail email - updateState userDetailsState (SetUserNameContact uid name email) + withTransaction $ dbModifyAccountDetails uid (\adetails -> adetails { accountName = name, accountContactEmail = email }) noContent $ toResponse () handlerDeleteUserNameContact :: DynamicPath -> ServerPartE Response handlerDeleteUserNameContact dpath = do uid <- lookupUserName =<< userNameInPath dpath guardAuthorised_ [IsUserId uid, InGroup adminGroup] - updateState userDetailsState (SetUserNameContact uid T.empty T.empty) + withTransaction $ dbModifyAccountDetails uid (\adetails -> adetails { accountName = "", accountContactEmail = "" }) noContent $ toResponse () handlerGetAdminInfo :: DynamicPath -> ServerPartE Response handlerGetAdminInfo dpath = do guardAuthorised_ [InGroup adminGroup] uid <- lookupUserName =<< userNameInPath dpath - udetails <- queryUserDetails uid + udetails <- withTransaction $ queryUserDetails uid return $ toResponse (Aeson.toJSON (render udetails)) where render Nothing = AdminInfo Nothing T.empty @@ -403,12 +429,87 @@ userDetailsFeature templates userDetailsState UserFeature{..} CoreFeature{..} Up guardAuthorised_ [InGroup adminGroup] uid <- lookupUserName =<< userNameInPath dpath AdminInfo akind notes <- expectAesonContent - updateState userDetailsState (SetUserAdminInfo uid akind notes) + withTransaction $ dbModifyAccountDetails uid (\adetails -> adetails { accountKind = akind, accountAdminNotes = notes }) noContent $ toResponse () handlerDeleteAdminInfo :: DynamicPath -> ServerPartE Response handlerDeleteAdminInfo dpath = do guardAuthorised_ [InGroup adminGroup] uid <- lookupUserName =<< userNameInPath dpath - updateState userDetailsState (SetUserAdminInfo uid Nothing T.empty) + withTransaction $ dbModifyAccountDetails uid (\adetails -> adetails { accountKind = Nothing, accountAdminNotes = "" }) noContent $ toResponse () + + +-- Database + +dbQueryUserDetails :: UserId -> Database.Transaction (Maybe AccountDetails) +dbQueryUserDetails uid = fmap toUserDetails <$> accountDetailsFindByUserId uid + +dbUpdateUserDetails :: UserId -> AccountDetails -> Database.Transaction () +dbUpdateUserDetails uid udetails = dbModifyAccountDetails uid (const udetails) + +-- convenient helper to update only part of the record. +-- We use the same record for information that is editable by the user and information that is only editable by admins. +dbModifyAccountDetails :: UserId -> (AccountDetails -> AccountDetails) -> Database.Transaction () +dbModifyAccountDetails uid change = do + -- NOTE: we need to query the current value because we are updating only some of the fields. + madetails <- dbQueryUserDetails uid + -- NOTE: We could assume that the record exist since updateUserDetails is called from UserSignup + let adetails = fromMaybe AccountDetails { + accountName = "", + accountContactEmail = "", + accountKind = Nothing, + accountAdminNotes = "" + } madetails + let cdetails = change adetails + + accountDetailsUpsert AccountDetailsRow { + _adUserId = uid, + _adName = accountName cdetails, + _adContactEmail = accountContactEmail cdetails, + _adKind = fmap fromAccountKind (accountKind cdetails), + _adAdminNotes = accountAdminNotes cdetails + } + +accountDetailsFindByUserId :: UserId -> Database.Transaction (Maybe AccountDetailsRow) +accountDetailsFindByUserId uid = + Database.runSelectReturningOne $ + lookup_ (_tblAccountDetails Database.hackageDb) (AccountDetailsId uid) + +-- Use the values from the INSERT that caused the conflict +accountDetailsUpsert :: AccountDetailsRow -> Database.Transaction () +accountDetailsUpsert details = + Database.runInsert $ + insertOnConflict + (_tblAccountDetails Database.hackageDb) + (insertValues [details]) + (conflictingFields primaryKey) + ( onConflictUpdateSet $ \fields _oldRow -> + mconcat + [ _adName fields <-. val_ (_adName details), + _adContactEmail fields <-. val_ (_adContactEmail details), + _adKind fields <-. val_ (_adKind details), + _adAdminNotes fields <-. val_ (_adAdminNotes details) + ] + ) + +toUserDetails :: AccountDetailsRow -> AccountDetails +toUserDetails AccountDetailsRow {..} = + AccountDetails + { accountName = _adName, + accountContactEmail = _adContactEmail, + accountKind = fmap toAccountKind _adKind, + accountAdminNotes = _adAdminNotes + } + +toAccountKind :: AccountDetailsKind -> AccountKind +toAccountKind adKind = + case adKind of + RealUser -> AccountKindRealUser + Special -> AccountKindSpecial + +fromAccountKind :: AccountKind -> AccountDetailsKind +fromAccountKind adKind = + case adKind of + AccountKindRealUser -> RealUser + AccountKindSpecial -> Special diff --git a/src/Distribution/Server/Features/UserDetails/State.hs b/src/Distribution/Server/Features/UserDetails/State.hs new file mode 100644 index 00000000..93933f51 --- /dev/null +++ b/src/Distribution/Server/Features/UserDetails/State.hs @@ -0,0 +1,47 @@ +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE TypeFamilies #-} + +module Distribution.Server.Features.UserDetails.State where + +import Data.Text (Text, unpack) +import Database.Beam +import Database.Beam.Backend.SQL.SQL92 (HasSqlValueSyntax (..), autoSqlValueSyntax) +import Database.Beam.Sqlite (Sqlite) +import Database.Beam.Sqlite.Syntax (SqliteValueSyntax (..)) +import Distribution.Server.Users.Types + +data AccountDetailsT f + = AccountDetailsRow + { _adUserId :: Columnar f UserId, + _adName :: Columnar f Text, + _adContactEmail :: Columnar f Text, + _adKind :: Columnar f (Maybe AccountDetailsKind), + _adAdminNotes :: Columnar f Text + } + deriving (Generic, Beamable) + +type AccountDetailsRow = AccountDetailsT Identity + +deriving instance Show AccountDetailsRow + +deriving instance Eq AccountDetailsRow + +type AccountDetailsId = PrimaryKey AccountDetailsT Identity + +instance Table AccountDetailsT where + data PrimaryKey AccountDetailsT f = AccountDetailsId (Columnar f UserId) deriving (Generic, Beamable) + primaryKey = AccountDetailsId . _adUserId + +data AccountDetailsKind = RealUser | Special + deriving (Eq, Show, Read, Enum, Bounded) + +instance HasSqlValueSyntax SqliteValueSyntax AccountDetailsKind where + sqlValueSyntax = autoSqlValueSyntax + +instance FromBackendRow Sqlite AccountDetailsKind where + fromBackendRow = read . unpack <$> fromBackendRow diff --git a/src/Distribution/Server/Features/UserNotify.hs b/src/Distribution/Server/Features/UserNotify.hs index 454302ec..940b34fa 100644 --- a/src/Distribution/Server/Features/UserNotify.hs +++ b/src/Distribution/Server/Features/UserNotify.hs @@ -44,6 +44,7 @@ import Distribution.Server.Framework.Templating import Distribution.Server.Features.AdminLog import Distribution.Server.Features.BuildReports +import Distribution.Server.Features.Database (DatabaseFeature(..)) import qualified Distribution.Server.Features.BuildReports.BuildReport as BuildReport import Distribution.Server.Features.Core import Distribution.Server.Features.ReverseDependencies (ReverseFeature(..)) @@ -430,7 +431,8 @@ notifyStateComponent stateDir = do -- initUserNotifyFeature :: ServerEnv - -> IO (UserFeature + -> IO (DatabaseFeature + -> UserFeature -> CoreFeature -> UploadFeature -> AdminLogFeature @@ -450,9 +452,9 @@ initUserNotifyFeature ServerEnv{ serverStateDir, serverTemplatesDir, [serverTemplatesDir, serverTemplatesDir "UserNotify"] [ "user-notify-form.html", "endorsements-complete.txt" ] - return $ \users core uploadfeature adminlog userdetails reports tags revers vouch -> do + return $ \database users core uploadfeature adminlog userdetails reports tags revers vouch -> do let feature = userNotifyFeature - users core uploadfeature adminlog userdetails reports tags + database users core uploadfeature adminlog userdetails reports tags revers vouch notifyState templates return feature @@ -575,7 +577,8 @@ pkgInfoToPkgId :: PkgInfo -> PackageIdentifier pkgInfoToPkgId pkgInfo = PackageIdentifier (packageName pkgInfo) (packageVersion pkgInfo) -userNotifyFeature :: UserFeature +userNotifyFeature :: DatabaseFeature + -> UserFeature -> CoreFeature -> UploadFeature -> AdminLogFeature @@ -587,7 +590,8 @@ userNotifyFeature :: UserFeature -> StateComponent AcidState NotifyData -> Templates -> UserNotifyFeature -userNotifyFeature UserFeature{..} +userNotifyFeature database + UserFeature{..} CoreFeature{..} UploadFeature{..} AdminLogFeature{..} @@ -714,8 +718,10 @@ userNotifyFeature UserFeature{..} vouchNotifications <- fmap (, NotifyVouchingCompleted) <$> drainQueuedNotifications + -- CHECK: maybe the transaction should be initialized in a different place + -- currently it is done in getNotificationEmails. emails <- - getNotificationEmails userFeatureServerEnv userDetailsFeature users templates $ + getNotificationEmails userFeatureServerEnv database userDetailsFeature users templates $ concat [ revisionUploadNotifications , groupActionNotifications @@ -921,6 +927,7 @@ data NotificationGroup -- | Get all the emails to send for the given notifications. getNotificationEmails :: ServerEnv + -> DatabaseFeature -> UserDetailsFeature -> Users.Users -> Templates @@ -928,12 +935,13 @@ getNotificationEmails -> IO [Mail] getNotificationEmails ServerEnv{serverBaseURI} + DatabaseFeature{withTransaction} UserDetailsFeature{queryUserDetails} allUsers templates notifications = do let userIds = Set.fromList $ map fst notifications - userIdToDetails <- Map.mapMaybe id <$> fromSetM queryUserDetails userIds + userIdToDetails <- withTransaction $ Map.mapMaybe id <$> fromSetM queryUserDetails userIds vouchTemplate <- renderTemplate . ($ []) <$> getTemplate templates "endorsements-complete.txt" pure $ let emails = groupNotifications $ map (fmap (renderNotification vouchTemplate)) notifications diff --git a/src/Distribution/Server/Features/UserSignup.hs b/src/Distribution/Server/Features/UserSignup.hs index 65c29f9e..e8867a5b 100644 --- a/src/Distribution/Server/Features/UserSignup.hs +++ b/src/Distribution/Server/Features/UserSignup.hs @@ -15,6 +15,7 @@ import Distribution.Server.Framework.Templating import Distribution.Server.Framework.BackupDump import Distribution.Server.Framework.BackupRestore +import Distribution.Server.Features.Database (DatabaseFeature (..)) import Distribution.Server.Features.Upload import Distribution.Server.Features.Users import Distribution.Server.Features.UserDetails @@ -276,7 +277,8 @@ resetInfoToCSV backuptype (SignupResetTable tbl) -- initUserSignupFeature :: ServerEnv - -> IO (UserFeature + -> IO (DatabaseFeature + -> UserFeature -> UserDetailsFeature -> UploadFeature -> IO UserSignupFeature) @@ -293,14 +295,15 @@ initUserSignupFeature env@ServerEnv{ serverStateDir, serverTemplatesDir, , "ResetRequest.html", "ResetConfirmation.email" , "ResetEmailSent.html", "ResetConfirm.html" ] - return $ \users userdetails upload -> do + return $ \database users userdetails upload -> do let feature = userSignupFeature env - users userdetails upload + database users userdetails upload signupResetState templates return feature userSignupFeature :: ServerEnv + -> DatabaseFeature -> UserFeature -> UserDetailsFeature -> UploadFeature @@ -308,6 +311,7 @@ userSignupFeature :: ServerEnv -> Templates -> UserSignupFeature userSignupFeature ServerEnv{serverBaseURI, serverCron} + DatabaseFeature{..} UserFeature{..} UserDetailsFeature{..} UploadFeature{uploadersGroup} signupResetState templates = UserSignupFeature {..} @@ -562,7 +566,7 @@ userSignupFeature ServerEnv{serverBaseURI, serverCron} } uid <- updateAddUser username userauth >>= either errNameClash return - updateUserDetails uid acctDetails + withTransaction $ updateUserDetails uid acctDetails liftIO $ addUserToGroup uploadersGroup uid seeOther (userPageUri userResource "" username) (toResponse ()) where @@ -593,7 +597,7 @@ userSignupFeature ServerEnv{serverBaseURI, serverCron} (supplied_username, supplied_useremail) <- lookUserNameEmail (uid, uinfo) <- lookupUserNameFull supplied_username - mudetails <- queryUserDetails uid + mudetails <- withTransaction $ queryUserDetails uid guardEmailMatches mudetails supplied_useremail AccountDetails{..} <- guardSuitableAccountType uinfo mudetails @@ -658,7 +662,7 @@ userSignupFeature ServerEnv{serverBaseURI, serverCron} nonce <- nonceInPath dpath ResetInfo{resetUserId} <- lookupResetInfo nonce uinfo@UserInfo{userName} <- lookupUserInfo resetUserId - mudetails <- queryUserDetails resetUserId + mudetails <- withTransaction $ queryUserDetails resetUserId AccountDetails{..} <- guardSuitableAccountType uinfo mudetails template <- getTemplate templates "ResetConfirm.html" @@ -676,7 +680,7 @@ userSignupFeature ServerEnv{serverBaseURI, serverCron} nonce <- nonceInPath dpath ResetInfo{resetUserId} <- lookupResetInfo nonce uinfo@UserInfo{userName} <- lookupUserInfo resetUserId - mudetails <- queryUserDetails resetUserId + mudetails <- withTransaction $ queryUserDetails resetUserId AccountDetails{..} <- guardSuitableAccountType uinfo mudetails (passwd, passwdRepeat) <- lookPasswd diff --git a/src/Distribution/Server/Features/Users.hs b/src/Distribution/Server/Features/Users.hs index 69b215b3..bfa032aa 100644 --- a/src/Distribution/Server/Features/Users.hs +++ b/src/Distribution/Server/Features/Users.hs @@ -14,6 +14,9 @@ import Distribution.Server.Framework.BackupDump import Distribution.Server.Framework.Templating import qualified Distribution.Server.Framework.Auth as Auth +import Distribution.Server.Features.Database (DatabaseFeature (..), HackageDb (..)) +import qualified Distribution.Server.Features.Database as Database + import Distribution.Server.Users.Types import Distribution.Server.Users.State import Distribution.Server.Users.Backup @@ -39,6 +42,8 @@ import Distribution.Text (display, simpleParse) import Happstack.Server.Cookie (addCookie, mkCookie, CookieLife(Session)) +import Database.Beam hiding (update) + -- | A feature to allow manipulation of the database of users. -- -- TODO: clean up mismatched and duplicate functionality (some noted below). @@ -227,7 +232,7 @@ deriveJSON (compatAesonOptionsDropPrefix "ui_") ''EnabledResource deriveJSON (compatAesonOptionsDropPrefix "ui_") ''UserGroupResource -- TODO: add renaming -initUserFeature :: ServerEnv -> IO (IO UserFeature) +initUserFeature :: ServerEnv -> IO (DatabaseFeature -> IO UserFeature) initUserFeature serverEnv@ServerEnv{serverStateDir, serverTemplatesDir, serverTemplatesMode} = do -- Canonical state usersState <- usersStateComponent serverStateDir @@ -248,7 +253,10 @@ initUserFeature serverEnv@ServerEnv{serverStateDir, serverTemplatesDir, serverTe [ "manage.html", "token-created.html", "token-revoked.html" ] - return $ do + return $ \database -> do + when (fresh database) + (migrateStateToDatabase usersState adminsState database) + -- Slightly tricky: we have an almost recursive knot between the group -- resource management functions, and creating the admin group -- resource that is part of the user feature. @@ -257,6 +265,7 @@ initUserFeature serverEnv@ServerEnv{serverStateDir, serverTemplatesDir, serverTe -- rec let (feature@UserFeature{groupResourceAt}, adminGroupDesc) = userFeature templates + database usersState adminsState groupIndex @@ -268,6 +277,57 @@ initUserFeature serverEnv@ServerEnv{serverStateDir, serverTemplatesDir, serverTe return feature +migrateStateToDatabase :: StateComponent AcidState Users.Users + -> StateComponent AcidState HackageAdmins + -> DatabaseFeature + -> IO () +migrateStateToDatabase usersState adminsState DatabaseFeature{..} = do + users <- queryState usersState GetUserDb + admins <- queryState adminsState GetAdminList + + let usersAndTokens = + map (\(uid, uinfo) -> + ( (uid, uinfo), + map (\(token, desc) -> (uid, token, desc)) + (Map.toList (userTokens uinfo)) + ) + ) (Users.enumerateAllUsers users) + + let usersToInsert = map fst usersAndTokens + let tokensToInsert = concatMap snd usersAndTokens + withTransaction $ do + Database.runInsert $ + insert + (_tblUsers Database.hackageDb) + (insertValues (map (\(uid, uinfo) -> + let (status, authInfo) = + case userStatus uinfo of + AccountEnabled a -> (Enabled, Just a) + AccountDisabled ma -> (Disabled, ma) + AccountDeleted -> (Deleted, Nothing) + in + UsersRow { + _uId = uid, + _uUsername = userName uinfo, + _uStatus = status, + _uAuthInfo = authInfo, + _uAdmin = Group.member uid admins + } + ) usersToInsert)) + + Database.runInsert $ + insert + (_tblUserTokens Database.hackageDb) + (insertExpressions (map (\(uid, token, desc) -> + UserTokensRow { + _utId = default_, + _utUserId = val_ uid, + _utToken = val_ token, + _utDescription = val_ desc + } + ) tokensToInsert)) + + usersStateComponent :: FilePath -> IO (StateComponent AcidState Users.Users) usersStateComponent stateDir = do st <- openLocalStateFrom (stateDir "db" "Users") initialUsers @@ -295,6 +355,7 @@ adminsStateComponent stateDir = do } userFeature :: Templates + -> DatabaseFeature -> StateComponent AcidState Users.Users -> StateComponent AcidState HackageAdmins -> MemState GroupIndex @@ -305,7 +366,7 @@ userFeature :: Templates -> GroupResource -> ServerEnv -> (UserFeature, UserGroup) -userFeature templates usersState adminsState +userFeature templates database usersState adminsState groupIndex userAdded authFailHook groupChangedHook adminGroup adminResource userFeatureServerEnv = (UserFeature {..}, adminGroupDesc) diff --git a/src/Distribution/Server/Framework/ServerEnv.hs b/src/Distribution/Server/Framework/ServerEnv.hs index 9e4cfdee..464097dc 100644 --- a/src/Distribution/Server/Framework/ServerEnv.hs +++ b/src/Distribution/Server/Framework/ServerEnv.hs @@ -73,6 +73,8 @@ data ServerEnv = ServerEnv { -- increasing the time taken to update the cache we can push this further. serverCacheDelay :: Int, + serverDatabasePath :: String, + serverVerbosity :: Verbosity } diff --git a/src/Distribution/Server/Users/AuthToken.hs b/src/Distribution/Server/Users/AuthToken.hs index 4a217427..bae92360 100644 --- a/src/Distribution/Server/Users/AuthToken.hs +++ b/src/Distribution/Server/Users/AuthToken.hs @@ -1,5 +1,6 @@ {-# LANGUAGE DeriveDataTypeable, GeneralizedNewtypeDeriving #-} {-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE MultiParamTypeClasses #-} module Distribution.Server.Users.AuthToken ( AuthToken , parseAuthToken, parseAuthTokenM, renderAuthToken @@ -25,6 +26,11 @@ import qualified Distribution.Compat.CharParsing as P import Data.SafeCopy +import Database.Beam +import Database.Beam.Backend +import Database.Beam.Sqlite +import Database.Beam.Sqlite.Syntax + -- | Contains the original token which will be shown to the user -- once and is NOT stored on the server. The user is expected -- to provide this token on each request that should be @@ -36,6 +42,12 @@ newtype OriginalToken = OriginalToken Nonce newtype AuthToken = AuthToken BSS.ShortByteString deriving (Eq, Ord, Read, Show, MemSize) +instance FromBackendRow Sqlite AuthToken where + fromBackendRow = AuthToken . BSS.toShort <$> fromBackendRow + +instance HasSqlValueSyntax SqliteValueSyntax AuthToken where + sqlValueSyntax (AuthToken v) = sqlValueSyntax (BSS.fromShort v) + convertToken :: OriginalToken -> AuthToken convertToken (OriginalToken bs) = AuthToken $ BSS.toShort $ SHA256.hash $ getRawNonceBytes bs diff --git a/src/Distribution/Server/Users/State.hs b/src/Distribution/Server/Users/State.hs index 1d15c97e..e5d2cc95 100644 --- a/src/Distribution/Server/Users/State.hs +++ b/src/Distribution/Server/Users/State.hs @@ -1,6 +1,7 @@ {-# LANGUAGE DeriveDataTypeable, TypeFamilies, TemplateHaskell, FlexibleInstances, FlexibleContexts, MultiParamTypeClasses, - TypeOperators #-} + TypeOperators, StandaloneDeriving, DeriveGeneric, DeriveAnyClass, + DerivingStrategies, GeneralisedNewtypeDeriving #-} {-# OPTIONS_GHC -fno-warn-orphans #-} module Distribution.Server.Users.State where @@ -19,6 +20,13 @@ import Control.Monad.Reader import qualified Control.Monad.State as State import qualified Data.Text as T +import Data.Int +import Data.Text +import Database.Beam +import Database.Beam.Backend.SQL.SQL92 (HasSqlValueSyntax (..), autoSqlValueSyntax) +import Database.Beam.Sqlite (Sqlite) +import Database.Beam.Sqlite.Syntax (SqliteValueSyntax (..)) + initialUsers :: Users.Users initialUsers = Users.emptyUsers @@ -175,3 +183,60 @@ $(makeAcidic ''MirrorClients ,'addMirrorClient ,'removeMirrorClient ,'replaceMirrorClients]) + +-- Database + +data UsersT f + = UsersRow + { _uId :: Columnar f UserId, + _uUsername :: Columnar f UserName, + _uStatus :: Columnar f UsersStatus, + _uAuthInfo :: Columnar f (Maybe UserAuth), + _uAdmin :: Columnar f Bool + } + deriving (Generic, Beamable) + +type UserRow = UsersT Identity + +deriving instance Show UserRow + +deriving instance Eq UserRow + +type UsersId = PrimaryKey UsersT Identity + +instance Table UsersT where + data PrimaryKey UsersT f = UsersId (Columnar f UserId) deriving (Generic, Beamable) + primaryKey = UsersId . _uId + +data UsersStatus = Enabled | Disabled | Deleted + deriving (Eq, Show, Read, Enum, Bounded) + +instance HasSqlValueSyntax SqliteValueSyntax UsersStatus where + sqlValueSyntax = autoSqlValueSyntax + +instance FromBackendRow Sqlite UsersStatus where + fromBackendRow = read . unpack <$> fromBackendRow + +newtype DBUserName = DBUserName Text + deriving newtype (Eq, Ord, Read, Show, FromBackendRow Sqlite, HasSqlValueSyntax SqliteValueSyntax) + +data UserTokensT f + = UserTokensRow + { _utId :: Columnar f Int32, + _utUserId :: Columnar f UserId, + _utDescription :: Columnar f Text, + _utToken :: Columnar f AuthToken + } + deriving (Generic, Beamable) + +type UserTokenRow = UserTokensT Identity + +deriving instance Show UserTokenRow + +deriving instance Eq UserTokenRow + +type UserTokensId = PrimaryKey UserTokensT Identity + +instance Table UserTokensT where + data PrimaryKey UserTokensT f = UserTokensId (Columnar f Int32) deriving (Generic, Beamable) + primaryKey = UserTokensId . _utId \ No newline at end of file diff --git a/src/Distribution/Server/Users/Types.hs b/src/Distribution/Server/Users/Types.hs index b297ab80..611747eb 100644 --- a/src/Distribution/Server/Users/Types.hs +++ b/src/Distribution/Server/Users/Types.hs @@ -1,6 +1,9 @@ {-# LANGUAGE DeriveDataTypeable, GeneralizedNewtypeDeriving, TemplateHaskell #-} +{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE TypeApplications #-} {-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE MultiParamTypeClasses #-} module Distribution.Server.Users.Types ( module Distribution.Server.Users.Types, module Distribution.Server.Users.AuthToken, @@ -23,17 +26,41 @@ import qualified Data.Text as T import qualified Data.Map as M import qualified Data.List as L +import Data.Proxy +import Data.Text (unpack) +import Data.Int import Data.Aeson (ToJSON, FromJSON) import Data.SafeCopy (base, extension, deriveSafeCopy, Migrate(..)) import Data.Hashable import Data.Serialize (Serialize) - +import Database.Beam +import Database.Beam.Backend +import Database.Beam.Sqlite +import Database.Beam.Sqlite.Syntax newtype UserId = UserId Int deriving newtype (Eq, Ord, Read, Show, MemSize, ToJSON, FromJSON, Pretty) +-- NOTE: To use UserId directly we need to change it to a non machine-dependent size for Beam +-- We force Beam to treat UserId as a Int32 + +instance FromBackendRow Sqlite UserId where + fromBackendRow = UserId . fromIntegral @Int32 <$> fromBackendRow + +instance HasSqlValueSyntax SqliteValueSyntax UserId where + sqlValueSyntax (UserId v) = sqlValueSyntax (fromIntegral v :: Int32) + +instance HasSqlEqualityCheck Sqlite UserId where + sqlEqE _ = sqlEqE (Proxy :: Proxy Int32) + sqlNeqE _ = sqlNeqE (Proxy :: Proxy Int32) + sqlEqTriE _ = sqlEqTriE (Proxy :: Proxy Int32) + sqlNeqTriE _ = sqlNeqTriE (Proxy :: Proxy Int32) + newtype UserName = UserName String - deriving newtype (Eq, Ord, Read, Show, MemSize, ToJSON, FromJSON, Hashable, Serialize) + deriving newtype (Eq, Ord, Read, Show, MemSize, ToJSON, FromJSON, Hashable, Serialize, HasSqlValueSyntax SqliteValueSyntax) + +instance FromBackendRow Sqlite UserName where + fromBackendRow = UserName . unpack <$> fromBackendRow data UserInfo = UserInfo { userName :: !UserName, @@ -49,6 +76,12 @@ data UserStatus = AccountEnabled UserAuth newtype UserAuth = UserAuth PasswdHash deriving (Show, Eq) +instance HasSqlValueSyntax SqliteValueSyntax UserAuth where + sqlValueSyntax (UserAuth (PasswdHash v)) = sqlValueSyntax v + +instance FromBackendRow Sqlite UserAuth where + fromBackendRow = UserAuth . PasswdHash . unpack <$> fromBackendRow + isActiveAccount :: UserStatus -> Bool isActiveAccount (AccountEnabled _) = True isActiveAccount (AccountDisabled _) = True diff --git a/tests/HackageClientUtils.hs b/tests/HackageClientUtils.hs index 53a4b3ec..579a0cc7 100644 --- a/tests/HackageClientUtils.hs +++ b/tests/HackageClientUtils.hs @@ -59,6 +59,7 @@ serverRunningArgs = , "--base-uri", "http://127.0.0.1:" <> show testPort , "--user-content-uri", "http://localhost:" <> show testPort , "--required-base-host-header", "127.0.0.1:" <> show testPort + , "--database-path", "hackageTest.db" -- NOTE: :memory: does not work for the HighLevelTest ] waitForServer :: IO () diff --git a/tests/ReverseDependenciesTest.hs b/tests/ReverseDependenciesTest.hs index fe78692e..b20a2363 100644 --- a/tests/ReverseDependenciesTest.hs +++ b/tests/ReverseDependenciesTest.hs @@ -28,7 +28,7 @@ import Distribution.Server.Features.PreferredVersions.State (PreferredVersions(. import Distribution.Server.Features.ReverseDependencies (ReverseFeature(..), ReverseCount(..), reverseFeature) import Distribution.Server.Features.ReverseDependencies.State (ReverseIndex(..), addPackage, constructReverseIndex, emptyReverseIndex, getDependenciesFlat, getDependencies, getDependenciesFlatRaw, getDependenciesRaw) import Distribution.Server.Features.Tags (Tag(..)) -import Distribution.Server.Features.UserDetails (AccountDetails(..), UserDetailsFeature(..)) +import Distribution.Server.Features.UserDetails (AccountDetails(..), UserDetailsFeature(..), dbQueryUserDetails) import Distribution.Server.Features.UserNotify ( Notification(..) , NotifyMaintainerUpdateType(..) @@ -46,7 +46,8 @@ import Distribution.Server.Framework.BackupRestore (runRestore) import Distribution.Server.Framework.Hook (newHook) import Distribution.Server.Framework.MemState (newMemStateWHNF) import Distribution.Server.Framework.ServerEnv (ServerEnv(..)) -import Distribution.Server.Packages.PackageIndex as PackageIndex +import Distribution.Server.Packages.PackageIndex (PackageIndex) +import qualified Distribution.Server.Packages.PackageIndex as PackageIndex import Distribution.Server.Packages.Types (CabalFileText(..), PkgInfo(..)) import Distribution.Server.Framework.Templating import Distribution.Server.Users.Types @@ -55,11 +56,11 @@ import Distribution.Server.Users.Types , UserId(..) , UserName(..) ) -import Distribution.Server.Users.UserIdSet as UserIdSet +import qualified Distribution.Server.Users.UserIdSet as UserIdSet import qualified Distribution.Server.Users.Users as Users import Distribution.Version (mkVersion, version0) -import Test.Tasty (TestName, TestTree, defaultMain, testGroup) +import Test.Tasty (TestName, TestTree, defaultMain, testGroup, withResource) import Test.Tasty.Golden (goldenVsString) import Test.Tasty.Hedgehog (testProperty) import Test.Tasty.HUnit @@ -74,6 +75,7 @@ import Hedgehog , PropertyT , Range , checkSequential + , evalIO , forAll , property , withTests @@ -81,6 +83,11 @@ import Hedgehog import RevDepCommon (Package(..), TestPackage(..), mkPackage, mkPackageWithCabalFileSuffix, packToPkgInfo) +import Distribution.Server.Features.Database +import Distribution.Server.Features.UserDetails.State +import Database.Beam +import Control.Exception (bracket) + mtlBeelineLens :: [PkgInfo] mtlBeelineLens = [ mkPackage "base" [4,15] [] @@ -326,10 +333,13 @@ getNotificationEmailsTests = testGroup "getNotificationEmails" [ testProperty "All general notifications batched in one email" . withTests 30 . property $ do notifs <- forAll $ Gen.list (Range.linear 1 10) $ Gen.filterT isGeneral genNotification - emails <- liftIO $ getNotificationEmailsMocked $ map (userWatcher,) notifs + emails <- evalIO $ testDatabaseFeature bracket $ \ database -> do + withTransaction database seedDatabase + getNotificationEmailsMocked database $ map (userWatcher,) notifs length emails === 1 - , testGolden "Render NotifyNewVersion" "getNotificationEmails-NotifyNewVersion.golden" $ - fmap renderMail . getNotificationEmailMocked userWatcher $ + , testGolden "Render NotifyNewVersion" "getNotificationEmails-NotifyNewVersion.golden" $ \database -> do + withTransaction database seedDatabase + fmap renderMail . getNotificationEmailMocked database userWatcher $ NotifyNewVersion { notifyPackageInfo = PkgInfo @@ -338,18 +348,20 @@ getNotificationEmailsTests = , pkgTarballRevisions = mempty } } - , testGolden "Render NotifyNewRevision" "getNotificationEmails-NotifyNewRevision.golden" $ do + , testGolden "Render NotifyNewRevision" "getNotificationEmails-NotifyNewRevision.golden" $ \database -> do + withTransaction database seedDatabase let mkRev rev = (CabalFileText "", (rev, userActor)) rev0 = (0 * Time.nominalDay) `Time.addUTCTime` timestamp rev1 = (1 * Time.nominalDay) `Time.addUTCTime` timestamp rev2 = (2 * Time.nominalDay) `Time.addUTCTime` timestamp - fmap renderMail . getNotificationEmailMocked userWatcher $ + fmap renderMail . getNotificationEmailMocked database userWatcher $ NotifyNewRevision { notifyPackageId = PackageIdentifier "base" (mkVersion [4, 18, 0, 0]) , notifyRevisions = map (, userActor) [rev1, rev2] } - , testGolden "Render NotifyMaintainerUpdate-MaintainerAdded" "getNotificationEmails-NotifyMaintainerUpdate-MaintainerAdded.golden" $ - fmap renderMail . getNotificationEmailMocked userWatcher $ + , testGolden "Render NotifyMaintainerUpdate-MaintainerAdded" "getNotificationEmails-NotifyMaintainerUpdate-MaintainerAdded.golden" $ \database -> do + withTransaction database seedDatabase + fmap renderMail . getNotificationEmailMocked database userWatcher $ NotifyMaintainerUpdate { notifyMaintainerUpdateType = MaintainerAdded , notifyUserActor = userActor @@ -358,8 +370,9 @@ getNotificationEmailsTests = , notifyReason = "User is cool" , notifyUpdatedAt = timestamp } - , testGolden "Render NotifyMaintainerUpdate-MaintainerRemoved" "getNotificationEmails-NotifyMaintainerUpdate-MaintainerRemoved.golden" $ - fmap renderMail . getNotificationEmailMocked userWatcher $ + , testGolden "Render NotifyMaintainerUpdate-MaintainerRemoved" "getNotificationEmails-NotifyMaintainerUpdate-MaintainerRemoved.golden" $ \database -> do + withTransaction database seedDatabase + fmap renderMail . getNotificationEmailMocked database userWatcher $ NotifyMaintainerUpdate { notifyMaintainerUpdateType = MaintainerRemoved , notifyUserActor = userActor @@ -368,29 +381,34 @@ getNotificationEmailsTests = , notifyReason = "User is no longer cool" , notifyUpdatedAt = timestamp } - , testGolden "Render NotifyDocsBuild-success" "getNotificationEmails-NotifyDocsBuild-success.golden" $ - fmap renderMail . getNotificationEmailMocked userWatcher $ + , testGolden "Render NotifyDocsBuild-success" "getNotificationEmails-NotifyDocsBuild-success.golden" $ \database -> do + withTransaction database seedDatabase + fmap renderMail . getNotificationEmailMocked database userWatcher $ NotifyDocsBuild { notifyPackageId = PackageIdentifier "base" (mkVersion [4, 18, 0, 0]) , notifyBuildSuccess = True } - , testGolden "Render NotifyDocsBuild-failure" "getNotificationEmails-NotifyDocsBuild-failure.golden" $ - fmap renderMail . getNotificationEmailMocked userWatcher $ + , testGolden "Render NotifyDocsBuild-failure" "getNotificationEmails-NotifyDocsBuild-failure.golden" $ \database -> do + withTransaction database seedDatabase + fmap renderMail . getNotificationEmailMocked database userWatcher $ NotifyDocsBuild { notifyPackageId = PackageIdentifier "base" (mkVersion [4, 18, 0, 0]) , notifyBuildSuccess = False } - , testGolden "Render NotifyUpdateTags" "getNotificationEmails-NotifyUpdateTags.golden" $ - fmap renderMail . getNotificationEmailMocked userWatcher $ + , testGolden "Render NotifyUpdateTags" "getNotificationEmails-NotifyUpdateTags.golden" $ \database -> do + withTransaction database seedDatabase + fmap renderMail . getNotificationEmailMocked database userWatcher $ NotifyUpdateTags { notifyPackageName = "base" , notifyAddedTags = Set.fromList . map Tag $ ["bsd3", "library", "prelude"] , notifyDeletedTags = Set.fromList . map Tag $ ["example", "bad", "foo"] } - , testGolden "Render NotifyDependencyUpdate-Always" "getNotificationEmails-NotifyDependencyUpdate-Always.golden" $ + , testGolden "Render NotifyDependencyUpdate-Always" "getNotificationEmails-NotifyDependencyUpdate-Always.golden" $ \database -> do + withTransaction database seedDatabase fmap renderMail . getNotificationEmail testServerEnv + database testUserDetailsFeature allUsers userWatcher @@ -399,10 +417,12 @@ getNotificationEmailsTests = , notifyWatchedPackages = [PackageIdentifier "mtl" (mkVersion [2, 3])] , notifyTriggerBounds = Always } - , testGolden "Render NotifyDependencyUpdate-NewIncompatibility" "getNotificationEmails-NotifyDependencyUpdate-NewIncompatibility.golden" $ + , testGolden "Render NotifyDependencyUpdate-NewIncompatibility" "getNotificationEmails-NotifyDependencyUpdate-NewIncompatibility.golden" $ \database -> do + withTransaction database seedDatabase fmap renderMail . getNotificationEmail testServerEnv + database testUserDetailsFeature allUsers userWatcher @@ -411,10 +431,12 @@ getNotificationEmailsTests = , notifyWatchedPackages = [PackageIdentifier "mtl" (mkVersion [2, 3])] , notifyTriggerBounds = NewIncompatibility } - , testGolden "Render NotifyDependencyUpdate-BoundsOutOfRange" "getNotificationEmails-NotifyDependencyUpdate-BoundsOutOfRange.golden" $ + , testGolden "Render NotifyDependencyUpdate-BoundsOutOfRange" "getNotificationEmails-NotifyDependencyUpdate-BoundsOutOfRange.golden" $ \database -> do + withTransaction database seedDatabase fmap renderMail . getNotificationEmail testServerEnv + database testUserDetailsFeature allUsers userWatcher @@ -423,11 +445,13 @@ getNotificationEmailsTests = , notifyWatchedPackages = [PackageIdentifier "mtl" (mkVersion [2, 3])] , notifyTriggerBounds = BoundsOutOfRange } - , testGolden "Render NotifyVouchingCompleted" "getNotificationEmails-NotifyVouchingCompleted.golden" $ - fmap renderMail $ getNotificationEmailMocked userWatcher NotifyVouchingCompleted - , testGolden "Render general notifications in single batched email" "getNotificationEmails-batched.golden" $ do + , testGolden "Render NotifyVouchingCompleted" "getNotificationEmails-NotifyVouchingCompleted.golden" $ \database -> do + withTransaction database seedDatabase + fmap renderMail $ getNotificationEmailMocked database userWatcher NotifyVouchingCompleted + , testGolden "Render general notifications in single batched email" "getNotificationEmails-batched.golden" $ \database -> do + withTransaction database seedDatabase emails <- - getNotificationEmailsMocked . map (userWatcher,) $ + getNotificationEmailsMocked database . map (userWatcher,) $ [ NotifyNewRevision { notifyPackageId = PackageIdentifier "base" (mkVersion [4, 18, 0, 0]) , notifyRevisions = [(timestamp, userActor)] @@ -483,8 +507,22 @@ getNotificationEmailsTests = <*> addUser "user-actor" <*> addUser "user-subject" - getNotificationEmail env details users uid notif = - getNotificationEmails env details users (mockTemplates ["datafiles/templates/UserNotify"] ["endorsements-complete.txt"]) [(uid, notif)] >>= \case + seedDatabase :: Transaction () + seedDatabase = do + Distribution.Server.Features.Database.runInsert $ + insert (_tblAccountDetails hackageDb) $ + insertValues + [ AccountDetailsRow + { _adUserId = userWatcher + , _adName = "user-watcher" + , _adContactEmail = "user-watcher@example.com" + , _adKind = Nothing + , _adAdminNotes = "" + } + ] + + getNotificationEmail env database details users uid notif = + getNotificationEmails env database details users (mockTemplates ["datafiles/templates/UserNotify"] ["endorsements-complete.txt"]) [(uid, notif)] >>= \case [email] -> pure email _ -> error "Did not get exactly one email" @@ -492,28 +530,24 @@ getNotificationEmailsTests = ServerEnv { serverBaseURI = fromJust $ parseURI "https://hackage.haskell.org" } + testUserDetailsFeature = UserDetailsFeature - { queryUserDetails = \uid -> - pure $ do - guard $ uid == userWatcher - Just - AccountDetails - { accountName = "user-watcher" - , accountContactEmail = "user-watcher@example.com" - , accountKind = Nothing - , accountAdminNotes = "" - } + { userDetailsFeatureInterface = undefined, + queryUserDetails = dbQueryUserDetails, + updateUserDetails = undefined } - getNotificationEmailsMocked = + getNotificationEmailsMocked database = getNotificationEmails testServerEnv + database testUserDetailsFeature allUsers (mockTemplates ["datafiles/templates/UserNotify"] ["endorsements-complete.txt"]) - getNotificationEmailMocked = + getNotificationEmailMocked database = getNotificationEmail testServerEnv + database testUserDetailsFeature allUsers @@ -665,8 +699,14 @@ hedgehogTests = , ("prop_csvBackupRoundtrips", prop_csvBackupRoundtrips) ] -testGolden :: TestName -> FilePath -> IO Lazy.ByteString -> TestTree -testGolden name fp = goldenVsString name ("tests/golden/ReverseDependenciesTest/" <> fp) +testGolden :: TestName -> FilePath -> (DatabaseFeature -> IO Lazy.ByteString) -> TestTree +testGolden name fp body = + testDatabaseFeatureIO withResource $ \ioDatabase -> + goldenVsString name ("tests/golden/ReverseDependenciesTest/" <> fp) (do + database <- ioDatabase + body database + ) + main :: IO () main = defaultMain allTests