Disable poolboy (#783)

* Disable poolboy

* Add changelog entry

app/cli/Main.hs

@@ -3,7 +3,6 @@ module Main where
 import Codec.Compression.GZip qualified as GZip
 import Data.ByteString.Lazy.Char8 qualified as BS
 import Data.Maybe
-import Data.Poolboy (poolboySettingsWith)
 import Data.Text (Text)
 import Data.Text qualified as Text
 import Data.Text.Display (display)
@@ -24,7 +23,6 @@ import Options.Applicative
 import Sel.Hashing.Password qualified as Sel
 import System.FilePath ((</>))
 
-import Effectful.Poolboy
 import Flora.Environment
 import Flora.Import.Categories (importCategories)
 import Flora.Import.Package.Bulk (importAllFilesInRelativeDirectory, importFromIndex)
@@ -77,7 +75,6 @@ main = Log.withStdOutLogger $ \logger -> do
     . runDB env.pool
     . runFailIO
     . runTime
-    . runPoolboy (poolboySettingsWith capabilities)
     . ( case env.features.blobStoreImpl of
           Just (BlobStoreFS fp) -> runBlobStoreFS fp
           _ -> runBlobStorePure
@@ -163,7 +160,6 @@ runOptions
      , Fail :> es
      , IOE :> es
      , BlobStoreAPI :> es
-     , Poolboy :> es
      )
   => Options
   -> Eff es ()
@@ -202,7 +198,6 @@ importFolderOfCabalFiles
   :: ( FileSystem :> es
      , Time :> es
      , Log :> es
-     , Poolboy :> es
      , DB :> es
      , IOE :> es
      )
@@ -220,7 +215,6 @@ importFolderOfCabalFiles path repository = do
 importIndex
   :: ( Time :> es
      , Log :> es
-     , Poolboy :> es
      , DB :> es
      , IOE :> es
      )

changelog.d/783

@@ -0,0 +1,5 @@
+synopsis: Disable poolboy when inserting components
+description: {
+  The queueing for parallel insertion seems to be at the source of problems when inserting large numbers of packages
+}
+prs: #783

flora.cabal

@@ -441,7 +441,6 @@ executable flora-cli
     , optics-core
     , optparse-applicative
     , pg-transact-effectful
-    , poolboy
     , PyF
     , sel
     , text
@@ -480,7 +479,6 @@ test-suite flora-test
     , pg-entity
     , pg-transact
     , pg-transact-effectful
-    , poolboy
     , postgresql-migration
     , postgresql-simple
     , resource-pool

src/core/Flora/Environment.hs

@@ -50,6 +50,7 @@ data TestEnv = TestEnv
   { pool :: Pool PG.Connection
   , dbConfig :: PoolConfig
   , httpPort :: Word16
+  , mltp :: MLTP
   }
   deriving stock (Generic)
 

src/core/Flora/Environment/Config.hs

@@ -138,6 +138,7 @@ data TestConfig = TestConfig
   { httpPort :: Word16
   , dbConfig :: PoolConfig
   , connectionInfo :: ByteString
+  , mltp :: MLTP
   }
   deriving stock (Generic)
 
@@ -200,6 +201,7 @@ parseTestConfig =
     <$> parsePort
     <*> parsePoolConfig
     <*> parseConnectionInfo
+    <*> parseMLTP
 
 -- Env parser helpers
 

src/core/Flora/Import/Package.hs

@@ -84,8 +84,6 @@ import Distribution.Utils.ShortText qualified as Cabal
 import Distribution.Version qualified as Version
 import Effectful
 import Effectful.Log (Log)
-import Effectful.Poolboy (Poolboy)
-import Effectful.Poolboy qualified as Poolboy
 import Effectful.PostgreSQL.Transact.Effect (DB)
 import Effectful.Time (Time)
 import Effectful.Time qualified as Time
@@ -267,7 +265,7 @@ parseString parser name bs = do
 
 -- | Persists an 'ImportOutput' to the database. An 'ImportOutput' can be obtained
 --  by extracting relevant information from a Cabal file using 'extractPackageDataFromCabal'
-persistImportOutput :: forall es. (Log :> es, Poolboy :> es, DB :> es, IOE :> es) => ImportOutput -> Eff es ()
+persistImportOutput :: forall es. (Log :> es, DB :> es, IOE :> es) => ImportOutput -> Eff es ()
 persistImportOutput (ImportOutput package categories release components) = do
   Log.logInfo "Persisting package" $
     object
@@ -276,10 +274,8 @@ persistImportOutput (ImportOutput package categories release components) = do
       ]
   persistPackage
   Update.upsertRelease release
-  parallelRun persistComponent components
+  forM_ components persistComponent
   where
-    parallelRun :: Foldable t => (a -> Eff es ()) -> t a -> Eff es ()
-    parallelRun f xs = forM_ xs (Poolboy.enqueue . f)
     packageName = display package.namespace <> "/" <> display package.name
     persistPackage = do
       let packageId = package.packageId
@@ -295,7 +291,7 @@ persistImportOutput (ImportOutput package categories release components) = do
           , "number_of_dependencies" .= display (length deps)
           ]
       Update.upsertPackageComponent packageComponent
-      parallelRun persistImportDependency deps
+      forM_ deps persistImportDependency
 
     persistImportDependency :: ImportDependency -> Eff es ()
     persistImportDependency dep = do

src/core/Flora/Import/Package/Bulk.hs

@@ -31,7 +31,6 @@ import Effectful.FileSystem qualified as FileSystem
 import Effectful.FileSystem.IO.ByteString qualified as FileSystem
 import Effectful.Log (Log)
 import Effectful.Log qualified as Log
-import Effectful.Poolboy
 import Effectful.PostgreSQL.Transact.Effect (DB)
 import Effectful.Time (Time)
 import GHC.Conc (numCapabilities)
@@ -67,7 +66,12 @@ import Flora.Model.User
 
 -- | Same as 'importAllFilesInDirectory' but accepts a relative path to the current working directory
 importAllFilesInRelativeDirectory
-  :: (Log :> es, Time :> es, FileSystem :> es, Poolboy :> es, DB :> es, IOE :> es)
+  :: ( Log :> es
+     , Time :> es
+     , FileSystem :> es
+     , DB :> es
+     , IOE :> es
+     )
   => UserId
   -> (Text, Text)
   -> FilePath
@@ -77,7 +81,7 @@ importAllFilesInRelativeDirectory user (repositoryName, repositoryURL) dir = do
   importAllFilesInDirectory user (repositoryName, repositoryURL) workdir
 
 importFromIndex
-  :: (Time :> es, Log :> es, Poolboy :> es, DB :> es, IOE :> es)
+  :: (Time :> es, Log :> es, DB :> es, IOE :> es)
   => UserId
   -> Text
   -> FilePath
@@ -126,7 +130,12 @@ importFromIndex user repositoryName index = do
 
 -- | Finds all cabal files in the specified directory, and inserts them into the database after extracting the relevant data
 importAllFilesInDirectory
-  :: (Time :> es, Log :> es, FileSystem :> es, Poolboy :> es, DB :> es, IOE :> es)
+  :: ( Time :> es
+     , Log :> es
+     , FileSystem :> es
+     , DB :> es
+     , IOE :> es
+     )
   => UserId
   -> (Text, Text)
   -> FilePath
@@ -139,7 +148,7 @@ importAllFilesInDirectory user (repositoryName, repositoryURL) dir = do
 
 importFromStream
   :: forall es
-   . (Time :> es, Log :> es, Poolboy :> es, DB :> es, IOE :> es)
+   . (Time :> es, Log :> es, DB :> es, IOE :> es)
   => UserId
   -> (Text, Set PackageName)
   -> Stream (Eff es) (ImportFileType, UTCTime, StrictByteString)

src/jobs-worker/FloraJobs/Runner.hs

@@ -15,7 +15,6 @@ import Effectful (Eff, IOE, type (:>))
 import Effectful.FileSystem (FileSystem)
 import Effectful.FileSystem qualified as FileSystem
 import Effectful.Log
-import Effectful.Poolboy (Poolboy)
 import Effectful.PostgreSQL.Transact.Effect (DB)
 import Effectful.Process.Typed
 import Effectful.Reader.Static (Reader)
@@ -224,7 +223,6 @@ refreshIndexes
      , DB :> es
      , TypedProcess :> es
      , Log :> es
-     , Poolboy :> es
      , IOE :> es
      , FileSystem :> es
      )

test/Flora/TestUtils.hs

@@ -85,7 +85,6 @@ import Data.List qualified as List
 import Data.List.NonEmpty qualified as NE
 import Data.Maybe (fromJust)
 import Data.Pool hiding (PoolConfig)
-import Data.Poolboy (poolboySettingsWith)
 import Data.Text (Text)
 import Data.Text.Lazy qualified as Text
 import Data.Time (UTCTime (UTCTime), fromGregorian, secondsToDiffTime)
@@ -109,7 +108,6 @@ import Effectful.Fail (Fail, runFailIO)
 import Effectful.FileSystem
 import Effectful.Log (Log, Logger)
 import Effectful.Log qualified as Log
-import Effectful.Poolboy
 import Effectful.PostgreSQL.Transact.Effect
 import Effectful.Reader.Static
 import Effectful.Time
@@ -172,7 +170,7 @@ import Flora.Model.User.Update qualified as Update
 import Flora.Publish
 import FloraWeb.Client
 
-type TestEff = Eff '[FileSystem, Poolboy, Fail, BlobStoreAPI, Reader PoolConfig, DB, Log, Time, IOE]
+type TestEff = Eff '[FileSystem, Fail, BlobStoreAPI, Reader TestEnv, DB, Log, Time, IOE]
 
 data Fixtures = Fixtures
   { hackageUser :: User
@@ -197,25 +195,24 @@ importAllPackages fixtures = do
 
   liftIO $ threadDelay 20000
 
-runTestEff :: TestEff a -> Pool Connection -> PoolConfig -> IO a
-runTestEff comp pool poolCfg = runEff $
-  Log.withStdOutLogger $ \stdOutLogger ->
+runTestEff :: TestEff a -> TestEnv -> IO a
+runTestEff comp env = runEff $ do
+  let withLogger = Logging.makeLogger env.mltp.logger
+  withLogger $ \logger ->
     runTime
       . withUnliftStrategy (ConcUnlift Ephemeral Unlimited)
-      . Log.runLog "flora-test" stdOutLogger LogAttention
-      . runDB pool
-      . runReader poolCfg
+      . Log.runLog "flora-test" logger LogInfo
+      . runDB env.pool
+      . runReader env
       . runBlobStorePure
       . runFailIO
-      . runPoolboy (poolboySettingsWith poolCfg.connections)
       . runFileSystem
       $ comp
 
 testThis :: String -> TestEff () -> TestEff TestTree
 testThis name assertion = do
-  pool <- getPool
-  poolCfg <- ask @PoolConfig
-  let test = runTestEff assertion pool poolCfg
+  env <- ask @TestEnv
+  let test = runTestEff assertion env
   pure $ Test.testCase name test
 
 testThese :: String -> [TestEff TestTree] -> TestEff TestTree

test/Main.hs

@@ -43,9 +43,8 @@ main = do
           importAllPackages f'
           pure f'
       )
-      env.pool
-      env.dbConfig
-  spec <- traverse (\comp -> runTestEff comp env.pool env.dbConfig) (specs fixtures)
+      env
+  spec <- traverse (\comp -> runTestEff comp env) (specs fixtures)
   defaultMain . testGroup "Flora Tests" $ OddJobSpec.spec : spec
 
 specs :: Fixtures -> [TestEff TestTree]