Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Disable poolboy #783

Merged
merged 2 commits into from
Nov 6, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Disable poolboy
tchoutri committed Nov 6, 2024
commit ce553e945f3aa89c8b8942862b624c6dd84ab14a
6 changes: 0 additions & 6 deletions app/cli/Main.hs
Original file line number Diff line number Diff line change
@@ -3,7 +3,6 @@ module Main where
import Codec.Compression.GZip qualified as GZip
import Data.ByteString.Lazy.Char8 qualified as BS
import Data.Maybe
import Data.Poolboy (poolboySettingsWith)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Display (display)
@@ -24,7 +23,6 @@ import Options.Applicative
import Sel.Hashing.Password qualified as Sel
import System.FilePath ((</>))

import Effectful.Poolboy
import Flora.Environment
import Flora.Import.Categories (importCategories)
import Flora.Import.Package.Bulk (importAllFilesInRelativeDirectory, importFromIndex)
@@ -77,7 +75,6 @@ main = Log.withStdOutLogger $ \logger -> do
. runDB env.pool
. runFailIO
. runTime
. runPoolboy (poolboySettingsWith capabilities)
. ( case env.features.blobStoreImpl of
Just (BlobStoreFS fp) -> runBlobStoreFS fp
_ -> runBlobStorePure
@@ -163,7 +160,6 @@ runOptions
, Fail :> es
, IOE :> es
, BlobStoreAPI :> es
, Poolboy :> es
)
=> Options
-> Eff es ()
@@ -202,7 +198,6 @@ importFolderOfCabalFiles
:: ( FileSystem :> es
, Time :> es
, Log :> es
, Poolboy :> es
, DB :> es
, IOE :> es
)
@@ -220,7 +215,6 @@ importFolderOfCabalFiles path repository = do
importIndex
:: ( Time :> es
, Log :> es
, Poolboy :> es
, DB :> es
, IOE :> es
)
2 changes: 0 additions & 2 deletions flora.cabal
Original file line number Diff line number Diff line change
@@ -441,7 +441,6 @@ executable flora-cli
, optics-core
, optparse-applicative
, pg-transact-effectful
, poolboy
, PyF
, sel
, text
@@ -480,7 +479,6 @@ test-suite flora-test
, pg-entity
, pg-transact
, pg-transact-effectful
, poolboy
, postgresql-migration
, postgresql-simple
, resource-pool
1 change: 1 addition & 0 deletions src/core/Flora/Environment.hs
Original file line number Diff line number Diff line change
@@ -50,6 +50,7 @@ data TestEnv = TestEnv
{ pool :: Pool PG.Connection
, dbConfig :: PoolConfig
, httpPort :: Word16
, mltp :: MLTP
}
deriving stock (Generic)

2 changes: 2 additions & 0 deletions src/core/Flora/Environment/Config.hs
Original file line number Diff line number Diff line change
@@ -138,6 +138,7 @@ data TestConfig = TestConfig
{ httpPort :: Word16
, dbConfig :: PoolConfig
, connectionInfo :: ByteString
, mltp :: MLTP
}
deriving stock (Generic)

@@ -200,6 +201,7 @@ parseTestConfig =
<$> parsePort
<*> parsePoolConfig
<*> parseConnectionInfo
<*> parseMLTP

-- Env parser helpers

10 changes: 3 additions & 7 deletions src/core/Flora/Import/Package.hs
Original file line number Diff line number Diff line change
@@ -84,8 +84,6 @@ import Distribution.Utils.ShortText qualified as Cabal
import Distribution.Version qualified as Version
import Effectful
import Effectful.Log (Log)
import Effectful.Poolboy (Poolboy)
import Effectful.Poolboy qualified as Poolboy
import Effectful.PostgreSQL.Transact.Effect (DB)
import Effectful.Time (Time)
import Effectful.Time qualified as Time
@@ -267,7 +265,7 @@ parseString parser name bs = do

-- | Persists an 'ImportOutput' to the database. An 'ImportOutput' can be obtained
-- by extracting relevant information from a Cabal file using 'extractPackageDataFromCabal'
persistImportOutput :: forall es. (Log :> es, Poolboy :> es, DB :> es, IOE :> es) => ImportOutput -> Eff es ()
persistImportOutput :: forall es. (Log :> es, DB :> es, IOE :> es) => ImportOutput -> Eff es ()
persistImportOutput (ImportOutput package categories release components) = do
Log.logInfo "Persisting package" $
object
@@ -276,10 +274,8 @@ persistImportOutput (ImportOutput package categories release components) = do
]
persistPackage
Update.upsertRelease release
parallelRun persistComponent components
forM_ components persistComponent
where
parallelRun :: Foldable t => (a -> Eff es ()) -> t a -> Eff es ()
parallelRun f xs = forM_ xs (Poolboy.enqueue . f)
packageName = display package.namespace <> "/" <> display package.name
persistPackage = do
let packageId = package.packageId
@@ -295,7 +291,7 @@ persistImportOutput (ImportOutput package categories release components) = do
, "number_of_dependencies" .= display (length deps)
]
Update.upsertPackageComponent packageComponent
parallelRun persistImportDependency deps
forM_ deps persistImportDependency

persistImportDependency :: ImportDependency -> Eff es ()
persistImportDependency dep = do
21 changes: 15 additions & 6 deletions src/core/Flora/Import/Package/Bulk.hs
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ import Codec.Archive.Tar qualified as Tar
import Codec.Archive.Tar.Entry qualified as Tar
import Codec.Archive.Tar.Index qualified as Tar
import Codec.Compression.GZip qualified as GZip
import GHC.Conc (numCapabilities )
import Control.Monad (when, (>=>))
import Data.Aeson
import Data.ByteString (StrictByteString)
@@ -31,10 +32,8 @@ import Effectful.FileSystem qualified as FileSystem
import Effectful.FileSystem.IO.ByteString qualified as FileSystem
import Effectful.Log (Log)
import Effectful.Log qualified as Log
import Effectful.Poolboy
import Effectful.PostgreSQL.Transact.Effect (DB)
import Effectful.Time (Time)
import GHC.Conc (numCapabilities)
import Streamly.Data.Fold qualified as SFold
import Streamly.Data.Stream (Stream)
import Streamly.Data.Stream.Prelude (maxThreads, ordered)
@@ -67,7 +66,12 @@ import Flora.Model.User

-- | Same as 'importAllFilesInDirectory' but accepts a relative path to the current working directory
importAllFilesInRelativeDirectory
:: (Log :> es, Time :> es, FileSystem :> es, Poolboy :> es, DB :> es, IOE :> es)
:: ( Log :> es
, Time :> es
, FileSystem :> es
, DB :> es
, IOE :> es
)
=> UserId
-> (Text, Text)
-> FilePath
@@ -77,7 +81,7 @@ importAllFilesInRelativeDirectory user (repositoryName, repositoryURL) dir = do
importAllFilesInDirectory user (repositoryName, repositoryURL) workdir

importFromIndex
:: (Time :> es, Log :> es, Poolboy :> es, DB :> es, IOE :> es)
:: (Time :> es, Log :> es, DB :> es, IOE :> es)
=> UserId
-> Text
-> FilePath
@@ -126,7 +130,12 @@ importFromIndex user repositoryName index = do

-- | Finds all cabal files in the specified directory, and inserts them into the database after extracting the relevant data
importAllFilesInDirectory
:: (Time :> es, Log :> es, FileSystem :> es, Poolboy :> es, DB :> es, IOE :> es)
:: ( Time :> es
, Log :> es
, FileSystem :> es
, DB :> es
, IOE :> es
)
=> UserId
-> (Text, Text)
-> FilePath
@@ -139,7 +148,7 @@ importAllFilesInDirectory user (repositoryName, repositoryURL) dir = do

importFromStream
:: forall es
. (Time :> es, Log :> es, Poolboy :> es, DB :> es, IOE :> es)
. (Time :> es, Log :> es, DB :> es, IOE :> es)
=> UserId
-> (Text, Set PackageName)
-> Stream (Eff es) (ImportFileType, UTCTime, StrictByteString)
2 changes: 0 additions & 2 deletions src/jobs-worker/FloraJobs/Runner.hs
Original file line number Diff line number Diff line change
@@ -15,7 +15,6 @@ import Effectful (Eff, IOE, type (:>))
import Effectful.FileSystem (FileSystem)
import Effectful.FileSystem qualified as FileSystem
import Effectful.Log
import Effectful.Poolboy (Poolboy)
import Effectful.PostgreSQL.Transact.Effect (DB)
import Effectful.Process.Typed
import Effectful.Reader.Static (Reader)
@@ -224,7 +223,6 @@ refreshIndexes
, DB :> es
, TypedProcess :> es
, Log :> es
, Poolboy :> es
, IOE :> es
, FileSystem :> es
)
23 changes: 10 additions & 13 deletions test/Flora/TestUtils.hs
Original file line number Diff line number Diff line change
@@ -85,7 +85,6 @@ import Data.List qualified as List
import Data.List.NonEmpty qualified as NE
import Data.Maybe (fromJust)
import Data.Pool hiding (PoolConfig)
import Data.Poolboy (poolboySettingsWith)
import Data.Text (Text)
import Data.Text.Lazy qualified as Text
import Data.Time (UTCTime (UTCTime), fromGregorian, secondsToDiffTime)
@@ -109,7 +108,6 @@ import Effectful.Fail (Fail, runFailIO)
import Effectful.FileSystem
import Effectful.Log (Log, Logger)
import Effectful.Log qualified as Log
import Effectful.Poolboy
import Effectful.PostgreSQL.Transact.Effect
import Effectful.Reader.Static
import Effectful.Time
@@ -172,7 +170,7 @@ import Flora.Model.User.Update qualified as Update
import Flora.Publish
import FloraWeb.Client

type TestEff = Eff '[FileSystem, Poolboy, Fail, BlobStoreAPI, Reader PoolConfig, DB, Log, Time, IOE]
type TestEff = Eff '[FileSystem, Fail, BlobStoreAPI, Reader TestEnv, DB, Log, Time, IOE]

data Fixtures = Fixtures
{ hackageUser :: User
@@ -197,25 +195,24 @@ importAllPackages fixtures = do

liftIO $ threadDelay 20000

runTestEff :: TestEff a -> Pool Connection -> PoolConfig -> IO a
runTestEff comp pool poolCfg = runEff $
Log.withStdOutLogger $ \stdOutLogger ->
runTestEff :: TestEff a -> TestEnv -> IO a
runTestEff comp env = runEff $ do
let withLogger = Logging.makeLogger env.mltp.logger
withLogger $ \logger ->
runTime
. withUnliftStrategy (ConcUnlift Ephemeral Unlimited)
. Log.runLog "flora-test" stdOutLogger LogAttention
. runDB pool
. runReader poolCfg
. Log.runLog "flora-test" logger LogInfo
. runDB env.pool
. runReader env
. runBlobStorePure
. runFailIO
. runPoolboy (poolboySettingsWith poolCfg.connections)
. runFileSystem
$ comp

testThis :: String -> TestEff () -> TestEff TestTree
testThis name assertion = do
pool <- getPool
poolCfg <- ask @PoolConfig
let test = runTestEff assertion pool poolCfg
env <- ask @TestEnv
let test = runTestEff assertion env
pure $ Test.testCase name test

testThese :: String -> [TestEff TestTree] -> TestEff TestTree
5 changes: 2 additions & 3 deletions test/Main.hs
Original file line number Diff line number Diff line change
@@ -43,9 +43,8 @@ main = do
importAllPackages f'
pure f'
)
env.pool
env.dbConfig
spec <- traverse (\comp -> runTestEff comp env.pool env.dbConfig) (specs fixtures)
env
spec <- traverse (\comp -> runTestEff comp env) (specs fixtures)
defaultMain . testGroup "Flora Tests" $ OddJobSpec.spec : spec

specs :: Fixtures -> [TestEff TestTree]