[FLORA-442] Implement package import from multiple repos (#444)

.github/workflows/backend.yml

@@ -101,6 +101,6 @@ jobs:
         migrate init "${FLORA_DB_CONNSTRING}"
         migrate migrate "${FLORA_DB_CONNSTRING}" migrations
         cabal run -- flora-cli create-user --username "hackage-user" --email "[email protected]" --password "foobar2000"
-        cabal test
+        make test
       env:
         PGPASSWORD: "postgres"

.gitignore

@@ -9,7 +9,7 @@
 .hspec-failures
 .vscode/
 /.stack-work/
-01-index
+01-index*
 <
 Session.vim
 _data/

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 1.0.14 -- XXXX-XX-XX
 * Added more matches to the the natural language processing catergory ([#44](https://github.com/flora-pm/flora-server/pull/440))
 * Colourise in red deprecation markers on the package page ([#438](https://github.com/flora-pm/flora-server/pull/439))
+* Allow package imports from multiple repositories ([#444](https://github.com/flora-pm/flora-server/pull/444))
 
 ## 1.0.13 -- 2023-09-17
 * Exclude deprecated releases from latest versions and search ([#373](https://github.com/flora-pm/flora-server/pull/373))

Makefile

@@ -42,11 +42,13 @@ db-migrate: ## Apply database migrations
 
 db-reset: db-drop db-setup db-provision ## Reset the dev database
 
-db-provision: build ## Load the development data in the database
+db-provision: ## Create categories and repositories
 	@cabal run -- flora-cli create-user --username "hackage-user" --email "[email protected]" --password "foobar2000"
 	@cabal run -- flora-cli provision categories
+	@cabal run -- flora-cli provision-repository --name "hackage" --url https://hackage.haskell.org
+	@cabal run -- flora-cli provision-repository --name "cardano" --url https://input-output-hk.github.io/cardano-haskell-packages
 
-db-provision-test-packages:
+db-provision-test-packages: ## Load development data in the database
 	@cabal run -- flora-cli provision test-packages
 
 import-from-hackage: ## Imports every cabal file from the ./index-01 directory

app/cli/Main.hs

@@ -3,12 +3,12 @@ module Main where
 import Data.Maybe
 import Data.Password.Types
 import Data.Text (Text)
+import Data.Text qualified as Text
 import DesignSystem (generateComponents)
 import Effectful
 import Effectful.Fail
 import Effectful.PostgreSQL.Transact.Effect
 import Effectful.Reader.Static (Reader, runReader)
-import Flora.Model.User.Query qualified as Query
 import GHC.Generics (Generic)
 import Log.Backend.StandardOutput qualified as Log
 import Optics.Core
@@ -18,7 +18,11 @@ import Flora.Environment
 import Flora.Environment.Config (PoolConfig (..))
 import Flora.Import.Categories (importCategories)
 import Flora.Import.Package.Bulk (importAllFilesInRelativeDirectory, importFromIndex)
+import Flora.Model.PackageIndex.Query qualified as Query
+import Flora.Model.PackageIndex.Types
+import Flora.Model.PackageIndex.Update qualified as Update
 import Flora.Model.User
+import Flora.Model.User.Query qualified as Query
 import Flora.Model.User.Update
 
 data Options = Options
@@ -30,8 +34,9 @@ data Command
   = Provision ProvisionTarget
   | CreateUser UserCreationOptions
   | GenDesignSystemComponents
-  | ImportPackages FilePath (Maybe Text)
-  | ImportIndex FilePath (Maybe Text)
+  | ImportPackages FilePath Text
+  | ImportIndex FilePath Text
+  | ProvisionRepository Text Text
   deriving stock (Show, Eq)
 
 data ProvisionTarget
@@ -70,6 +75,7 @@ parseCommand =
       <> command "gen-design-system" (parseGenDesignSystem `withInfo` "Generate Design System components from the code")
       <> command "import-packages" (parseImportPackages `withInfo` "Import cabal packages from a directory")
       <> command "import-index" (parseImportIndex `withInfo` "Import cabal packages from the index tarball")
+      <> command "provision-repository" (parseProvisionRepository `withInfo` "Create a package repository")
 
 parseProvision :: Parser Command
 parseProvision =
@@ -95,27 +101,23 @@ parseImportPackages :: Parser Command
 parseImportPackages =
   ImportPackages
     <$> argument str (metavar "PATH")
-    <*> optional
-      ( strOption $
-          long "repository"
-            <> metavar "<repository>"
-            <> help "Which repository we're importing from"
-      )
+    <*> option str (long "repository" <> metavar "<repository>" <> help "Which repository we're importing from (hackage, cardano…)")
 
 parseImportIndex :: Parser Command
 parseImportIndex =
   ImportIndex
     <$> argument str (metavar "PATH")
-    <*> optional
-      ( strOption $
-          long "repository"
-            <> metavar "<repository>"
-            <> help "Which repository we're importing from"
-      )
+    <*> option str (long "repository" <> metavar "<repository>" <> help "Which repository we're importing from (hackage, cardano…)")
+
+parseProvisionRepository :: Parser Command
+parseProvisionRepository =
+  ProvisionRepository
+    <$> option str (long "name" <> metavar "<repository name>" <> help "Name of the repository")
+    <*> option str (long "url" <> metavar "<repository url>" <> help "Link to the package repository")
 
 runOptions :: (Reader PoolConfig :> es, DB :> es, Fail :> es, IOE :> es) => Options -> Eff es ()
 runOptions (Options (Provision Categories)) = importCategories
-runOptions (Options (Provision TestPackages)) = importFolderOfCabalFiles "./test/fixtures/Cabal/" Nothing
+runOptions (Options (Provision TestPackages)) = importFolderOfCabalFiles "./test/fixtures/Cabal/" "hackage"
 runOptions (Options (CreateUser opts)) = do
   let username = opts ^. #username
       email = opts ^. #email
@@ -135,16 +137,29 @@ runOptions (Options (CreateUser opts)) = do
 runOptions (Options GenDesignSystemComponents) = generateComponents
 runOptions (Options (ImportPackages path repository)) = importFolderOfCabalFiles path repository
 runOptions (Options (ImportIndex path repository)) = importIndex path repository
+runOptions (Options (ProvisionRepository name url)) = provisionRepository name url
+
+provisionRepository :: (DB :> es, IOE :> es) => Text -> Text -> Eff es ()
+provisionRepository name url = do
+  Update.createPackageIndex name url Nothing
 
-importFolderOfCabalFiles :: (Reader PoolConfig :> es, DB :> es, IOE :> es) => FilePath -> Maybe Text -> Eff es ()
+importFolderOfCabalFiles :: (Reader PoolConfig :> es, DB :> es, IOE :> es) => FilePath -> Text -> Eff es ()
 importFolderOfCabalFiles path repository = Log.withStdOutLogger $ \appLogger -> do
   user <- fromJust <$> Query.getUserByUsername "hackage-user"
-  importAllFilesInRelativeDirectory appLogger (user ^. #userId) repository path True
+  mPackageIndex <- Query.getPackageIndexByName repository
+  case mPackageIndex of
+    Nothing -> error $ Text.unpack $ "Package index " <> repository <> " not found in the database!"
+    Just packageIndex ->
+      importAllFilesInRelativeDirectory appLogger (user ^. #userId) (repository, packageIndex.url) path True
 
-importIndex :: (Reader PoolConfig :> es, DB :> es, IOE :> es) => FilePath -> Maybe Text -> Eff es ()
+importIndex :: (Reader PoolConfig :> es, DB :> es, IOE :> es) => FilePath -> Text -> Eff es ()
 importIndex path repository = Log.withStdOutLogger $ \logger -> do
   user <- fromJust <$> Query.getUserByUsername "hackage-user"
-  importFromIndex logger (user ^. #userId) repository path True
+  mPackageIndex <- Query.getPackageIndexByName repository
+  case mPackageIndex of
+    Nothing -> error $ Text.unpack $ "Package index " <> repository <> " not found in the database!"
+    Just packageIndex ->
+      importFromIndex logger (user ^. #userId) (repository, packageIndex.url) path True
 
 withInfo :: Parser a -> String -> ParserInfo a
 withInfo opts desc = info (helper <*> opts) $ progDesc desc

environment.test.sh

@@ -9,3 +9,5 @@ export FLORA_DB_DATABASE="flora_test"
 export FLORA_DB_CONNSTRING="host=${FLORA_DB_HOST} dbname=${FLORA_DB_DATABASE} \
   user=${FLORA_DB_USER} password=${FLORA_DB_PASSWORD} \
   sslmode=allow"
+
+export FLORA_DB_POOL_CONNECTIONS=50

flora.cabal

@@ -111,7 +111,9 @@ library
     Flora.Model.Package.Query
     Flora.Model.Package.Types
     Flora.Model.Package.Update
-    Flora.Model.PackageIndex
+    Flora.Model.PackageIndex.Query
+    Flora.Model.PackageIndex.Types
+    Flora.Model.PackageIndex.Update
     Flora.Model.PersistentSession
     Flora.Model.Release
     Flora.Model.Release.Query

migrations/20231002224826_add_url_to_package_indexes.sql

@@ -0,0 +1,2 @@
+alter table package_indexes
+  add url text not null;

src/core/Flora/Environment/Config.hs

@@ -135,7 +135,7 @@ parsePoolConfig =
     <*> var
       (int >=> nonNegative)
       "FLORA_DB_POOL_CONNECTIONS"
-      (help "Number of connections per sub-pool")
+      (help "Number of connections across all sub-pools")
 
 parseLoggingEnv :: Parser Error LoggingEnv
 parseLoggingEnv =