From 8c25bb59b70181c33bf333fbc5c075800de821e0 Mon Sep 17 00:00:00 2001 From: Vladimir Ciobanu Date: Thu, 19 Jul 2018 19:20:09 +0300 Subject: [PATCH] Limit threads (#127) * moving * test * fix? * whoops * limitThreads all the things * addressed reviews, still need to ttest * removed two more formatting changes * forgot to rename argument to jobs * signalQSem * should use bracket_, removed the styling changes again * couple of extra spacing causing noise in the diff * and one more --- Makefile | 36 ++++++++++++++++++++++++++++ app/Main.hs | 67 ++++++++++++++++++++++++++++++++++------------------- 2 files changed, 79 insertions(+), 24 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..54f5028 --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +package = psc-package + +stack_yaml = STACK_YAML="stack.yaml" +stack = $(stack_yaml) stack + +build: + $(stack) build $(package) + +build-dirty: + $(stack) build --ghc-options=-fforce-recomp $(package) + +run: + $(stack) build --fast && $(stack) exec -- $(package) + +install: + $(stack) install + +ghci: + $(stack) ghci $(package) + +test: + $(stack) test $(package) + +test-ghci: + $(stack) ghci $(package):test:$(package)-tests + +bench: + $(stack) bench $(package) + +ghcid: + $(stack) exec -- ghcid -c "stack ghci $(package) --test --ghci-options='-fobject-code -fno-warn-unused-do-bind'" + +dev-deps: + stack install ghcid + +.PHONY : build build-dirty run install ghci test test-ghci ghcid dev-deps diff --git a/app/Main.hs b/app/Main.hs index 19926c1..ae20a74 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -11,6 +11,8 @@ module Main where import qualified Control.Foldl as Foldl import Control.Concurrent.Async (forConcurrently_, mapConcurrently) +import Control.Concurrent.QSem (newQSem, signalQSem, waitQSem) +import Control.Exception (bracket_) import qualified Data.Aeson as Aeson import Data.Aeson.Encode.Pretty import Data.Either.Combinators (rightToMaybe) @@ -237,13 +239,18 @@ getTransitiveDeps db deps = sansPrefix <- T.stripPrefix "purescript-" (runPackageName pkg) rightToMaybe (mkPackageName sansPrefix) -installImpl :: PackageConfig -> IO () -installImpl config@PackageConfig{ depends } = do +installImpl :: PackageConfig -> Maybe Int -> IO () +installImpl config@PackageConfig{ depends } limitJobs = do getPackageSet config db <- readPackageSet config trans <- getTransitiveDeps db depends echoT ("Installing " <> pack (show (length trans)) <> " packages...") - forConcurrently_ trans . uncurry $ performInstall $ set config + case limitJobs of + Nothing -> + forConcurrently_ trans . uncurry $ performInstall $ set config + Just max' -> do + sem <- newQSem max' + forConcurrently_ trans . uncurry . (\x y z -> bracket_ (waitQSem sem) (signalQSem sem) (performInstall x y z)) $ set config getPureScriptVersion :: IO Version getPureScriptVersion = do @@ -256,8 +263,8 @@ getPureScriptVersion = do | otherwise -> exitWithErr "Unable to parse output of purs --version" _ -> exitWithErr "Unexpected output from purs --version" -initialize :: Maybe (Text, Maybe Text) -> IO () -initialize setAndSource = do +initialize :: Maybe (Text, Maybe Text) -> Maybe Int -> IO () +initialize setAndSource limitJobs = do exists <- testfile "psc-package.json" when exists $ exitWithErr "psc-package.json already exists" echoT "Initializing new project in current directory" @@ -281,33 +288,33 @@ initialize setAndSource = do } writePackageFile pkg - installImpl pkg + installImpl pkg limitJobs where packageNameFromPWD = either (const untitledPackageName) id . mkPackageName -install :: Maybe String -> IO () -install pkgName' = do +install :: Maybe String -> Maybe Int -> IO () +install pkgName' limitJobs = do pkg <- readPackageFile case pkgName' of Nothing -> do - installImpl pkg + installImpl pkg limitJobs echoT "Install complete" Just str -> do pkgName <- packageNameFromString str let pkg' = pkg { depends = List.nub (pkgName : depends pkg) } - updateAndWritePackageFile pkg' + updateAndWritePackageFile pkg' limitJobs -uninstall :: String -> IO () -uninstall pkgName' = do +uninstall :: String -> Maybe Int -> IO () +uninstall pkgName' limitJobs = do pkg <- readPackageFile pkgName <- packageNameFromString pkgName' let pkg' = pkg { depends = filter (/= pkgName) $ depends pkg } - updateAndWritePackageFile pkg' + updateAndWritePackageFile pkg' limitJobs -updateAndWritePackageFile :: PackageConfig -> IO () -updateAndWritePackageFile pkg = do - installImpl pkg +updateAndWritePackageFile :: PackageConfig -> Maybe Int -> IO () +updateAndWritePackageFile pkg limitJobs = do + installImpl pkg limitJobs writePackageFile pkg echoT "psc-package.json file was updated" @@ -371,10 +378,10 @@ listSourcePaths = do -- | Helper for calling through to @purs@ -- -- Extra args will be appended to the options -exec :: [String] -> Bool -> [String] -> IO () -exec execNames onlyDeps passthroughOptions = do +exec :: [String] -> Bool -> [String] -> Maybe Int -> IO () +exec execNames onlyDeps passthroughOptions limitJobs = do pkg <- readPackageFile - installImpl pkg + installImpl pkg limitJobs paths <- getPaths let cmdParts = tail execNames @@ -471,8 +478,8 @@ checkForUpdates applyMinorUpdates applyMajorUpdates = do data VerifyArgs a = Package a | VerifyAll (Maybe a) deriving (Functor, Foldable, Traversable) -verify :: VerifyArgs Text -> IO () -verify arg = do +verify :: VerifyArgs Text -> Maybe Int -> IO () +verify arg limitJobs = do pkg <- readPackageFile db <- readPackageSet pkg case traverse mkPackageName arg of @@ -505,7 +512,11 @@ verify arg = do Just pkgInfo -> performInstall (set pkg) pkgName pkgInfo echoT ("Verifying package " <> runPackageName name) dependencies <- map fst <$> getTransitiveDeps db [name] - dirs <- mapConcurrently dirFor dependencies + dirs <- case limitJobs of + Nothing -> mapConcurrently dirFor dependencies + Just max' -> do + sem <- newQSem max' + mapConcurrently (bracket_ (waitQSem sem) (signalQSem sem) . dirFor) dependencies let srcGlobs = map (pathToTextUnsafe . ( ("src" "**" "*.purs"))) dirs procs "purs" ("compile" : srcGlobs) empty @@ -539,24 +550,27 @@ main = do [ Opts.command "init" (Opts.info (initialize <$> optional ((,) <$> (fromString <$> set) <*> optional (fromString <$> source)) + <*> optional limitJobs Opts.<**> Opts.helper) (Opts.progDesc "Create a new psc-package.json file")) , Opts.command "uninstall" - (Opts.info (uninstall <$> pkg Opts.<**> Opts.helper) + (Opts.info (uninstall <$> pkg <*> optional limitJobs Opts.<**> Opts.helper) (Opts.progDesc "Uninstall the named package")) , Opts.command "install" - (Opts.info (install <$> optional pkg Opts.<**> Opts.helper) + (Opts.info (install <$> optional pkg <*> optional limitJobs Opts.<**> Opts.helper) (Opts.progDesc "Install/update the named package and add it to 'depends' if not already listed. If no package is specified, install/update all dependencies.")) , Opts.command "build" (Opts.info (exec ["purs", "compile"] <$> onlyDeps "Compile only the package's dependencies" <*> passthroughArgs "purs compile" + <*> optional limitJobs Opts.<**> Opts.helper) (Opts.progDesc "Install dependencies and compile the current package")) , Opts.command "repl" (Opts.info (exec ["purs", "repl"] <$> onlyDeps "Load only the package's dependencies" <*> passthroughArgs "purs repl" + <*> optional limitJobs Opts.<**> Opts.helper) (Opts.progDesc "Open an interactive environment for PureScript")) , Opts.command "dependencies" @@ -575,6 +589,7 @@ main = do (Opts.info (verify <$> ((Package . fromString <$> pkg) <|> (VerifyAll <$> optional (fromString <$> after))) + <*> optional limitJobs Opts.<**> Opts.helper) (Opts.progDesc "Verify that the named package builds correctly. If no package is specified, verify that all packages in the package set build correctly.")) , Opts.command "format" @@ -586,6 +601,10 @@ main = do Opts.metavar "PACKAGE" <> Opts.help "The name of the package to install" + limitJobs = Opts.option Opts.auto $ + Opts.long "jobs" + <> Opts.help "Limit the number of jobs that can run concurrently" + source = Opts.strOption $ Opts.long "source" <> Opts.help "The Git repository for the package set"