From 8527fc086aa174f0cd02f3dbe6328a80b10ca05a Mon Sep 17 00:00:00 2001 From: Takano Akio Date: Mon, 26 Jun 2017 07:33:34 +0000 Subject: [PATCH] Implement rule memoization --- shake.cabal | 5 + src/Development/Shake/Command.hs | 21 +- src/Development/Shake/Forward.hs | 3 +- src/Development/Shake/Internal/Args.hs | 3 + src/Development/Shake/Internal/CmdOption.hs | 3 + .../Shake/Internal/Core/Database.hs | 9 +- src/Development/Shake/Internal/Core/Rules.hs | 23 ++- src/Development/Shake/Internal/Core/Run.hs | 3 +- src/Development/Shake/Internal/Core/Types.hs | 9 +- src/Development/Shake/Internal/Memo.hs | 192 ++++++++++++++++++ src/Development/Shake/Internal/Options.hs | 22 +- .../Shake/Internal/Rules/Directory.hs | 18 +- src/Development/Shake/Internal/Rules/File.hs | 10 +- src/Development/Shake/Internal/Rules/Files.hs | 12 +- .../Shake/Internal/Rules/Oracle.hs | 2 +- src/Development/Shake/Internal/Rules/Rerun.hs | 7 +- src/Development/Shake/Memo.hs | 17 ++ 17 files changed, 324 insertions(+), 35 deletions(-) create mode 100644 src/Development/Shake/Internal/Memo.hs create mode 100644 src/Development/Shake/Memo.hs diff --git a/shake.cabal b/shake.cabal index d3a17f1a6..96fe23e70 100644 --- a/shake.cabal +++ b/shake.cabal @@ -94,6 +94,7 @@ library transformers >= 0.2, extra >= 1.4.10, deepseq >= 1.1, + SHA, primitive if flag(portable) @@ -113,6 +114,7 @@ library Development.Shake.Config Development.Shake.FilePath Development.Shake.Forward + Development.Shake.Memo Development.Shake.Rule Development.Shake.Util @@ -137,6 +139,7 @@ library Development.Shake.Internal.FilePattern Development.Shake.Internal.Core.Monad Development.Shake.Internal.Core.Pool + Development.Shake.Internal.Memo Development.Shake.Internal.Profile Development.Shake.Internal.Progress Development.Shake.Internal.Resource @@ -194,6 +197,7 @@ executable shake transformers >= 0.2, extra >= 1.4.10, deepseq >= 1.1, + SHA, primitive if flag(portable) @@ -230,6 +234,7 @@ executable shake Development.Shake.Internal.FileInfo Development.Shake.FilePath Development.Shake.Internal.FilePattern + Development.Shake.Internal.Memo Development.Shake.Internal.Core.Monad Development.Shake.Internal.Core.Pool Development.Shake.Internal.Profile diff --git a/src/Development/Shake/Command.hs b/src/Development/Shake/Command.hs index a5151550c..41693fdd9 100644 --- a/src/Development/Shake/Command.hs +++ b/src/Development/Shake/Command.hs @@ -40,7 +40,7 @@ import System.Process import System.Info.Extra import System.Time.Extra import System.IO.Unsafe(unsafeInterleaveIO) -import qualified Data.ByteString as BS +import qualified Data.ByteString.Char8 as BS import qualified Data.ByteString.Lazy.Char8 as LBS import General.Process import Control.Applicative @@ -50,6 +50,7 @@ import Development.Shake.Internal.CmdOption import Development.Shake.Internal.Core.Run import Development.Shake.FilePath import Development.Shake.Internal.FilePattern +import Development.Shake.Internal.Memo import Development.Shake.Internal.Options import Development.Shake.Internal.Rules.File import Development.Shake.Internal.Derived @@ -133,10 +134,17 @@ commandExplicit funcName oopts results exe args = do verb <- getVerbosity (if verb >= Loud then quietly else id) act - let tracer = case reverse [x | Traced x <- opts] of - "":_ -> liftIO - msg:_ -> traced msg - [] -> traced (takeFileName exe) + let traceMsg = case reverse [x | Traced x <- opts] of + "":_ -> Nothing + msg:_ -> Just msg + [] -> Just (takeFileName exe) + + let tracer = maybe liftIO traced traceMsg + + let memoiser exe args = case reverse [x | Capture x <- opts] of + [] -> id + xs -> fmap (fromMaybe []) + . memoFiles' (show $ exe:args) traceMsg (concat xs) let tracker act | useLint = fsatrace act @@ -207,8 +215,7 @@ commandExplicit funcName oopts results exe args = do unsafeAllowApply $ need $ ham cwd xs return res - skipper $ tracker $ \exe args -> verboser $ tracer $ commandExplicitIO funcName opts results exe args - + skipper $ tracker $ \exe args -> memoiser exe args $ verboser $ tracer $ commandExplicitIO funcName opts results exe args -- | Given a shell command, call the continuation with the sanitised exec-style arguments runShell :: String -> (String -> [String] -> Action a) -> Action a diff --git a/src/Development/Shake/Forward.hs b/src/Development/Shake/Forward.hs index 38ab49be3..9818ac2e2 100755 --- a/src/Development/Shake/Forward.hs +++ b/src/Development/Shake/Forward.hs @@ -76,7 +76,8 @@ shakeArgsForward opts act = shakeArgs (forwardOptions opts) (forwardRule act) -- | Given an 'Action', turn it into a 'Rules' structure which runs in forward mode. forwardRule :: Action () -> Rules () forwardRule act = do - addBuiltinRule noLint $ \k old dirty -> + let summary _ _ = error "Rule memoization cannot be used with the Forward mode" + addBuiltinRule noLint summary $ \k old dirty -> case old of Just old | not dirty -> return $ RunResult ChangedNothing old () _ -> do diff --git a/src/Development/Shake/Internal/Args.hs b/src/Development/Shake/Internal/Args.hs index 060868b19..950473c62 100644 --- a/src/Development/Shake/Internal/Args.hs +++ b/src/Development/Shake/Internal/Args.hs @@ -7,6 +7,7 @@ import Development.Shake.Internal.Options import Development.Shake.Internal.Core.Rules import Development.Shake.Internal.Demo import Development.Shake.FilePath +import Development.Shake.Internal.Memo import Development.Shake.Internal.Rules.File import Development.Shake.Internal.Progress import Development.Shake.Internal.Shake @@ -250,6 +251,8 @@ shakeOptsEx = ,yes $ Option "" ["lint-fsatrace"] (noArg $ \s -> s{shakeLint=Just LintFSATrace}) "Use fsatrace to do validation." ,yes $ Option "" ["no-lint"] (noArg $ \s -> s{shakeLint=Nothing}) "Turn off --lint." ,yes $ Option "" ["live"] (OptArg (\x -> Right ([], \s -> s{shakeLiveFiles=shakeLiveFiles s ++ [fromMaybe "live.txt" x]})) "FILE") "List the files that are live [to live.txt]." + ,yes $ Option "" ["memo-store"] (reqArg "DIRECTORY" $ \x s -> s{shakeMemoSave = fsMemoSave x, shakeMemoRestore = fsMemoRestore x}) "Enable rule memoization, storing files in DIRECTORY" + ,yes $ Option "" ["no-memo"] (noArg $ \s -> s{shakeMemoSave = \_ _ _ -> return (), shakeMemoRestore = \_ -> return False}) "Disable rule memoization" ,yes $ Option "m" ["metadata"] (reqArg "PREFIX" $ \x s -> s{shakeFiles=x}) "Prefix for storing metadata files." ,no $ Option "" ["numeric-version"] (NoArg $ Right ([NumericVersion],id)) "Print just the version number and exit." ,yes $ Option "" ["skip-commands"] (noArg $ \s -> s{shakeRunCommands=False}) "Try and avoid running external programs." diff --git a/src/Development/Shake/Internal/CmdOption.hs b/src/Development/Shake/Internal/CmdOption.hs index 428e91ebd..fed869bc2 100644 --- a/src/Development/Shake/Internal/CmdOption.hs +++ b/src/Development/Shake/Internal/CmdOption.hs @@ -4,6 +4,8 @@ module Development.Shake.Internal.CmdOption(CmdOption(..)) where import Data.Data import qualified Data.ByteString.Lazy.Char8 as LBS +import Development.Shake.Internal.FilePattern + -- | Options passed to 'command' or 'cmd' to control how processes are executed. data CmdOption = Cwd FilePath -- ^ Change the current directory in the spawned process. By default uses this processes current directory. @@ -25,4 +27,5 @@ data CmdOption | FileStdout FilePath -- ^ Should I put the @stdout@ to a file. | FileStderr FilePath -- ^ Should I put the @stderr@ to a file. | AutoDeps -- ^ Compute dependencies automatically. + | Capture [FilePattern] -- ^ Output files captured for rule memoization. See 'Development.Shake.Memo.memoFiles' for more information. deriving (Eq,Ord,Show,Data,Typeable) diff --git a/src/Development/Shake/Internal/Core/Database.hs b/src/Development/Shake/Internal/Core/Database.hs index b0fe0cebc..1aa6ed476 100644 --- a/src/Development/Shake/Internal/Core/Database.hs +++ b/src/Development/Shake/Internal/Core/Database.hs @@ -5,7 +5,7 @@ module Development.Shake.Internal.Core.Database( Trace(..), newTrace, Database, withDatabase, assertFinishedDatabase, - listDepends, lookupDependencies, + listDepends, lookupValue, lookupDependencies, BuildKey(..), build, Depends, Step, Result(..), progress, @@ -419,6 +419,13 @@ listLive Database{..} = do status <- Ids.toList status return [k | (_, (k, Ready{})) <- status] +lookupValue :: Database -> Key -> IO Value +lookupValue Database{..} k = do + withLock lock $ do + intern <- readIORef intern + let Just i = Intern.lookup k intern + Just (_, Ready r) <- Ids.lookup status i + return $ result r listDepends :: Database -> Depends -> IO [Key] listDepends Database{..} (Depends xs) = diff --git a/src/Development/Shake/Internal/Core/Rules.hs b/src/Development/Shake/Internal/Core/Rules.hs index d206cc7d2..cd88ca3eb 100644 --- a/src/Development/Shake/Internal/Core/Rules.hs +++ b/src/Development/Shake/Internal/Core/Rules.hs @@ -5,13 +5,15 @@ module Development.Shake.Internal.Core.Rules( Rules, runRules, - RuleResult, addBuiltinRule, addBuiltinRuleEx, noLint, + RuleResult, addBuiltinRule, addBuiltinRuleEx, + noLint, binarySummary, showSummary, getShakeOptionsRules, userRuleMatch, getUserRules, addUserRule, alternatives, priority, action, withoutActions ) where import Control.Applicative +import Control.DeepSeq (force) import Data.Tuple.Extra import Control.Monad.Extra import Control.Monad.Fix @@ -21,6 +23,7 @@ import Control.Monad.Trans.Reader import Control.Monad.Trans.Writer.Strict import Data.Binary import General.Binary +import qualified Data.Digest.Pure.SHA as SHA import Data.Typeable.Extra import Data.Function import Data.List.Extra @@ -126,25 +129,35 @@ addUserRule r = newRules mempty{userRules = Map.singleton (typeOf r) $ UserRule_ noLint :: BuiltinLint key value noLint _ _ = return Nothing +-- | A 'BuiltinSummary' based on a 'Binary' instance. +binarySummary :: (Binary value) => BuiltinSummary key value +binarySummary _ value = return $! force $ SHA.showDigest $ SHA.sha256 $ + encode value + +-- | A 'BuiltinSummary' based on a 'Show' instance. +showSummary :: (Show value) => BuiltinSummary key value +showSummary _ value = return $! force $ show value + type family RuleResult key -- = value -- | Add a builtin rule, comprising of a lint rule and an action. Each builtin rule must be identified by -- a unique key. -addBuiltinRule :: (RuleResult key ~ value, ShakeValue key, ShakeValue value) => BuiltinLint key value -> BuiltinRun key value -> Rules () +addBuiltinRule :: (RuleResult key ~ value, ShakeValue key, ShakeValue value) => BuiltinLint key value -> BuiltinSummary key value -> BuiltinRun key value -> Rules () addBuiltinRule = addBuiltinRuleEx $ BinaryOp (putEx . Bin.toLazyByteString . execPut . put) (runGet get . LBS.fromChunks . return) -- | Initial version of 'addBuiltinRule', which also lets me set the 'BinaryOp'. -addBuiltinRuleEx :: (RuleResult key ~ value, ShakeValue key, ShakeValue value) => BinaryOp key -> BuiltinLint key value -> BuiltinRun key value -> Rules () -addBuiltinRuleEx binary lint (run :: BuiltinRun key value) = do +addBuiltinRuleEx :: (RuleResult key ~ value, ShakeValue key, ShakeValue value) => BinaryOp key -> BuiltinLint key value -> BuiltinSummary key value -> BuiltinRun key value -> Rules () +addBuiltinRuleEx binary lint summary (run :: BuiltinRun key value) = do let k = Proxy :: Proxy key v = Proxy :: Proxy value let run_ k v b = fmap newValue <$> run (fromKey k) v b let lint_ k v = lint (fromKey k) (fromValue v) + let summary_ k v = summary (fromKey k) (fromValue v) let binary_ = BinaryOp (putOp binary . fromKey) (newKey . getOp binary) - newRules mempty{builtinRules = Map.singleton (typeRep k) $ BuiltinRule run_ lint_ (typeRep v) binary_} + newRules mempty{builtinRules = Map.singleton (typeRep k) $ BuiltinRule run_ lint_ summary_ (typeRep v) binary_} -- | Change the priority of a given set of rules, where higher priorities take precedence. diff --git a/src/Development/Shake/Internal/Core/Run.hs b/src/Development/Shake/Internal/Core/Run.hs index 5fa64d60e..4e6012ff3 100644 --- a/src/Development/Shake/Internal/Core/Run.hs +++ b/src/Development/Shake/Internal/Core/Run.hs @@ -88,6 +88,7 @@ run opts@ShakeOptions{..} rs = (if shakeLineBuffering then lineBuffering else id after <- newIORef [] absent <- newIORef [] + cacheRef <- newIORef Map.empty withCleanup $ \cleanup -> do _ <- addCleanup cleanup $ do when shakeTimings printTimings @@ -108,7 +109,7 @@ run opts@ShakeOptions{..} rs = (if shakeLineBuffering then lineBuffering else id addTiming "Running rules" runPool (shakeThreads == 1) shakeThreads $ \pool -> do - let s0 = Global database pool cleanup start ruleinfo output opts diagnostic curdir after absent getProgress userRules + let s0 = Global database pool cleanup start ruleinfo output opts diagnostic curdir after absent getProgress userRules cacheRef let s1 = newLocal emptyStack shakeVerbosity forM_ actions $ \act -> addPoolLowPriority pool $ runAction s0 s1 act $ \x -> case x of diff --git a/src/Development/Shake/Internal/Core/Types.hs b/src/Development/Shake/Internal/Core/Types.hs index fddcb3e51..908ab1527 100755 --- a/src/Development/Shake/Internal/Core/Types.hs +++ b/src/Development/Shake/Internal/Core/Types.hs @@ -2,7 +2,7 @@ {-# LANGUAGE ExistentialQuantification, ConstraintKinds, DeriveFunctor #-} module Development.Shake.Internal.Core.Types( - BuiltinRun, BuiltinLint, RunResult(..), RunChanged(..), + BuiltinRun, BuiltinLint, BuiltinSummary, RunResult(..), RunChanged(..), UserRule(..), UserRule_(..), BuiltinRule(..), Global(..), Local(..), Action(..), newLocal @@ -82,9 +82,15 @@ type BuiltinRun key value = key -> Maybe BS.ByteString -> Bool -> Action (RunRes -- For builtin rules where the value is expected to change use 'Development.Shake.Rules.noLint'. type BuiltinLint key value = key -> value -> IO (Maybe String) +-- | A function that summarizes the current value into a short string, typically a hash. +-- The result should be stable, and should not depend on the state of the Shake database. +-- Used for rule memoization. +type BuiltinSummary key value = key -> value -> IO String + data BuiltinRule = BuiltinRule {builtinRun :: BuiltinRun Key Value ,builtinLint :: BuiltinLint Key Value + ,builtinSummary :: BuiltinSummary Key Value ,builtinResult :: TypeRep ,builtinKey :: BinaryOp Key } @@ -123,6 +129,7 @@ data Global = Global ,globalTrackAbsent :: IORef [(Key, Key)] -- ^ Tracked things, in rule fst, snd must be absent ,globalProgress :: IO Progress -- ^ Request current progress state ,globalUserRules :: Map.HashMap TypeRep UserRule_ + ,globalSHACache :: IORef (Map.HashMap Key String) -- ^ SHA cache for memoization } -- local variables of Action diff --git a/src/Development/Shake/Internal/Memo.hs b/src/Development/Shake/Internal/Memo.hs new file mode 100644 index 000000000..cde9f9550 --- /dev/null +++ b/src/Development/Shake/Internal/Memo.hs @@ -0,0 +1,192 @@ +{-# LANGUAGE LambdaCase #-} + +module Development.Shake.Internal.Memo( + memoFiles, memoFiles', hashState, + + fsMemoRestore, fsMemoSave, + ) where + +import Control.DeepSeq (force) +import Control.Exception.Extra +import Control.Monad.Extra +import Control.Monad.IO.Class +import Control.Monad.Trans.Maybe +import qualified Data.Digest.Pure.SHA as SHA +import qualified Data.ByteString.Char8 as BS +import qualified Data.ByteString.Lazy.Char8 as LBS +import qualified Data.HashMap.Strict as Map +import Data.IORef +import Data.Maybe +import Data.Traversable +import System.Directory +import System.IO +import System.IO.Error +import Text.Read + +import Development.Shake.FilePath +import Development.Shake.Internal.Core.Action +import Development.Shake.Internal.Core.Database +import Development.Shake.Internal.Core.Monad +import Development.Shake.Internal.Core.Types +import Development.Shake.Internal.FilePattern +import Development.Shake.Internal.Options +import Development.Shake.Internal.Rules.Directory (getDirectoryFilesIO) +import Development.Shake.Internal.Value + +-- | Cache files produced by an action using the persisitent store. The action +-- identifier must be unique to the action, as it's used as a key to indexing +-- into the cache. +-- +-- The example below caches the output of @gcc@: +-- +-- @ +-- "*.o" %> \\f -> memoFiles "gcc" [f] $ cmd "gcc -c" [f -\<.\> "c"] +-- @ +-- +-- Alternatively, you can use the `Development.Shake.Command.Capture` option +-- for `Development.Shake.Command.cmd`: +-- +-- @ +-- "*.o" %> \\f -> cmd "gcc -c" [f -\<.\> "c"] (Capture [f]) +-- @ +memoFiles + :: String -- ^ Action identifier + -> [FilePattern] -- ^ Files to capture + -> Action a + -> Action (Maybe a) +memoFiles name = memoFiles' name (Just name) + +memoFiles' + :: String -- ^ action identifier + -> Maybe String -- ^ short description + -> [FilePattern] -- ^ files to capture + -> Action a + -> Action (Maybe a) +memoFiles' actName descr patterns act = do + queryString <- stateString actName + options <- Action $ getsRO globalOptions + let shasum = SHA.showDigest $ SHA.sha256 $ LBS.fromStrict $ BS.pack queryString + + liftIO (shakeMemoRestore options shasum) >>= \case + False -> do + r <- act + filesToCapture <- liftIO $ getDirectoryFilesIO "" patterns + liftIO $ shakeMemoSave options shasum queryString filesToCapture + putNormal $ "Saving to cache (" ++ shasum ++ ")" + return $ Just r + True -> do + verb <- getVerbosity + if verb < Loud + then case descr of + Just d -> do + stack <- Action $ getsRW localStack + putNormal $ "# " ++ d ++ + " (cached) (for " ++ showTopStack stack ++ ")" + _ -> return () + else putNormal $ + "Using cache for " ++ actName ++ " (" ++ shasum ++ ")" + return Nothing + +ignoreErrors :: IO () -> IO () +ignoreErrors a = a `catch` \e -> const (return ()) (e :: IOException) + +-- | Hash the current state of the dependencies, together with the given string. +-- This function can be useful to write rules that don't depend on timestamps. +-- For example, instead of writing a rule that touches an empty file upon +-- completion, you could make it write the string returned by 'hashState' +-- to the file. +hashState :: String -> Action String +hashState givenString = SHA.showDigest . SHA.sha256 . LBS.pack <$> + stateString givenString + +stateString :: String -> Action String +stateString givenString = do + database <- Action $ getsRO globalDatabase + deps <- Action $ getsRW localDepends + dependKeys <- liftIO $ concat <$> mapM (listDepends database) deps + inputs <- forM dependKeys $ \key -> do + hash <- getSummary key + return $ show (typeKey key) ++ ": " ++ show key ++ ": " ++ hash + return $ unlines $ givenString : inputs + +getSummary :: Key -> Action String +getSummary key = do + cacheRef <- Action $ getsRO globalSHACache + cache <- liftIO $ readIORef cacheRef + case Map.lookup key cache of + Just r -> return r + Nothing -> do + rules <- Action $ getsRO globalRules + database <- Action $ getsRO globalDatabase + case Map.lookup (typeKey key) rules of + Just builtinRule -> do + value <- liftIO $ lookupValue database key + summary <- liftIO $ builtinSummary builtinRule key value + liftIO $ atomicModifyIORef' cacheRef $ \cache' -> + ( Map.insert key summary cache', ()) + return summary + Nothing -> error $ "No builtin rule is found for " ++ show + (typeKey key) + +hashFileIO :: FilePath -> IO String +hashFileIO path = do + sha <- SHA.showDigest . SHA.sha256 <$> LBS.readFile path + return $! force sha + +-- | The 'Development.Shake.shakeMemoRestore' handler for the file system +-- backend. +fsMemoRestore :: FilePath -> String -> IO Bool{- success? -} +fsMemoRestore cacheDir shasum = fmap isJust $ runMaybeT $ do + Right answer <- liftIO $ tryJust (guard . isDoesNotExistError) $ + readFile $ answerFile cacheDir shasum + anss <- for (lines answer) $ \xs -> do + (keyStr, _:val) <- return $ break (==':') xs + Just key <- return $ readMaybe keyStr + return (key, val) + forM_ anss $ \(k, v) -> do + liftIO $ createDirectoryIfMissing True $ takeDirectory k + Right _ <- liftIO $ tryJust (guard . isDoesNotExistError) $ do + copyFile (contentFile cacheDir v) k + touchFile (contentFile cacheDir v) + return () + liftIO $ touchFile $ answerFile cacheDir shasum + +-- | The 'Development.Shake.shakeMemoSave' handler for the file system backend. +fsMemoSave :: FilePath -> String -> String -> [FilePattern] -> IO () +fsMemoSave cacheDir shasum queryString filesToCapture = ignoreErrors $ do + lns <- forM filesToCapture $ \file -> do + hash <- hashFileIO file + createDirectoryIfMissing True $ takeDirectory $ contentFile cacheDir hash + c <- doesFileExist $ contentFile cacheDir hash + unless c $ copyFileAtomic file $ contentFile cacheDir hash + return (show file ++ ":" ++ hash) + createDirectoryIfMissing True $ takeDirectory $ questionFile cacheDir shasum + writeFileAtomic (questionFile cacheDir shasum) queryString + writeFileAtomic (answerFile cacheDir shasum) $ unlines lns + +touchFile :: FilePath -> IO () +touchFile path = withFile path AppendMode $ \_ -> return () + +copyFileAtomic :: FilePath -> FilePath -> IO () +copyFileAtomic sourcePath destPath = withNewFileAtomic destPath $ \hd -> + LBS.hPutStr hd =<< LBS.readFile sourcePath + +writeFileAtomic :: FilePath -> String -> IO () +writeFileAtomic destPath str = withNewFileAtomic destPath $ \h -> + hPutStr h str + +withNewFileAtomic :: FilePath -> (Handle -> IO ()) -> IO () +withNewFileAtomic destPath writer = do + withFile tempPath WriteMode writer + renameFile tempPath destPath + where + tempPath = destPath <.> "tmp" + +answerFile :: String -> String -> FilePath +answerFile basePath hs = basePath "qa" take 2 hs drop 2 hs <.> "a" + +questionFile :: String -> String -> FilePath +questionFile basePath hs = basePath "qa" take 2 hs drop 2 hs <.> "q" + +contentFile :: String -> String -> FilePath +contentFile basePath hs = basePath "content" take 2 hs drop 2 hs diff --git a/src/Development/Shake/Internal/Options.hs b/src/Development/Shake/Internal/Options.hs index 45845686a..4563f2623 100644 --- a/src/Development/Shake/Internal/Options.hs +++ b/src/Development/Shake/Internal/Options.hs @@ -159,6 +159,16 @@ data ShakeOptions = ShakeOptions -- ^ Defaults to writing using 'putStrLn'. A function called to output messages from Shake, along with the 'Verbosity' at -- which that message should be printed. This function will be called atomically from all other 'shakeOutput' functions. -- The 'Verbosity' will always be greater than or higher than 'shakeVerbosity'. + ,shakeMemoSave :: String -> String -> [FilePath] -> IO () + -- ^ Defaults to @\_ _ _ -> return ()@. A function for saving files into the memoization table. + -- + -- The function will be called like @f key query paths@. @key@ is the key under which the files + -- are to be stored, @query@ is an ASCII representation of the key (for debuggin), and + -- @paths@ is the list of files to be stored. + ,shakeMemoRestore :: String -> IO Bool + -- ^ Defaults to @\_ -> return False@. A function for restoring an entry from the memoization table. + -- + -- The function should return 'True' if it has successfully restored the entry specified by the given key. ,shakeExtra :: Map.HashMap TypeRep Dynamic -- ^ This a map which can be used to store arbitrary extra information that a user may need when writing rules. -- The key of each entry must be the 'dynTypeRep' of the value. @@ -174,6 +184,8 @@ shakeOptions = ShakeOptions True ChangeModtime True [] False (const $ return ()) (const $ BS.putStrLn . UTF8.fromString) -- try and output atomically using BS + (\_ _ _ -> return ()) + (const $ return False) Map.empty fieldsShakeOptions = @@ -184,15 +196,15 @@ fieldsShakeOptions = ,"shakeLiveFiles","shakeVersionIgnore","shakeProgress", "shakeOutput", "shakeExtra"] tyShakeOptions = mkDataType "Development.Shake.Types.ShakeOptions" [conShakeOptions] conShakeOptions = mkConstr tyShakeOptions "ShakeOptions" fieldsShakeOptions Prefix -unhide x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 y1 y2 y3 = - ShakeOptions x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 (fromHidden y1) (fromHidden y2) (fromHidden y3) +unhide x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 y1 y2 y3 y4 y5 = + ShakeOptions x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 (fromHidden y1) (fromHidden y2) (fromHidden y3) (fromHidden y4) (fromHidden y5) instance Data ShakeOptions where - gfoldl k z (ShakeOptions x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 y1 y2 y3) = + gfoldl k z (ShakeOptions x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 y1 y2 y3 y4 y5) = z unhide `k` x1 `k` x2 `k` x3 `k` x4 `k` x5 `k` x6 `k` x7 `k` x8 `k` x9 `k` x10 `k` x11 `k` x12 `k` x13 `k` x14 `k` x15 `k` x16 `k` x17 `k` x18 `k` x19 `k` x20 `k` x21 `k` - Hidden y1 `k` Hidden y2 `k` Hidden y3 - gunfold k z c = k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ z unhide + Hidden y1 `k` Hidden y2 `k` Hidden y3 `k` Hidden y4 `k` Hidden y5 + gunfold k z c = k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ z unhide toConstr ShakeOptions{} = conShakeOptions dataTypeOf _ = tyShakeOptions diff --git a/src/Development/Shake/Internal/Rules/Directory.hs b/src/Development/Shake/Internal/Rules/Directory.hs index abfeab784..fb29aac09 100644 --- a/src/Development/Shake/Internal/Rules/Directory.hs +++ b/src/Development/Shake/Internal/Rules/Directory.hs @@ -115,11 +115,12 @@ instance Show GetDirectoryA where -- RULE DEFINITIONS queryRule :: (RuleResult key ~ value, BinaryEx key, BinaryEx witness, Eq witness, ShakeValue key, ShakeValue value) - => (value -> witness) -> (key -> IO value) -> Rules () -queryRule witness query = addBuiltinRuleEx newBinaryOp + => (value -> witness) -> (BuiltinSummary key value) -> (key -> IO value) -> Rules () +queryRule witness summary query = addBuiltinRuleEx newBinaryOp (\k old -> do new <- query k return $ if old == new then Nothing else Just $ show new) + summary (\k old _ -> liftIO $ do new <- query k let wnew = witness new @@ -132,13 +133,12 @@ defaultRuleDirectory :: Rules () defaultRuleDirectory = do -- for things we are always going to rerun, and which might take up a lot of memory to store, -- we only store their hash, so we can compute change, but not know what changed happened - queryRule id (\(DoesFileExistQ x) -> DoesFileExistA <$> IO.doesFileExist x) - queryRule id (\(DoesDirectoryExistQ x) -> DoesDirectoryExistA <$> IO.doesDirectoryExist x) - queryRule hash (\(GetEnvQ x) -> GetEnvA <$> IO.lookupEnv x) - queryRule hash (\(GetDirectoryContentsQ x) -> GetDirectoryA <$> getDirectoryContentsIO x) - queryRule hash (\(GetDirectoryFilesQ (a,b)) -> GetDirectoryA <$> getDirectoryFilesIO a b) - queryRule hash (\(GetDirectoryDirsQ x) -> GetDirectoryA <$> getDirectoryDirsIO x) - + queryRule id showSummary (\(DoesFileExistQ x) -> DoesFileExistA <$> IO.doesFileExist x) + queryRule id showSummary (\(DoesDirectoryExistQ x) -> DoesDirectoryExistA <$> IO.doesDirectoryExist x) + queryRule hash binarySummary (\(GetEnvQ x) -> GetEnvA <$> IO.lookupEnv x) + queryRule hash binarySummary (\(GetDirectoryContentsQ x) -> GetDirectoryA <$> getDirectoryContentsIO x) + queryRule hash binarySummary (\(GetDirectoryFilesQ (a,b)) -> GetDirectoryA <$> getDirectoryFilesIO a b) + queryRule hash binarySummary (\(GetDirectoryDirsQ x) -> GetDirectoryA <$> getDirectoryDirsIO x) --------------------------------------------------------------------- -- RULE ENTRY POINTS diff --git a/src/Development/Shake/Internal/Rules/File.hs b/src/Development/Shake/Internal/Rules/File.hs index d694aa816..592d87746 100644 --- a/src/Development/Shake/Internal/Rules/File.hs +++ b/src/Development/Shake/Internal/Rules/File.hs @@ -11,6 +11,7 @@ module Development.Shake.Internal.Rules.File( ) where import Control.Applicative +import Control.DeepSeq (force) import Control.Monad.Extra import Control.Monad.IO.Class import System.Directory @@ -18,7 +19,9 @@ import Data.Typeable import Data.List import Data.Bits import Data.Maybe +import qualified Data.Digest.Pure.SHA as SHA import qualified Data.ByteString.Char8 as BS +import qualified Data.ByteString.Lazy.Char8 as LBS import qualified Data.HashSet as Set import Foreign.Storable import Data.Word @@ -267,7 +270,12 @@ defaultRuleFile = do Just now -> case fileEqualValue opts v now of EqualCheap -> return Nothing _ -> return $ Just $ show now - addBuiltinRuleEx newBinaryOp lint run + + let summary (FileQ k) _ = do + str <- LBS.readFile (fileNameToString k) + return $! force $ SHA.showDigest $ SHA.sha256 str + + addBuiltinRuleEx newBinaryOp lint summary run apply_ :: (a -> FileName) -> [a] -> Action [Maybe FileA] diff --git a/src/Development/Shake/Internal/Rules/Files.hs b/src/Development/Shake/Internal/Rules/Files.hs index efbd49bf8..937eec3d1 100644 --- a/src/Development/Shake/Internal/Rules/Files.hs +++ b/src/Development/Shake/Internal/Rules/Files.hs @@ -5,8 +5,11 @@ module Development.Shake.Internal.Rules.Files( (&?>), (&%>), defaultRuleFiles ) where +import Control.DeepSeq (force) import Control.Monad import Control.Monad.IO.Class +import qualified Data.Digest.Pure.SHA as SHA +import qualified Data.ByteString.Lazy.Char8 as LBS import Data.Maybe import Data.List.Extra import System.Directory @@ -71,7 +74,14 @@ defaultRuleFiles = do [r] -> r rs -> liftIO $ errorMultipleRulesMatch (typeOf k) (show k) (length rs) } - addBuiltinRuleEx newBinaryOp lint run + + let summary (FilesQ ks) _ = do + hashes <- forM ks $ \(FileQ k) -> do + str <- LBS.readFile (fileNameToString k) + return $ SHA.showDigest $ SHA.sha256 str + return $ force $ SHA.showDigest $ SHA.sha256 $ LBS.pack $ show hashes + + addBuiltinRuleEx newBinaryOp lint summary run diff --git a/src/Development/Shake/Internal/Rules/Oracle.hs b/src/Development/Shake/Internal/Rules/Oracle.hs index 065bee248..24c4c1693 100644 --- a/src/Development/Shake/Internal/Rules/Oracle.hs +++ b/src/Development/Shake/Internal/Rules/Oracle.hs @@ -84,7 +84,7 @@ addOracle :: (RuleResult q ~ a, ShakeValue q, ShakeValue a) => (q -> Action a) - addOracle = f where f :: forall q a . (RuleResult q ~ a, ShakeValue q, ShakeValue a) => (q -> Action a) -> Rules (q -> Action a) f act = do - addBuiltinRule noLint $ \(OracleQ q) old _ -> do + addBuiltinRule noLint binarySummary $ \(OracleQ q) old _ -> do new <- OracleA <$> act q return $ RunResult (if fmap decode' old == Just new then ChangedRecomputeSame else ChangedRecomputeDiff) diff --git a/src/Development/Shake/Internal/Rules/Rerun.hs b/src/Development/Shake/Internal/Rules/Rerun.hs index 82cc4dc20..48fb91da8 100644 --- a/src/Development/Shake/Internal/Rules/Rerun.hs +++ b/src/Development/Shake/Internal/Rules/Rerun.hs @@ -38,6 +38,9 @@ alwaysRerun :: Action () alwaysRerun = apply1 $ AlwaysRerunQ () defaultRuleRerun :: Rules () -defaultRuleRerun = - addBuiltinRuleEx newBinaryOp noLint $ +defaultRuleRerun = do + let stateSummary _ _ = error $ + "An attempt has been made to take the state of alwaysRerun. " ++ + "Perhaps you tried to memoize a rule that depends on alwaysRerun?" + addBuiltinRuleEx newBinaryOp noLint stateSummary $ \AlwaysRerunQ{} _ _ -> return $ RunResult ChangedRecomputeDiff BS.empty () diff --git a/src/Development/Shake/Memo.hs b/src/Development/Shake/Memo.hs new file mode 100644 index 000000000..bdbefa5ef --- /dev/null +++ b/src/Development/Shake/Memo.hs @@ -0,0 +1,17 @@ +-- | This module provides functions for defining memoized build rules. Memoized +-- build rules save their results in a persistent store, and re-use them when +-- the exact same build is requested later. Two builds are considered the same +-- when they have the same set of dependencies, and the contents of the +-- dependencies (e.g. the file contents) match. The same store can be shared +-- between multiple build directories. +-- +-- In order to enable memoized rules, either use the @--memo-store@ option, or +-- override 'Development.Shake.shakeMemoSave' and +-- 'Development.Shake.shakeMemoRestore' in 'Development.Shake.ShakeOptions'. +module Development.Shake.Memo( + memoFiles, hashState, + -- * A filesystem-based reference implementation of the memo API + fsMemoRestore, fsMemoSave, + ) where + +import Development.Shake.Internal.Memo