PRIME_CACHE_MODE

README.md

@@ -37,6 +37,17 @@
 
 - Fast - if there's nothing to do, returns quickly (<1s, ideally <300ms)
 
+## "Prime cache" mode
+
+When migrating from another system behind a flag, it is sometimes desirable to build on the old system but still fill remote cache one the new one. For that occasion, a special "prime cache mode" is there.
+
+It modifies the behavior in the following way:
+- `snapshot` never downloads remote cache (incl. fuzzy) - to avoid overwriting stuff (which we assume is already built via another mechanism)
+- `snapshot` always skips the job
+- remote cache is uploaded, despite job being skipped
+
+To use it, first build using another system, and the run `taskrunner` with `TASKRUNNER_PRIME_CACHE_MODE=1`
+
 ## Possible features
 
 - Support stdin? For now redirected from `/dev/null`

src/App.hs

@@ -51,6 +51,7 @@ getSettings = do
   enableCommitStatus <- (==Just "1") <$> lookupEnv "TASKRUNNER_ENABLE_COMMIT_STATUS"
   uploadLogs <- (==Just "1") <$> lookupEnv "TASKRUNNER_UPLOAD_LOGS"
   fuzzyCacheFallbackBranches <- maybe [] (Text.words . toText) <$> lookupEnv "TASKRUNNER_FALLBACK_BRANCHES"
+  primeCacheMode <- (==Just "1") <$> lookupEnv "TASKRUNNER_PRIME_CACHE_MODE"
   pure Settings
         { stateDirectory
         , rootDirectory
@@ -61,6 +62,7 @@ getSettings = do
         , enableCommitStatus
         , uploadLogs
         , fuzzyCacheFallbackBranches
+        , primeCacheMode
         }
 
 main :: IO ()
@@ -141,7 +143,7 @@ main = do
           forM_ snapshotArgs.postUnpackCommands \cmd -> do
             runPostUnpackCmd appState cmd
 
-          when (hasOutputs snapshotArgs && settings.saveRemoteCache && not skipped) do
+          when (hasOutputs snapshotArgs && settings.saveRemoteCache && (not skipped || appState.settings.primeCacheMode)) do
             logDebug appState "Saving remote cache"
             s <- RemoteCache.getRemoteCacheSettingsFromEnv
             RemoteCache.saveCache appState s (fromMaybe settings.rootDirectory snapshotArgs.cacheRoot) snapshotArgs.outputs (archiveName appState snapshotArgs h.hash)
@@ -294,6 +296,11 @@ snapshot appState args = do
 
     logDebug appState $ "Hash mismatch, saved=" <>  savedHash <> ", current=" <>  currentHash
 
+    when appState.settings.primeCacheMode do
+      logDebug appState "Prime cache mode, assuming task is done and skippping!"
+      writeIORef appState.hashToSaveRef $ Just $ HashInfo currentHash currentHashInput
+      earlyReturn (False, Nothing)
+
     when (hasOutputs args) do
       s <- RemoteCache.getRemoteCacheSettingsFromEnv
       success <- liftIO $ RemoteCache.restoreCache appState s (fromMaybe appState.settings.rootDirectory args.cacheRoot) (archiveName appState args currentHash)

src/Types.hs

@@ -13,6 +13,7 @@ data Settings = Settings
   , enableCommitStatus :: Bool
   , uploadLogs :: Bool
   , fuzzyCacheFallbackBranches :: [Text]
+  , primeCacheMode :: Bool
   } deriving (Show)
 
 type JobName = String

test/t/prime-cache-mode.out

@@ -0,0 +1,3 @@
+-- output:
+Output in a is: some output
+Output in b is: some output

test/t/prime-cache-mode.txt

@@ -0,0 +1,36 @@
+# no toplevel
+# s3
+
+export TASKRUNNER_SAVE_REMOTE_CACHE=1
+
+mkdir a
+
+(
+  cd a
+  echo foo > input.txt
+  git init -q
+  git add input.txt
+)
+
+cp -r a b
+
+go() {
+  # Note: we want separate workdir for separate repos
+  TASKRUNNER_STATE_DIRECTORY="$(pwd)" taskrunner -n mytask bash -e -c '
+    snapshot input.txt --outputs output.txt
+    echo "Expensive computation"
+  '
+}
+
+(
+  cd a
+  echo "some output" > output.txt
+  TASKRUNNER_PRIME_CACHE_MODE=1 go # should not run expensive computation
+  echo "Output in a is: $(cat output.txt)"
+)
+
+(
+  cd b
+  go # also should not run computation, got it from cache
+  echo "Output in b is: $(cat output.txt)"
+)