fix: Increase GHA workflow timeout to 35 days from 72h

README.md

@@ -411,7 +411,7 @@ environment variables for passing your cloud service credentials to the
 workflow.
 
 Note that `cml runner` will also automatically restart your jobs (whether from a
-[GitHub Actions 72-hour timeout](https://docs.github.com/en/actions/reference/usage-limits-billing-and-administration#usage-limits)
+[GitHub Actions 35-day workflow timeout](https://docs.github.com/en/actions/reference/usage-limits-billing-and-administration#usage-limits)
 or a
 [AWS EC2 spot instance interruption](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html)).
 
@@ -438,7 +438,7 @@ jobs:
   train-model:
     needs: deploy-runner
     runs-on: [self-hosted, cml-gpu]
-    timeout-minutes: 4320 # 72h
+    timeout-minutes: 50400 # 35 days
     container:
       image: docker://iterativeai/cml:0-dvc2-base1-gpu
       options: --gpus all

bin/cml/runner.js

@@ -15,7 +15,7 @@ let RUNNER;
 let RUNNER_SHUTTING_DOWN = false;
 let RUNNER_TIMER = 0;
 const RUNNER_JOBS_RUNNING = [];
-const GH_5_MIN_TIMEOUT = (72 * 60 - 5) * 60 * 1000;
+const GH_5_MIN_TIMEOUT = (35 * 24 * 60 - 5) * 60 * 1000;
 
 const shutdown = async (opts) => {
   if (RUNNER_SHUTTING_DOWN) return;
@@ -319,7 +319,7 @@ const runLocal = async (opts) => {
             new Date().getTime() - new Date(job.date).getTime() >
             GH_5_MIN_TIMEOUT
           ) {
-            shutdown({ ...opts, reason: 'timeout:72h' });
+            shutdown({ ...opts, reason: 'timeout:35days' });
             clearInterval(watcherSeventyTwo);
           }
         });
@@ -401,6 +401,12 @@ const run = async (opts) => {
     }
   }
 
+  if (driver === 'github') {
+    winston.warn(
+      'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.'
+    );
+  }
+
   winston.info(`Preparing workdir ${workdir}...`);
   await fs.mkdir(workdir, { recursive: true });
   await fs.chmod(workdir, '766');