diff --git a/bin/cml/runner.js b/bin/cml/runner.js index 8de436506..048b0b9f7 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -115,6 +115,7 @@ const runCloud = async (opts) => { idleTimeout, name, single, + dockerVolumes, cloud, cloudRegion: region, cloudType: type, @@ -163,7 +164,8 @@ const runCloud = async (opts) => { spotPrice, startupScript, awsSecurityGroup, - awsSubnet + awsSubnet, + dockerVolumes }); } @@ -215,14 +217,16 @@ const runCloud = async (opts) => { const runLocal = async (opts) => { winston.info(`Launching ${cml.driver} runner`); - const { workdir, name, labels, single, idleTimeout, noRetry } = opts; + const { workdir, name, labels, single, idleTimeout, noRetry, dockerVolumes } = + opts; const proc = await cml.startRunner({ workdir, name, labels, single, - idleTimeout + idleTimeout, + dockerVolumes }); const dataHandler = async (data) => { @@ -331,12 +335,16 @@ const run = async (opts) => { labels, name, reuse, + dockerVolumes, tfResource, workdir } = opts; cml = new CML({ driver, repo, token }); + if (dockerVolumes.length && cml.driver !== 'gitlab') + winston.warn('Parameters --docker-volumes is only supported in gitlab'); + if (cloud || tfResource) await tf.checkMinVersion(); // prepare tf @@ -412,6 +420,11 @@ exports.handler = async (opts) => { exports.builder = (yargs) => yargs.env('CML_RUNNER').options( kebabcaseKeys({ + dockerVolumes: { + type: 'array', + default: [], + description: 'Docker volumes. This feature is only supported in GitLab' + }, labels: { type: 'string', default: 'cml', diff --git a/bin/cml/runner.test.js b/bin/cml/runner.test.js index c0d0aed2b..496c1f7db 100644 --- a/bin/cml/runner.test.js +++ b/bin/cml/runner.test.js @@ -14,6 +14,9 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --docker-volumes Docker volumes. This feature is only + supported in GitLab + [array] [default: []] --labels One or more user-defined labels for this runner (delimited with commas) [string] [default: \\"cml\\"] diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js index 160d62c03..41043792c 100644 --- a/src/drivers/gitlab.js +++ b/src/drivers/gitlab.js @@ -162,7 +162,14 @@ class Gitlab { } async startRunner(opts) { - const { workdir, idleTimeout, single, labels, name } = opts; + const { + workdir, + idleTimeout, + single, + labels, + name, + dockerVolumes = [] + } = opts; let gpu = true; try { @@ -178,14 +185,19 @@ class Gitlab { try { const bin = resolve(workdir, 'gitlab-runner'); if (!(await fse.pathExists(bin))) { - const url = - 'https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64'; + const arch = process.platform === 'darwin' ? 'darwin' : 'linux'; + const url = `https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-${arch}-amd64`; await download({ url, path: bin }); await fs.chmod(bin, '777'); } const { protocol, host } = new URL(this.repo); const { token } = await this.registerRunner({ tags: labels, name }); + + let dockerVolumesTpl = ''; + dockerVolumes.forEach((vol) => { + dockerVolumesTpl += `--docker-volumes ${vol} `; + }); const command = `${bin} --log-format="json" run-single \ --builds-dir "${workdir}" \ --cache-dir "${workdir}" \ @@ -195,7 +207,8 @@ class Gitlab { --wait-timeout ${idleTimeout} \ --executor "${IN_DOCKER ? 'shell' : 'docker'}" \ --docker-image "iterativeai/cml:${gpu ? 'latest-gpu' : 'latest'}" \ - --docker-runtime "${gpu ? 'nvidia' : ''}" \ + ${gpu ? '--docker-runtime nvidia' : ''} \ + ${dockerVolumesTpl} \ ${single ? '--max-builds 1' : ''}`; return spawn(command, { shell: true }); diff --git a/src/terraform.js b/src/terraform.js index d1570333f..442464521 100644 --- a/src/terraform.js +++ b/src/terraform.js @@ -86,7 +86,8 @@ const iterativeCmlRunnerTpl = (opts = {}) => { spotPrice, startupScript, awsSecurityGroup, - awsSubnet + awsSubnet, + dockerVolumes } = opts; const template = ` @@ -117,6 +118,7 @@ resource "iterative_cml_runner" "runner" { ? `metadata = {\n ${mapCloudMetadata(metadata).join('\n ')}\n }` : '' } + ${dockerVolumes ? `docker_volumes = ${JSON.stringify(dockerVolumes)}` : ''} } `; return template; diff --git a/src/terraform.test.js b/src/terraform.test.js index cd7cae4a8..0a840b7bb 100644 --- a/src/terraform.test.js +++ b/src/terraform.test.js @@ -4,43 +4,44 @@ describe('Terraform tests', () => { test('default options', async () => { const output = iterativeCmlRunnerTpl({}); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - - - - - - - - - - - - - - - - - - - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + + + + + + + + + + + + + + + + + + + + + + } + " + `); }); test('basic settings', async () => { @@ -63,43 +64,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { awsSecurityGroup: 'mysg' }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 300 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - - aws_security_group = \\"mysg\\" - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + aws_security_group = \\"mysg\\" + + + + } + " + `); }); test('basic settings with runner forever', async () => { @@ -121,43 +123,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { spotPrice: '0.0001' }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 0 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - - - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 0 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + + + + + } + " + `); }); test('basic settings with metadata', async () => { @@ -180,46 +183,107 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { metadata: { one: 'value', two: null } }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 300 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - - - - metadata = { - one = \\"value\\" - two = \\"\\" - } -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + + + metadata = { + one = \\"value\\" + two = \\"\\" + } + + } + " + `); + }); + + test('basic settings with docker volumes', async () => { + const output = iterativeCmlRunnerTpl({ + repo: 'https://', + token: 'abc', + driver: 'gitlab', + labels: 'mylabel', + idleTimeout: 300, + name: 'myrunner', + single: true, + cloud: 'aws', + region: 'west', + type: 'mymachinetype', + gpu: 'mygputype', + hddSize: 50, + sshPrivate: 'myprivate', + spot: true, + spotPrice: '0.0001', + dockerVolumes: ['/aa:/aa', '/bb:/bb'] + }); + expect(output).toMatchInlineSnapshot(` + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + + + + docker_volumes = [\\"/aa:/aa\\",\\"/bb:/bb\\"] + } + " + `); }); test('basic settings with permission set', async () => { @@ -243,43 +307,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { awsSecurityGroup: 'mysg' }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 300 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - instance_permission_set = \\"arn:aws:iam::1:instance-profile/x\\" - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - - aws_security_group = \\"mysg\\" - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + instance_permission_set = \\"arn:aws:iam::1:instance-profile/x\\" + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + aws_security_group = \\"mysg\\" + + + + } + " + `); }); test('Startup script', async () => { @@ -302,42 +367,43 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { startupScript: 'c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=' }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 300 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\" - - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\" + + + + + } + " + `); }); });