diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index f3a343de3..d07122032 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -37,7 +37,7 @@ import urllib.error import urllib.request from pathlib import Path from threading import Thread -from typing import Optional +from typing import Optional, Union SECRET_BASEURL_TPL = "http://taskcluster/secrets/v1/secret/{}" @@ -598,6 +598,7 @@ def git_checkout( commit: Optional[str], ssh_key_file: Optional[Path], ssh_known_hosts_file: Optional[Path], + submodules: Union[str, bool, None], ): env = { # abort if transfer speed is lower than 1kB/s for 1 minute @@ -705,23 +706,40 @@ def git_checkout( run_required_command(b"vcs", args, cwd=destination_path) - if os.path.exists(os.path.join(destination_path, ".gitmodules")): - args = [ - "git", - "submodule", - "init", - ] - - run_required_command(b"vcs", args, cwd=destination_path) - - args = [ - "git", - "submodule", - "update", - "--force", # Overrides any potential local changes + if submodules is not None: + def _filter_submodule(status): + # A boolean value selects/disables all submodules. + if isinstance(submodules, bool): + return submodules + + # Otherwise, a colon-separated stringlist of selected submodules. + smpath = status.split()[1] + return smpath in submodules.split(':') + + submodpaths = [ + status.split()[1] + for status in subprocess.check_output(["git", "submodule", "status"], + cwd=destination_path, universal_newlines=True).splitlines() + if _filter_submodule(status) ] - run_required_command(b"vcs", args, cwd=destination_path) + for p in submodpaths: + args = [ + "git", + "submodule", + "init", + p + ] + run_required_command(b"vcs", args, cwd=destination_path) + + args = [ + "git", + "submodule", + "update", + "--force", # Overrides any potential local changes + p + ] + run_required_command(b"vcs", args, cwd=destination_path) _clean_git_checkout(destination_path) @@ -894,6 +912,12 @@ def collect_vcs_options(args, project, name): ref = os.environ.get("%s_HEAD_REF" % env_prefix) pip_requirements = os.environ.get("%s_PIP_REQUIREMENTS" % env_prefix) private_key_secret = os.environ.get("%s_SSH_SECRET_NAME" % env_prefix) + submodules = os.environ.get("%s_SUBMODULES" % env_prefix) + + # Some special values can be used to request all submodules. + if submodules is not None: + if submodules.lower() in ["auto", "true", "yes"]: + submodules = True store_path = os.environ.get("HG_STORE_PATH") @@ -928,6 +952,7 @@ def collect_vcs_options(args, project, name): "repo-type": repo_type, "ssh-secret-name": private_key_secret, "pip-requirements": pip_requirements, + "submodules": submodules, } @@ -976,6 +1001,7 @@ def vcs_checkout_from_args(options): revision, ssh_key_file, ssh_known_hosts_file, + options["submodules"], ) elif options["repo-type"] == "hg": if not revision and not ref: diff --git a/src/taskgraph/transforms/base.py b/src/taskgraph/transforms/base.py index fda0c584f..82205b8ca 100644 --- a/src/taskgraph/transforms/base.py +++ b/src/taskgraph/transforms/base.py @@ -26,6 +26,7 @@ class RepoConfig: path: str = "" head_rev: Union[str, None] = None ssh_secret_name: Union[str, None] = None + submodules: Union[bool, List[str], None] = None @dataclass(frozen=True, eq=False) diff --git a/src/taskgraph/transforms/run/common.py b/src/taskgraph/transforms/run/common.py index 66466bc5f..7802b1062 100644 --- a/src/taskgraph/transforms/run/common.py +++ b/src/taskgraph/transforms/run/common.py @@ -143,6 +143,11 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False): } ) for repo_config in repo_configs.values(): + if isinstance(repo_config.submodules, list): + repo_submods = ":".join(repo_config.submodules) + else: + repo_submods = "auto" if repo_config.submodules else None + env.update( { f"{repo_config.prefix.upper()}_{key}": value @@ -153,6 +158,7 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False): "HEAD_REF": repo_config.head_ref, "REPOSITORY_TYPE": repo_config.type, "SSH_SECRET_NAME": repo_config.ssh_secret_name, + "SUBMODULES": repo_submods, }.items() if value is not None } diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index f0ed878c1..15cc86e09 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -126,7 +126,41 @@ def test_install_pip_requirements( { "base-repo": "https://hg.mozilla.org/mozilla-unified", }, - ) + ), + pytest.param( + { + "REPOSITORY_TYPE": "git", + "BASE_REPOSITORY": "https://github.com/mozilla-mobile/mozilla-vpn-client.git", + "HEAD_REPOSITORY": "https://github.com/mozilla-mobile/mozilla-vpn-client.git", + "HEAD_REV": "abcdef", + "SUBMODULES": "3rdparty/i18n", + }, + {}, + ), + pytest.param( + { + "REPOSITORY_TYPE": "git", + "BASE_REPOSITORY": "https://github.com/mozilla-mobile/mozilla-vpn-client.git", + "HEAD_REPOSITORY": "https://github.com/mozilla-mobile/mozilla-vpn-client.git", + "HEAD_REV": "abcdef", + "SUBMODULES": "auto", + }, + { + "submodules": True, + }, + ), + pytest.param( + { + "REPOSITORY_TYPE": "git", + "BASE_REPOSITORY": "https://github.com/mozilla-mobile/mozilla-vpn-client.git", + "HEAD_REPOSITORY": "https://github.com/mozilla-mobile/mozilla-vpn-client.git", + "HEAD_REV": "abcdef", + "SUBMODULES": "yes", + }, + { + "submodules": True, + }, + ), ], ) def test_collect_vcs_options(monkeypatch, run_task_mod, env, extra_expected): @@ -159,6 +193,7 @@ def test_collect_vcs_options(monkeypatch, run_task_mod, env, extra_expected): "ssh-secret-name": env.get("SSH_SECRET_NAME"), "sparse-profile": False, "store-path": env.get("HG_STORE_PATH"), + "submodules": env.get("SUBMODULES"), } if "PIP_REQUIREMENTS" in env: expected["pip-requirements"] = os.path.join( @@ -283,32 +318,87 @@ def git_current_rev(cwd): ).strip() +@pytest.fixture() +def mock_homedir(monkeypatch): + "Mock home directory to isolate git config changes" + with tempfile.TemporaryDirectory() as fakehome: + env = os.environ.copy() + env["HOME"] = str(fakehome) + monkeypatch.setattr(os, "environ", env) + + # Enable the git file protocol for testing. + subprocess.check_call( + ["git", "config", "--global", "protocol.file.allow", "always"], env=env + ) + yield str(fakehome) + + @pytest.fixture(scope="session") # Tests shouldn't change this repo def mock_git_repo(): "Mock repository with files, commits and branches for using as source" + + def _create_empty_repo(path): + subprocess.check_call(["git", "init", "-b", "main", path]) + subprocess.check_call(["git", "config", "user.name", "pytest"], cwd=path) + subprocess.check_call(["git", "config", "user.email", "py@tes.t"], cwd=path) + + def _commit_file(message, filename, path): + with open(os.path.join(path, filename), "w") as fout: + fout.write("test file content") + subprocess.check_call(["git", "add", filename], cwd=path) + subprocess.check_call(["git", "commit", "-m", message], cwd=path) + return git_current_rev(path) + with tempfile.TemporaryDirectory() as repo: - repo_path = str(repo) - # Init git repo and setup user config - subprocess.check_call(["git", "init", "-b", "main"], cwd=repo_path) - subprocess.check_call(["git", "config", "user.name", "pytest"], cwd=repo_path) + # Create the submodule repositories + sm1_path = os.path.join(str(repo), "submodule1") + _create_empty_repo(sm1_path) + _commit_file("The first submodule", "first", sm1_path) + + sm2_path = os.path.join(str(repo), "submodule2") + _create_empty_repo(sm2_path) + _commit_file("The second submodule", "second", sm2_path) + + # Create the testing repository. + repo_path = os.path.join(str(repo), "repository") + _create_empty_repo(repo_path) + + # Add submodules (to main branch) subprocess.check_call( - ["git", "config", "user.email", "py@tes.t"], cwd=repo_path + [ + "git", + "-c", + "protocol.file.allow=always", + "submodule", + "add", + f"file://{os.path.abspath(sm1_path)}", + "sm1", + ], + cwd=repo_path, ) - - def _commit_file(message, filename): - with open(os.path.join(repo, filename), "w") as fout: - fout.write("test file content") - subprocess.check_call(["git", "add", filename], cwd=repo_path) - subprocess.check_call(["git", "commit", "-m", message], cwd=repo_path) - return git_current_rev(repo_path) + subprocess.check_call( + [ + "git", + "-c", + "protocol.file.allow=always", + "submodule", + "add", + f"file://{os.path.abspath(sm2_path)}", + "sm2", + ], + cwd=repo_path, + ) + subprocess.check_call(["git", "add", "sm1"], cwd=repo_path) + subprocess.check_call(["git", "add", "sm2"], cwd=repo_path) + subprocess.check_call(["git", "commit", "-m", "Add submodules"], cwd=repo_path) # Commit mainfile (to main branch) - main_commit = _commit_file("Initial commit", "mainfile") + main_commit = _commit_file("Initial commit", "mainfile", repo_path) # New branch mybranch subprocess.check_call(["git", "checkout", "-b", "mybranch"], cwd=repo_path) # Commit branchfile to mybranch branch - branch_commit = _commit_file("File in mybranch", "branchfile") + branch_commit = _commit_file("File in mybranch", "branchfile", repo_path) # Set current branch back to main subprocess.check_call(["git", "checkout", "main"], cwd=repo_path) @@ -316,21 +406,47 @@ def _commit_file(message, filename): @pytest.mark.parametrize( - "base_ref,ref,files,hash_key", + "base_ref,ref,submodules,files,hash_key", [ - (None, None, ["mainfile"], "main"), - (None, "main", ["mainfile"], "main"), - (None, "mybranch", ["mainfile", "branchfile"], "branch"), - ("main", "main", ["mainfile"], "main"), - ("main", "mybranch", ["mainfile", "branchfile"], "branch"), + # Check out the repository in a bunch of states. + (None, None, None, ["mainfile"], "main"), + (None, "main", None, ["mainfile"], "main"), + (None, "mybranch", None, ["mainfile", "branchfile"], "branch"), + ("main", "main", None, ["mainfile"], "main"), + ("main", "mybranch", None, ["mainfile", "branchfile"], "branch"), + # Same tests again - but with submodules. + (None, None, True, ["mainfile", "sm1/first", "sm2/second"], "main"), + (None, "main", True, ["mainfile", "sm1/first", "sm2/second"], "main"), + ( + None, + "mybranch", + True, + ["mainfile", "branchfile", "sm1/first", "sm2/second"], + "branch", + ), + ("main", "main", True, ["mainfile", "sm1/first", "sm2/second"], "main"), + ( + "main", + "mybranch", + True, + ["mainfile", "branchfile", "sm1/first", "sm2/second"], + "branch", + ), + # Tests for submodule matching rules. + (None, "main", "sm1", ["mainfile", "sm1/first"], "main"), + (None, "main", "sm2", ["mainfile", "sm2/second"], "main"), + (None, "main", "one:two:three", ["mainfile"], "main"), + (None, "main", "one:two:sm1", ["mainfile", "sm1/first"], "main"), ], ) def test_git_checkout( mock_stdin, + mock_homedir, run_task_mod, mock_git_repo, base_ref, ref, + submodules, files, hash_key, ): @@ -346,6 +462,7 @@ def test_git_checkout( commit=None, ssh_key_file=None, ssh_known_hosts_file=None, + submodules=submodules, ) # Check desired files exist @@ -367,6 +484,7 @@ def test_git_checkout( def test_git_checkout_with_commit( mock_stdin, + mock_homedir, run_task_mod, mock_git_repo, ): @@ -382,6 +500,7 @@ def test_git_checkout_with_commit( commit=mock_git_repo["branch"], ssh_key_file=None, ssh_known_hosts_file=None, + submodules=None, ) diff --git a/test/test_transforms_run_run_task.py b/test/test_transforms_run_run_task.py index 027732971..2a963b7fa 100644 --- a/test/test_transforms_run_run_task.py +++ b/test/test_transforms_run_run_task.py @@ -168,6 +168,78 @@ def assert_run_task_command_generic_worker(task): ] +def assert_no_checkouts(task): + assert task["worker"]["env"] == { + "MOZ_SCM_LEVEL": "1", + } + assert task["worker"]["command"] == [ + "/usr/local/bin/run-task", + "--", + "bash", + "-cx", + "echo hello world", + ] + + +def assert_with_all_submodules(task): + assert task["worker"]["env"] == { + "CI_BASE_REPOSITORY": "http://hg.example.com", + "CI_HEAD_REF": "default", + "CI_HEAD_REPOSITORY": "http://hg.example.com", + "CI_HEAD_REV": "abcdef", + "CI_REPOSITORY_TYPE": "hg", + "CI_SUBMODULES": "auto", + "HG_STORE_PATH": "/builds/worker/checkouts/hg-store", + "MOZ_SCM_LEVEL": "1", + "REPOSITORIES": '{"ci": "Taskgraph"}', + "VCS_PATH": "/builds/worker/checkouts/vcs", + } + + +def assert_with_one_submodule(task): + assert task["worker"]["env"] == { + "CI_BASE_REPOSITORY": "http://hg.example.com", + "CI_HEAD_REF": "default", + "CI_HEAD_REPOSITORY": "http://hg.example.com", + "CI_HEAD_REV": "abcdef", + "CI_REPOSITORY_TYPE": "hg", + "CI_SUBMODULES": "apple", + "HG_STORE_PATH": "/builds/worker/checkouts/hg-store", + "MOZ_SCM_LEVEL": "1", + "REPOSITORIES": '{"ci": "Taskgraph"}', + "VCS_PATH": "/builds/worker/checkouts/vcs", + } + + +def assert_with_two_submodules(task): + assert task["worker"]["env"] == { + "CI_BASE_REPOSITORY": "http://hg.example.com", + "CI_HEAD_REF": "default", + "CI_HEAD_REPOSITORY": "http://hg.example.com", + "CI_HEAD_REV": "abcdef", + "CI_REPOSITORY_TYPE": "hg", + "CI_SUBMODULES": "orange:banana", + "HG_STORE_PATH": "/builds/worker/checkouts/hg-store", + "MOZ_SCM_LEVEL": "1", + "REPOSITORIES": '{"ci": "Taskgraph"}', + "VCS_PATH": "/builds/worker/checkouts/vcs", + } + + +def assert_change_head(task): + assert task["worker"]["env"] == { + "CI_BASE_REPOSITORY": "http://hg.example.com", + "CI_HEAD_REF": "default", + "CI_HEAD_REPOSITORY": "http://hg.somewhere.com", + "CI_HEAD_REV": "an-awesome-branch", + "CI_REPOSITORY_TYPE": "hg", + "HG_STORE_PATH": "/builds/worker/checkouts/hg-store", + "MOZ_SCM_LEVEL": "1", + "REPOSITORIES": '{"ci": "Taskgraph"}', + "VCS_PATH": "/builds/worker/checkouts/vcs", + } + + @pytest.mark.parametrize( "task", ( @@ -208,6 +280,63 @@ def assert_run_task_command_generic_worker(task): }, id="run_task_command_generic_worker", ), + pytest.param( + { + "run": { + "checkout": False, + }, + }, + id="no_checkouts", + ), + pytest.param( + { + "run": { + "checkout": { + "ci": { + "submodules": True, + } + }, + }, + }, + id="with_all_submodules", + ), + pytest.param( + { + "run": { + "checkout": { + "ci": { + "submodules": ["apple"], + } + }, + }, + }, + id="with_one_submodule", + ), + pytest.param( + { + "run": { + "checkout": { + "ci": { + "submodules": ["orange", "banana"], + } + }, + }, + }, + id="with_two_submodules", + ), + pytest.param( + { + "run": { + "checkout": { + "ci": { + "head_repository": "http://hg.somewhere.com", + "head_rev": "an-awesome-branch", + }, + }, + }, + }, + id="change_head", + ), ), ) def test_run_task(request, run_task_using, task):