From 91f9803e7d0486fbc64da37ab5686e2f06852b27 Mon Sep 17 00:00:00 2001 From: Frederic Lepied Date: Mon, 11 Sep 2023 17:10:29 +0200 Subject: [PATCH] container mode --- .github/workflows/main.yml | 8 +++++- README.md | 2 +- dist/index.js | 3 +++ golang.py | 30 ++++++++++++++++----- index.js | 3 +++ python.py | 54 ++++++++++++++++++++++++++------------ stage1.py | 26 +++++++++++++----- stage2.py | 11 ++++++-- 8 files changed, 103 insertions(+), 34 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4db2e29..680435c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,9 +8,15 @@ on: jobs: test-inject-dependencies: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04, macos-latest, ubuntu-latest] steps: + - name: Python3 version + run: python3 --version + - name: Checkout code uses: actions/checkout@v3 diff --git a/README.md b/README.md index d2f527f..e2dd853 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ jobs: - [x] [stage 2: python support](https://github.com/depends-on/depends-on-action/issues/8) - [x] [stage 2: python poetry support](https://github.com/depends-on/depends-on-action/issues/18) - [x] [stage 2: python subdir support](https://github.com/depends-on/depends-on-action/issues/19) -- [ ] [stage 2: Container support](https://github.com/depends-on/depends-on-action/issues/17) +- [x] [stage 2: Container support](https://github.com/depends-on/depends-on-action/issues/17) - [ ] [stage 2: custom injection](https://github.com/depends-on/depends-on-action/issues/4) - [ ] [stage 2: Github action support](https://github.com/depends-on/depends-on-action/issues/5) - [ ] [stage 1: gerrit support for software-factory.io](https://github.com/depends-on/depends-on-action/issues/6) diff --git a/dist/index.js b/dist/index.js index 5090827..30f5fa5 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9868,6 +9868,9 @@ async function run() { fs.writeFileSync('.depends-on.json', JSON.stringify(changeJson)); console.log(`writing .depends-on.json: ${JSON.stringify(changeJson)}`); + // export token as the GITHUB_TOKEN env variable + core.exportVariable('GITHUB_TOKEN', token); + // the bundle is in the dist sub-directory execSync(`${__dirname}/../stage1.py ${checkUnmergedPr}`, { encoding: 'utf-8' }); } catch (error) { diff --git a/golang.py b/golang.py index e750586..657b579 100644 --- a/golang.py +++ b/golang.py @@ -5,7 +5,7 @@ import sys -def process_golang(main_dir, dirs): +def process_golang(main_dir, dirs, container_mode): "Add replace directives in go.mod for the local dependencies." go_mod = os.path.join(main_dir, "go.mod") if not os.path.exists(go_mod): @@ -22,12 +22,28 @@ def process_golang(main_dir, dirs): with open(go_mod, "a", encoding="UTF-8") as out_stream: for mod in github_mods: if mod in dirs: - print( - f'Adding replace directive in go.mod for {mod} => {dirs[mod]["path"]}', - file=sys.stderr, - ) - # do not use "go mod edit -replace" as go could be not installed at this stage - out_stream.write(f'replace {mod} => {dirs[mod]["path"]}\n') + if container_mode: + # remove https:// at the beginning of the url and .git at the end + fork_url = ( + dirs[mod]["fork_url"] + .replace("https://", "", 1) + .replace(".git", "", 1) + ) + print( + f'Adding replace directive in go.mod for {mod} => {fork_url} {dirs[mod]["branch"]}', + file=sys.stderr, + ) + # do not use "go mod edit -replace" as go could be not installed at this stage + out_stream.write( + f'replace {mod} => {fork_url} {dirs[mod]["branch"]}\n' + ) + else: + print( + f'Adding replace directive in go.mod for {mod} => {dirs[mod]["path"]}', + file=sys.stderr, + ) + # do not use "go mod edit -replace" as go could be not installed at this stage + out_stream.write(f'replace {mod} => {dirs[mod]["path"]}\n') nb_replace += 1 return nb_replace > 0 diff --git a/index.js b/index.js index 2c33302..e851c88 100644 --- a/index.js +++ b/index.js @@ -47,6 +47,9 @@ async function run() { fs.writeFileSync('.depends-on.json', JSON.stringify(changeJson)); console.log(`writing .depends-on.json: ${JSON.stringify(changeJson)}`); + // export token as the GITHUB_TOKEN env variable + core.exportVariable('GITHUB_TOKEN', token); + // the bundle is in the dist sub-directory execSync(`${__dirname}/../stage1.py ${checkUnmergedPr}`, { encoding: 'utf-8' }); } catch (error) { diff --git a/python.py b/python.py index 3a95dbb..42151d7 100644 --- a/python.py +++ b/python.py @@ -24,11 +24,11 @@ def get_modules(dirs): os.path.join(dirs[_dir]["path"], "pyproject.toml") ) if name: - python_mods[name] = dirs[_dir]["path"] + python_mods[name] = dirs[_dir] return python_mods -def process_python_requirements(main_dir, dirs): +def process_python_requirements(main_dir, dirs, container_mode): "Replace modules in requirements.txt for the local dependencies." requirements_txt = os.path.join(main_dir, "requirements.txt") requirements_txt_new = requirements_txt + ".new" @@ -45,11 +45,20 @@ def process_python_requirements(main_dir, dirs): match = re.match(r"^\s*(\w+)", line) if match and match.group(1) in module_dirs: mod = match.group(1) - print( - f"Replacing {mod} in requirements.txt with {module_dirs[mod]}", - file=sys.stderr, - ) - out_stream.write(f"-e {module_dirs[mod]}\n") + if container_mode: + # doc at https://pip.pypa.io/en/stable/cli/pip_install/#git + pkg = f"{mod} @ git+{module_dirs[mod]['fork_url']}@{module_dirs[mod]['branch']}" + print( + f"Replacing {mod} in requirements.txt with {pkg}", + file=sys.stderr, + ) + out_stream.write(f"{pkg}\n") + else: + print( + f"Replacing {mod} in requirements.txt with {module_dirs[mod]['path']}", + file=sys.stderr, + ) + out_stream.write(f"-e {module_dirs[mod]['path']}\n") nb_replace += 1 else: out_stream.write(line) @@ -57,7 +66,7 @@ def process_python_requirements(main_dir, dirs): return nb_replace > 0 -def process_python_pyproject(main_dir, dirs): +def process_python_pyproject(main_dir, dirs, container_mode): "Replace modules in pyproject.toml for the local dependencies." pyproject_toml = os.path.join(main_dir, "pyproject.toml") pyproject_toml_new = pyproject_toml + ".new" @@ -75,11 +84,22 @@ def process_python_pyproject(main_dir, dirs): match = re.match(r"^\s*(\w+)\s*=", line) if match and match.group(1) in module_dirs: mod = match.group(1) - print( - f"Replacing {mod} in pyproject.toml with {module_dirs[mod]}", - file=sys.stderr, - ) - out_stream.write(f'{mod} = {{ path = "{module_dirs[mod]}" }}\n') + if container_mode: + # doc at https://python-poetry.org/docs/dependency-specification/#git-dependencies + pkg = f"{mod} = {{ git = \"{module_dirs[mod]['fork_url']}\", branch = \"{module_dirs[mod]['branch']}\" }}" + print( + f"Replacing {mod} in pyproject.toml with {pkg}", + file=sys.stderr, + ) + out_stream.write(f"{pkg}\n") + else: + print( + f"Replacing {mod} in pyproject.toml with {module_dirs[mod]['path']}", + file=sys.stderr, + ) + out_stream.write( + f'{mod} = {{ path = "{module_dirs[mod]["path"]}" }}\n' + ) nb_replace += 1 else: out_stream.write(line) @@ -87,13 +107,13 @@ def process_python_pyproject(main_dir, dirs): return nb_replace > 0 -def process_python(main_dir, dirs): +def process_python(main_dir, dirs, container_mode): "Process python dependencies." # process pyprohect.toml first because they can both be present # and it supposed to be the main one - return process_python_pyproject(main_dir, dirs) or process_python_requirements( - main_dir, dirs - ) + return process_python_pyproject( + main_dir, dirs, container_mode + ) or process_python_requirements(main_dir, dirs, container_mode) # python.py ends here diff --git a/stage1.py b/stage1.py index 3f6b6b4..9838585 100755 --- a/stage1.py +++ b/stage1.py @@ -6,8 +6,7 @@ import os import re import sys - -import requests +from urllib.request import Request, urlopen def load_depends_on(from_dir): @@ -39,6 +38,10 @@ def extract_github_change(fork_url, branch, main_url, main_branch, repo): command(f"cd {repo} && git remote add pr {fork_url} && git fetch pr {branch}") # extract the master/main branch name command(f"cd {repo} && git checkout -b {branch} --track pr/{branch}") + # set a dummy user name and email for the merge process to work + command( + f"cd {repo} && git config user.name 'Depends-On' && git config user.email 'depends-on@localhost'" + ) # merge the main branch into the PR branch command(f"cd {repo} && git merge origin/{main_branch} --no-edit") @@ -46,10 +49,21 @@ def extract_github_change(fork_url, branch, main_url, main_branch, repo): def get_pull_request_info(org, repo, pr_number): "Get the information about the Pull request." - # get the information about the Pull request - pr_info = requests.get( - f"https://api.github.com/repos/{org}/{repo}/pulls/{pr_number}" - ).json() + token = os.environ.get("GITHUB_TOKEN") + + # get the information about the Pull request using the GitHub API + # set the Authorization header to use the token + req = Request( + f"https://api.github.com/repos/{org}/{repo}/pulls/{pr_number}", + ) + req.add_header("Accept", "application/vnd.github.v3+json") + if token: + print("Using GitHub token", file=sys.stderr) + req.add_header("Authorization", f"token {token}") + with urlopen(req) as response: + response_content = response.read() + response_content.decode("utf-8") + pr_info = json.loads(response_content) return pr_info diff --git a/stage2.py b/stage2.py index 5132efb..b2950af 100755 --- a/stage2.py +++ b/stage2.py @@ -58,6 +58,11 @@ def directories(top_dir, main_dir): return ret +def detect_container_mode(main_dir): + "Return True if main_dir contains a Dockerfile" + return os.path.exists(os.path.join(main_dir, "Dockerfile")) + + def main(): "Main function." main_dir = os.getcwd() @@ -69,8 +74,10 @@ def main(): file=sys.stderr, ) - process_golang(main_dir, dirs) - process_python(main_dir, dirs) + container_mode = detect_container_mode(main_dir) + print(f"{container_mode=}", file=sys.stderr) + process_golang(main_dir, dirs, container_mode) + process_python(main_dir, dirs, container_mode) if __name__ == "__main__":