From a587f9beb335e90386a9c6eb7637188651187527 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 3 Nov 2022 11:47:59 -0700 Subject: [PATCH 01/18] Generate Releases --- .gitignore | 38 ++----------------- main.py | 14 +++++++ release_creation/__init__.py | 0 release_creation/github.py | 35 +++++++++++++++++ release_creation/install.sh | 4 ++ .../requirements/v1.2.latest.requirements.txt | 10 +++++ .../requirements/v1.2.pre.requirements.txt | 10 +++++ .../requirements/v1.3.latest.requirements.txt | 10 +++++ release_creation/snapshots.py | 36 ++++++++++++++++++ 9 files changed, 123 insertions(+), 34 deletions(-) create mode 100644 main.py create mode 100644 release_creation/__init__.py create mode 100644 release_creation/github.py create mode 100755 release_creation/install.sh create mode 100644 release_creation/requirements/v1.2.latest.requirements.txt create mode 100644 release_creation/requirements/v1.2.pre.requirements.txt create mode 100644 release_creation/requirements/v1.3.latest.requirements.txt create mode 100644 release_creation/snapshots.py diff --git a/.gitignore b/.gitignore index b6e47617..05099506 100644 --- a/.gitignore +++ b/.gitignore @@ -51,36 +51,9 @@ coverage.xml .hypothesis/ .pytest_cache/ -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - # PyBuilder target/ -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - # pyenv .python-version @@ -94,13 +67,6 @@ ipython_config.py # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - # Environments .env .venv @@ -127,3 +93,7 @@ dmypy.json # Pyre type checker .pyre/ + +# Project Specific +tmp/ +dbt-core*.zip \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 00000000..b1f23864 --- /dev/null +++ b/main.py @@ -0,0 +1,14 @@ +import os + +from release_creation.github import create_new_release_for_version, get_next_release_version +from release_creation.snapshots import generate_snapshot + + +def main(): + version = os.environ['INPUT_VERSION'] #1.3.4 + target_version = get_next_release_version(version) #1.3.6 + snapshot_path = generate_snapshot(target_version) + create_new_release_for_version(target_version, snapshot_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/release_creation/__init__.py b/release_creation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release_creation/github.py b/release_creation/github.py new file mode 100644 index 00000000..a4732b34 --- /dev/null +++ b/release_creation/github.py @@ -0,0 +1,35 @@ +import os +from semantic_version import Version, match +from github import Github + +_GH_SNAPSHOT_REPO = "dbt-labs/dbt-core-snapshots" +_GH_ACCESS_TOKEN = os.environ["GH_ACCESS_TOKEN"] + + +def get_github_client() -> Github: + return Github(_GH_ACCESS_TOKEN) + + +def get_next_release_version(input_version:str) -> Version: + gh = get_github_client() + target_version = Version(input_version) + target_match = f"~{target_version.major}.{target_version.minor}" + latest = Version(major=target_version.major, minor=target_version.minor, patch=0) + repo = gh.get_repo(_GH_SNAPSHOT_REPO) + releases = repo.get_releases() + for r in releases: + release_version = Version.coerce(r.tag_name) + if match(target_match, r.tag_name) and release_version > latest: + latest = release_version + return latest.next_patch() + + +def create_new_release_for_version(release_version:Version, asset_url)->None: + gh = get_github_client() + release_tag = str(release_version) + repo = gh.get_repo(_GH_SNAPSHOT_REPO) + created_release = repo.create_git_release(tag=release_tag, name=f'Snapshot Release', message='') + try: + created_release.upload_asset(path=asset_url, name=f"snapshot_core_all_adapters") + except: + created_release.delete_release() \ No newline at end of file diff --git a/release_creation/install.sh b/release_creation/install.sh new file mode 100755 index 00000000..9a451823 --- /dev/null +++ b/release_creation/install.sh @@ -0,0 +1,4 @@ +pip install -r release_creation/requirements.txt \ +--no-index \ +--force-reinstall \ +--find-links tmp/ \ No newline at end of file diff --git a/release_creation/requirements/v1.2.latest.requirements.txt b/release_creation/requirements/v1.2.latest.requirements.txt new file mode 100644 index 00000000..f57db78c --- /dev/null +++ b/release_creation/requirements/v1.2.latest.requirements.txt @@ -0,0 +1,10 @@ +dbt-core~=1.2.0b1 --no-binary dbt-postgres +dbt-snowflake~=1.2.0b1 +dbt-bigquery~=1.2.0b1 +dbt-redshift~=1.2.0b1 +dbt-postgres~=1.2.0b1 +dbt-spark[PyHive,ODBC]~=1.2.0b1 +dbt-databricks==1.2.3 +dbt-rpc~=0.1.1 +grpcio-status~=1.47.0 +pyasn1-modules~=0.2.1 \ No newline at end of file diff --git a/release_creation/requirements/v1.2.pre.requirements.txt b/release_creation/requirements/v1.2.pre.requirements.txt new file mode 100644 index 00000000..dbe2861a --- /dev/null +++ b/release_creation/requirements/v1.2.pre.requirements.txt @@ -0,0 +1,10 @@ +dbt-core~=1.2.0B1 --no-binary dbt-postgres +dbt-snowflake~=1.2.0B1 +dbt-bigquery~=1.2.0B1 +dbt-redshift~=1.2.0B1 +dbt-postgres~=1.2.0B1 +dbt-spark[PyHive,ODBC]~=1.2.0B1 +dbt-databricks==1.2.3 +dbt-rpc~=0.1.1 +grpcio-status~=1.47.0 +pyasn1-modules~=0.2.1 \ No newline at end of file diff --git a/release_creation/requirements/v1.3.latest.requirements.txt b/release_creation/requirements/v1.3.latest.requirements.txt new file mode 100644 index 00000000..23dd9f71 --- /dev/null +++ b/release_creation/requirements/v1.3.latest.requirements.txt @@ -0,0 +1,10 @@ +dbt-core~=1.3.0 --no-binary dbt-postgres +dbt-snowflake~=1.3.0 +dbt-bigquery~=1.3.0 +dbt-redshift~=1.3.0 +dbt-postgres~=1.3.0 +dbt-spark[PyHive,ODBC]~=1.3.0 +dbt-databricks==1.3.0 +dbt-rpc~=0.1.1 +grpcio-status~=1.47.0 +pyasn1-modules~=0.2.1 \ No newline at end of file diff --git a/release_creation/snapshots.py b/release_creation/snapshots.py new file mode 100644 index 00000000..7552471d --- /dev/null +++ b/release_creation/snapshots.py @@ -0,0 +1,36 @@ +from sqlite3 import adapters +import subprocess +import os +import shutil +from typing import Optional +from semantic_version import Version + +_BASE_DOWNLOAD_CMD = """pip3 download --dest ./tmp """ +_OUTPUT_ARCHIVE_FILE_BASE = "dbt-core-all-adapters-snapshot" +_FILE_DIR = os.path.dirname(os.path.realpath(__file__)) + +# def _generate_package_req(version: dict): +# suffix = version['suffix'] if 'suffix' in version else '' +# return f"{version['name']}{version['version_pinning']}{version['sem_ver']} {suffix} \\\n" + +def _generate_download_command(major_version:Optional[int], minor_version:Optional[int], is_pre:bool = False) -> str: + download_command = _BASE_DOWNLOAD_CMD + suffix = "latest" + if is_pre: + download_command += "--pre" + suffix = "pre" + download_command += f" -r {_FILE_DIR}/requirements/v{major_version}.{minor_version}.{suffix}.requirements.txt" + return download_command + +def generate_snapshot(target_version: Version) -> str: + is_pre = True if target_version.prerelease else False + download_cmd = _generate_download_command(major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre) + print(download_cmd) + subprocess.run("python -m pip install --upgrade pip", shell=True, check=True) + subprocess.run("rm -r tmp", shell=True, check=True) + subprocess.run("mkdir tmp", shell=True, check=True) + subprocess.run(download_cmd, shell=True, check=True) + subprocess.run(['sh',f"./release_creation/install.sh"], check=True) + archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{str(target_version)}" + shutil.make_archive(archive_path, 'zip', "tmp") + return archive_path + ".zip" \ No newline at end of file From af89f96dc2f463f1847ee536e5ef75a8a99d87ae Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 15 Nov 2022 17:15:20 -0800 Subject: [PATCH 02/18] clean up code and add snapshot diff functionality --- .gitignore | 3 +- main.py | 14 ---- release_creation/github.py | 35 -------- release_creation/github_client/__init__.py | 0 release_creation/github_client/releases.py | 79 +++++++++++++++++++ release_creation/install.sh | 4 - release_creation/main.py | 16 ++++ .../requirements/v1.2.pre.requirements.txt | 10 --- release_creation/snapshot/__init__.py | 0 release_creation/snapshot/create.py | 34 ++++++++ release_creation/snapshot/download.sh | 5 ++ release_creation/snapshot/freeze.sh | 1 + release_creation/snapshot/install.sh | 7 ++ .../requirements/test.requirements.txt | 8 ++ .../requirements/v1.2.latest.requirements.txt | 10 +++ .../requirements/v1.2.pre.requirements.txt} | 2 +- .../requirements/v1.3.latest.requirements.txt | 0 release_creation/snapshots.py | 36 --------- requirements.txt | 2 + 19 files changed, 165 insertions(+), 101 deletions(-) delete mode 100644 main.py delete mode 100644 release_creation/github.py create mode 100644 release_creation/github_client/__init__.py create mode 100644 release_creation/github_client/releases.py delete mode 100755 release_creation/install.sh create mode 100644 release_creation/main.py delete mode 100644 release_creation/requirements/v1.2.pre.requirements.txt create mode 100644 release_creation/snapshot/__init__.py create mode 100644 release_creation/snapshot/create.py create mode 100755 release_creation/snapshot/download.sh create mode 100755 release_creation/snapshot/freeze.sh create mode 100755 release_creation/snapshot/install.sh create mode 100644 release_creation/snapshot/requirements/test.requirements.txt create mode 100644 release_creation/snapshot/requirements/v1.2.latest.requirements.txt rename release_creation/{requirements/v1.2.latest.requirements.txt => snapshot/requirements/v1.2.pre.requirements.txt} (86%) rename release_creation/{ => snapshot}/requirements/v1.3.latest.requirements.txt (100%) delete mode 100644 release_creation/snapshots.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 05099506..e25d4b97 100644 --- a/.gitignore +++ b/.gitignore @@ -96,4 +96,5 @@ dmypy.json # Project Specific tmp/ -dbt-core*.zip \ No newline at end of file +dbt-core*.zip +snapshot*.txt \ No newline at end of file diff --git a/main.py b/main.py deleted file mode 100644 index b1f23864..00000000 --- a/main.py +++ /dev/null @@ -1,14 +0,0 @@ -import os - -from release_creation.github import create_new_release_for_version, get_next_release_version -from release_creation.snapshots import generate_snapshot - - -def main(): - version = os.environ['INPUT_VERSION'] #1.3.4 - target_version = get_next_release_version(version) #1.3.6 - snapshot_path = generate_snapshot(target_version) - create_new_release_for_version(target_version, snapshot_path) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/release_creation/github.py b/release_creation/github.py deleted file mode 100644 index a4732b34..00000000 --- a/release_creation/github.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -from semantic_version import Version, match -from github import Github - -_GH_SNAPSHOT_REPO = "dbt-labs/dbt-core-snapshots" -_GH_ACCESS_TOKEN = os.environ["GH_ACCESS_TOKEN"] - - -def get_github_client() -> Github: - return Github(_GH_ACCESS_TOKEN) - - -def get_next_release_version(input_version:str) -> Version: - gh = get_github_client() - target_version = Version(input_version) - target_match = f"~{target_version.major}.{target_version.minor}" - latest = Version(major=target_version.major, minor=target_version.minor, patch=0) - repo = gh.get_repo(_GH_SNAPSHOT_REPO) - releases = repo.get_releases() - for r in releases: - release_version = Version.coerce(r.tag_name) - if match(target_match, r.tag_name) and release_version > latest: - latest = release_version - return latest.next_patch() - - -def create_new_release_for_version(release_version:Version, asset_url)->None: - gh = get_github_client() - release_tag = str(release_version) - repo = gh.get_repo(_GH_SNAPSHOT_REPO) - created_release = repo.create_git_release(tag=release_tag, name=f'Snapshot Release', message='') - try: - created_release.upload_asset(path=asset_url, name=f"snapshot_core_all_adapters") - except: - created_release.delete_release() \ No newline at end of file diff --git a/release_creation/github_client/__init__.py b/release_creation/github_client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py new file mode 100644 index 00000000..851338fa --- /dev/null +++ b/release_creation/github_client/releases.py @@ -0,0 +1,79 @@ +import os +import requests +from typing import Dict, List, Optional, Set, Tuple +from semantic_version import Version, match +from github import Github +from github.GitRelease import GitRelease +from github.GitReleaseAsset import GitReleaseAsset + +_GH_SNAPSHOT_REPO = "dbt-labs/dbt-core-snapshots" +_GH_ACCESS_TOKEN = os.environ["GH_ACCESS_TOKEN"] +_SNAP_REQ_NAME = "snapshot_requirements" + + +def get_github_client() -> Github: + return Github(_GH_ACCESS_TOKEN) + + +def get_latest_release(input_version:str) -> Tuple[Version, Optional[GitRelease]]: + gh = get_github_client() + target_version = Version(input_version) + target_match = f"~{target_version.major}.{target_version.minor}" + latest = Version(major=target_version.major, minor=target_version.minor, patch=0) + repo = gh.get_repo(_GH_SNAPSHOT_REPO) + releases = repo.get_releases() + latest_release = None + for r in releases: + release_version = Version.coerce(r.tag_name) + if match(target_match, r.tag_name) and release_version > latest: + latest = release_version + latest_release = r + return latest, latest_release + +def _get_local_snapshot_reqs(snapshot_req_path:str) -> List[str]: + with open(snapshot_req_path) as f: + reqs = f.read() + return reqs.split() + +def _get_gh_release_asset(release_asset: GitReleaseAsset) -> List[str]: + resp = requests.get(release_asset.browser_download_url) + resp.raise_for_status() + return resp.content.decode("utf-8").split() + +def _compare_reqs(snapshot_req: List[str], release_req: List[str]) -> Tuple[Set[str], Set[str]]: + snapshot_req_set = set(snapshot_req) + release_req_set = set(release_req) + added_req = snapshot_req_set - release_req_set + removed_req = release_req_set - snapshot_req_set + return added_req, removed_req + + +def _diff_snapshot_requirements(snapshot_req_path:str, latest_release: Optional[GitRelease]): + diff_result = "" + if latest_release: + release_reqs = [_asset for _asset in latest_release.get_assets() if _SNAP_REQ_NAME in _asset.name] + snapshot_req = _get_local_snapshot_reqs(snapshot_req_path=snapshot_req_path) + release_req = _get_gh_release_asset(release_reqs[0]) + added, removed = _compare_reqs(snapshot_req=snapshot_req, release_req=release_req) + if added: + diff_result += "Added:\n* "+ "\n* ".join(added) + "\n___\n" + if removed: + diff_result += "\nRemoved:\n* " + "\n* ".join(removed) + "\n" + + return diff_result + else: + return "No prior snapshot" + + +def create_new_release_for_version(release_version:Version, assets:Dict, latest_release: Optional[GitRelease], )->None: + gh = get_github_client() + release_tag = str(release_version) + repo = gh.get_repo(_GH_SNAPSHOT_REPO) + release_body = _diff_snapshot_requirements(assets[_SNAP_REQ_NAME], latest_release=latest_release) + created_release = repo.create_git_release(tag=release_tag, name='Snapshot Release', message=release_body) + try: + for asset_name, asset_path in assets.items(): + created_release.upload_asset(path=asset_path, name=asset_name) + except Exception as e: + created_release.delete_release() + raise e \ No newline at end of file diff --git a/release_creation/install.sh b/release_creation/install.sh deleted file mode 100755 index 9a451823..00000000 --- a/release_creation/install.sh +++ /dev/null @@ -1,4 +0,0 @@ -pip install -r release_creation/requirements.txt \ ---no-index \ ---force-reinstall \ ---find-links tmp/ \ No newline at end of file diff --git a/release_creation/main.py b/release_creation/main.py new file mode 100644 index 00000000..8ac7f7e2 --- /dev/null +++ b/release_creation/main.py @@ -0,0 +1,16 @@ +import os + +from github_client.releases import create_new_release_for_version, get_latest_release +from snapshot.create import generate_snapshot + +BASE_DIR = os.path.dirname(os.path.realpath(__file__)) + +def main(): + version = os.environ['INPUT_VERSION'] #e.g. v1.3.4 + latest_version, latest_release = get_latest_release(version) + target_version = latest_version.next_patch() + snapshot_assets = generate_snapshot(target_version) + create_new_release_for_version(target_version, snapshot_assets, latest_release) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/release_creation/requirements/v1.2.pre.requirements.txt b/release_creation/requirements/v1.2.pre.requirements.txt deleted file mode 100644 index dbe2861a..00000000 --- a/release_creation/requirements/v1.2.pre.requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -dbt-core~=1.2.0B1 --no-binary dbt-postgres -dbt-snowflake~=1.2.0B1 -dbt-bigquery~=1.2.0B1 -dbt-redshift~=1.2.0B1 -dbt-postgres~=1.2.0B1 -dbt-spark[PyHive,ODBC]~=1.2.0B1 -dbt-databricks==1.2.3 -dbt-rpc~=0.1.1 -grpcio-status~=1.47.0 -pyasn1-modules~=0.2.1 \ No newline at end of file diff --git a/release_creation/snapshot/__init__.py b/release_creation/snapshot/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py new file mode 100644 index 00000000..415e5af3 --- /dev/null +++ b/release_creation/snapshot/create.py @@ -0,0 +1,34 @@ +from typing import Dict, List, Optional, Tuple +from semantic_version import Version +import os +import subprocess +import shutil + + +_OUTPUT_ARCHIVE_FILE_BASE = "dbt-core-all-adapters-snapshot" +_FILE_DIR = os.path.dirname(os.path.realpath(__file__)) +_SNAPSHOT_FILE = f"{_FILE_DIR}/snapshot.requirements.txt" + +def _generate_download_command_args(major_version:Optional[int], minor_version:Optional[int], is_pre:bool = False) -> Tuple[List, str]: + download_command = [] + suffix = "latest" + if is_pre: + download_command.append("--pre") + suffix = "pre" + requirements_prefix = f"v{major_version}.{minor_version}.{suffix}" + download_command.append(f"-r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt") + return download_command, requirements_prefix + +def generate_snapshot(target_version: Version) -> Dict[str, str]: + is_pre = True if target_version.prerelease else False + download_cmd, requirements_prefix = _generate_download_command_args(major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre) + subprocess.run(['sh',f"{_FILE_DIR}/download.sh", *download_cmd], check=True) + subprocess.run(['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix], check=True) + subprocess.run(['sh',f"{_FILE_DIR}/freeze.sh", _SNAPSHOT_FILE], check=True) + archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{str(target_version)}" + shutil.make_archive(archive_path, 'zip', 'tmp') + assets = { + "snapshot_core_all_adapters": archive_path + ".zip", + "snapshot_requirements": _SNAPSHOT_FILE + } + return assets \ No newline at end of file diff --git a/release_creation/snapshot/download.sh b/release_creation/snapshot/download.sh new file mode 100755 index 00000000..0cc2aa99 --- /dev/null +++ b/release_creation/snapshot/download.sh @@ -0,0 +1,5 @@ +python -m pip install --upgrade pip +rm -rf tmp +mkdir tmp +pip3 download --dest ./tmp \ + $1 \ No newline at end of file diff --git a/release_creation/snapshot/freeze.sh b/release_creation/snapshot/freeze.sh new file mode 100755 index 00000000..4a9df7cf --- /dev/null +++ b/release_creation/snapshot/freeze.sh @@ -0,0 +1 @@ +pip freeze --path ./target > $1 \ No newline at end of file diff --git a/release_creation/snapshot/install.sh b/release_creation/snapshot/install.sh new file mode 100755 index 00000000..49f4fc71 --- /dev/null +++ b/release_creation/snapshot/install.sh @@ -0,0 +1,7 @@ +rm -rf target +mkdir target +pip install -r $1/requirements/$2.requirements.txt \ +--no-index \ +--force-reinstall \ +--find-links ./tmp/ \ +--target ./target \ No newline at end of file diff --git a/release_creation/snapshot/requirements/test.requirements.txt b/release_creation/snapshot/requirements/test.requirements.txt new file mode 100644 index 00000000..df6f12fc --- /dev/null +++ b/release_creation/snapshot/requirements/test.requirements.txt @@ -0,0 +1,8 @@ +dbt-core --no-binary dbt-postgres +dbt-snowflake +dbt-bigquery +dbt-redshift +dbt-postgres +dbt-spark[PyHive,ODBC] +dbt-databricks +dbt-rpc \ No newline at end of file diff --git a/release_creation/snapshot/requirements/v1.2.latest.requirements.txt b/release_creation/snapshot/requirements/v1.2.latest.requirements.txt new file mode 100644 index 00000000..6e6eeac4 --- /dev/null +++ b/release_creation/snapshot/requirements/v1.2.latest.requirements.txt @@ -0,0 +1,10 @@ +dbt-core~=1.2.0 --no-binary dbt-postgres +dbt-snowflake~=1.2.0 +dbt-bigquery~=1.2.0 +dbt-redshift~=1.2.0 +dbt-postgres~=1.2.0 +dbt-spark[PyHive,ODBC]~=1.2.0 +dbt-databricks==1.2.3 +dbt-rpc~=0.1.1 +grpcio-status~=1.47.0 +pyasn1-modules~=0.2.1 \ No newline at end of file diff --git a/release_creation/requirements/v1.2.latest.requirements.txt b/release_creation/snapshot/requirements/v1.2.pre.requirements.txt similarity index 86% rename from release_creation/requirements/v1.2.latest.requirements.txt rename to release_creation/snapshot/requirements/v1.2.pre.requirements.txt index f57db78c..4f2612a2 100644 --- a/release_creation/requirements/v1.2.latest.requirements.txt +++ b/release_creation/snapshot/requirements/v1.2.pre.requirements.txt @@ -3,7 +3,7 @@ dbt-snowflake~=1.2.0b1 dbt-bigquery~=1.2.0b1 dbt-redshift~=1.2.0b1 dbt-postgres~=1.2.0b1 -dbt-spark[PyHive,ODBC]~=1.2.0b1 +dbt-spark[PyHive,ODbC]~=1.2.0b1 dbt-databricks==1.2.3 dbt-rpc~=0.1.1 grpcio-status~=1.47.0 diff --git a/release_creation/requirements/v1.3.latest.requirements.txt b/release_creation/snapshot/requirements/v1.3.latest.requirements.txt similarity index 100% rename from release_creation/requirements/v1.3.latest.requirements.txt rename to release_creation/snapshot/requirements/v1.3.latest.requirements.txt diff --git a/release_creation/snapshots.py b/release_creation/snapshots.py deleted file mode 100644 index 7552471d..00000000 --- a/release_creation/snapshots.py +++ /dev/null @@ -1,36 +0,0 @@ -from sqlite3 import adapters -import subprocess -import os -import shutil -from typing import Optional -from semantic_version import Version - -_BASE_DOWNLOAD_CMD = """pip3 download --dest ./tmp """ -_OUTPUT_ARCHIVE_FILE_BASE = "dbt-core-all-adapters-snapshot" -_FILE_DIR = os.path.dirname(os.path.realpath(__file__)) - -# def _generate_package_req(version: dict): -# suffix = version['suffix'] if 'suffix' in version else '' -# return f"{version['name']}{version['version_pinning']}{version['sem_ver']} {suffix} \\\n" - -def _generate_download_command(major_version:Optional[int], minor_version:Optional[int], is_pre:bool = False) -> str: - download_command = _BASE_DOWNLOAD_CMD - suffix = "latest" - if is_pre: - download_command += "--pre" - suffix = "pre" - download_command += f" -r {_FILE_DIR}/requirements/v{major_version}.{minor_version}.{suffix}.requirements.txt" - return download_command - -def generate_snapshot(target_version: Version) -> str: - is_pre = True if target_version.prerelease else False - download_cmd = _generate_download_command(major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre) - print(download_cmd) - subprocess.run("python -m pip install --upgrade pip", shell=True, check=True) - subprocess.run("rm -r tmp", shell=True, check=True) - subprocess.run("mkdir tmp", shell=True, check=True) - subprocess.run(download_cmd, shell=True, check=True) - subprocess.run(['sh',f"./release_creation/install.sh"], check=True) - archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{str(target_version)}" - shutil.make_archive(archive_path, 'zip', "tmp") - return archive_path + ".zip" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..bd6dc358 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +semantic-version +PyGithub \ No newline at end of file From 2f4b74f48f181299550ee547c1a9c780d28ab808 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 21 Nov 2022 09:24:16 -0800 Subject: [PATCH 03/18] formatting --- release_creation/github_client/releases.py | 3 +++ release_creation/main.py | 2 ++ release_creation/snapshot/create.py | 2 ++ 3 files changed, 7 insertions(+) diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index 851338fa..c774ee6a 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -30,16 +30,19 @@ def get_latest_release(input_version:str) -> Tuple[Version, Optional[GitRelease] latest_release = r return latest, latest_release + def _get_local_snapshot_reqs(snapshot_req_path:str) -> List[str]: with open(snapshot_req_path) as f: reqs = f.read() return reqs.split() + def _get_gh_release_asset(release_asset: GitReleaseAsset) -> List[str]: resp = requests.get(release_asset.browser_download_url) resp.raise_for_status() return resp.content.decode("utf-8").split() + def _compare_reqs(snapshot_req: List[str], release_req: List[str]) -> Tuple[Set[str], Set[str]]: snapshot_req_set = set(snapshot_req) release_req_set = set(release_req) diff --git a/release_creation/main.py b/release_creation/main.py index 8ac7f7e2..ccbbb43d 100644 --- a/release_creation/main.py +++ b/release_creation/main.py @@ -5,6 +5,7 @@ BASE_DIR = os.path.dirname(os.path.realpath(__file__)) + def main(): version = os.environ['INPUT_VERSION'] #e.g. v1.3.4 latest_version, latest_release = get_latest_release(version) @@ -12,5 +13,6 @@ def main(): snapshot_assets = generate_snapshot(target_version) create_new_release_for_version(target_version, snapshot_assets, latest_release) + if __name__ == "__main__": main() \ No newline at end of file diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py index 415e5af3..f0eb68d7 100644 --- a/release_creation/snapshot/create.py +++ b/release_creation/snapshot/create.py @@ -9,6 +9,7 @@ _FILE_DIR = os.path.dirname(os.path.realpath(__file__)) _SNAPSHOT_FILE = f"{_FILE_DIR}/snapshot.requirements.txt" + def _generate_download_command_args(major_version:Optional[int], minor_version:Optional[int], is_pre:bool = False) -> Tuple[List, str]: download_command = [] suffix = "latest" @@ -19,6 +20,7 @@ def _generate_download_command_args(major_version:Optional[int], minor_version:O download_command.append(f"-r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt") return download_command, requirements_prefix + def generate_snapshot(target_version: Version) -> Dict[str, str]: is_pre = True if target_version.prerelease else False download_cmd, requirements_prefix = _generate_download_command_args(major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre) From 9a44720fe4b1c5b81177846e11d34b8c8e599757 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 21 Nov 2022 15:02:46 -0800 Subject: [PATCH 04/18] add exception on no snapshot diff --- .github/workflows/release_snapshot.yml | 37 ++++++++++++++++++++++ release_creation/github_client/releases.py | 7 +++- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release_snapshot.yml diff --git a/.github/workflows/release_snapshot.yml b/.github/workflows/release_snapshot.yml new file mode 100644 index 00000000..167d84c8 --- /dev/null +++ b/.github/workflows/release_snapshot.yml @@ -0,0 +1,37 @@ +# **what?** +# This workflow will.... + +# **why?** +# Because I said so + +# **when?** +# This is triggered manually + +# **how** +# - poorly + +name: Release a Snapshot + +permissions: + packages: write + +on: + workflow_dispatch: + inputs: + version_number: + description: The release version number (i.e. 1.0.0b1). + required: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + - env: + INPUT_VERSION: ${{ inputs.version_number }} + - run: python release_creation/main.py + + \ No newline at end of file diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index c774ee6a..ccf7b841 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -52,6 +52,10 @@ def _compare_reqs(snapshot_req: List[str], release_req: List[str]) -> Tuple[Set[ def _diff_snapshot_requirements(snapshot_req_path:str, latest_release: Optional[GitRelease]): + # Scenarios being handled: + # 1. No change - raise exception + # 2. No prior patch version - Creat major.minor.0 snapshot + # 3. New changes - generate diff diff_result = "" if latest_release: release_reqs = [_asset for _asset in latest_release.get_assets() if _SNAP_REQ_NAME in _asset.name] @@ -62,7 +66,8 @@ def _diff_snapshot_requirements(snapshot_req_path:str, latest_release: Optional[ diff_result += "Added:\n* "+ "\n* ".join(added) + "\n___\n" if removed: diff_result += "\nRemoved:\n* " + "\n* ".join(removed) + "\n" - + if diff_result == "": + raise Exception("New snapshot does not contain any new changes") return diff_result else: return "No prior snapshot" From b444f4773727bb9c395dd166b9b1c4523a2e9759 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 21 Nov 2022 15:03:56 -0800 Subject: [PATCH 05/18] remove GHA --- .github/workflows/release_snapshot.yml | 37 -------------------------- 1 file changed, 37 deletions(-) delete mode 100644 .github/workflows/release_snapshot.yml diff --git a/.github/workflows/release_snapshot.yml b/.github/workflows/release_snapshot.yml deleted file mode 100644 index 167d84c8..00000000 --- a/.github/workflows/release_snapshot.yml +++ /dev/null @@ -1,37 +0,0 @@ -# **what?** -# This workflow will.... - -# **why?** -# Because I said so - -# **when?** -# This is triggered manually - -# **how** -# - poorly - -name: Release a Snapshot - -permissions: - packages: write - -on: - workflow_dispatch: - inputs: - version_number: - description: The release version number (i.e. 1.0.0b1). - required: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - env: - INPUT_VERSION: ${{ inputs.version_number }} - - run: python release_creation/main.py - - \ No newline at end of file From b7f50be4a0b3ce4b6500159d81a3e56f9c33525e Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 30 Nov 2022 14:40:10 -0800 Subject: [PATCH 06/18] make snapshot generation OS aware --- release_creation/github_client/releases.py | 68 ++++++++++++++++------ release_creation/main.py | 36 +++++++++--- release_creation/snapshot/create.py | 51 ++++++++++++---- release_creation/snapshot/download.sh | 9 +-- release_creation/snapshot/install.sh | 2 +- 5 files changed, 124 insertions(+), 42 deletions(-) diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index ccf7b841..31fea0b4 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -1,8 +1,10 @@ +import copy import os import requests from typing import Dict, List, Optional, Set, Tuple from semantic_version import Version, match from github import Github +from github.GithubException import GithubException from github.GitRelease import GitRelease from github.GitReleaseAsset import GitReleaseAsset @@ -15,23 +17,29 @@ def get_github_client() -> Github: return Github(_GH_ACCESS_TOKEN) -def get_latest_release(input_version:str) -> Tuple[Version, Optional[GitRelease]]: +def get_latest_snapshot_release(input_version: str) -> Tuple[Version, Optional[GitRelease]]: gh = get_github_client() - target_version = Version(input_version) + target_version = Version.coerce(input_version) target_match = f"~{target_version.major}.{target_version.minor}" - latest = Version(major=target_version.major, minor=target_version.minor, patch=0) + latest = copy.copy(target_version) + latest.patch = 0 repo = gh.get_repo(_GH_SNAPSHOT_REPO) releases = repo.get_releases() latest_release = None for r in releases: release_version = Version.coerce(r.tag_name) - if match(target_match, r.tag_name) and release_version > latest: + if ( + match(target_match, r.tag_name) + and target_version.build == release_version.build + and target_version.prerelease == release_version.prerelease + and release_version > latest + ): latest = release_version latest_release = r return latest, latest_release -def _get_local_snapshot_reqs(snapshot_req_path:str) -> List[str]: +def _get_local_snapshot_reqs(snapshot_req_path: str) -> List[str]: with open(snapshot_req_path) as f: reqs = f.read() return reqs.split() @@ -47,41 +55,65 @@ def _compare_reqs(snapshot_req: List[str], release_req: List[str]) -> Tuple[Set[ snapshot_req_set = set(snapshot_req) release_req_set = set(release_req) added_req = snapshot_req_set - release_req_set - removed_req = release_req_set - snapshot_req_set + removed_req = release_req_set - snapshot_req_set return added_req, removed_req -def _diff_snapshot_requirements(snapshot_req_path:str, latest_release: Optional[GitRelease]): +def _diff_snapshot_requirements( + snapshot_req_path: str, latest_release: Optional[GitRelease] +) -> str: # Scenarios being handled: # 1. No change - raise exception # 2. No prior patch version - Creat major.minor.0 snapshot # 3. New changes - generate diff - diff_result = "" if latest_release: - release_reqs = [_asset for _asset in latest_release.get_assets() if _SNAP_REQ_NAME in _asset.name] + diff_result = "" + release_reqs = [ + _asset for _asset in latest_release.get_assets() if _SNAP_REQ_NAME in _asset.name + ] snapshot_req = _get_local_snapshot_reqs(snapshot_req_path=snapshot_req_path) release_req = _get_gh_release_asset(release_reqs[0]) added, removed = _compare_reqs(snapshot_req=snapshot_req, release_req=release_req) if added: - diff_result += "Added:\n* "+ "\n* ".join(added) + "\n___\n" + diff_result += "Added:\n* " + "\n* ".join(added) + "\n___\n" if removed: diff_result += "\nRemoved:\n* " + "\n* ".join(removed) + "\n" - if diff_result == "": - raise Exception("New snapshot does not contain any new changes") return diff_result - else: + else: return "No prior snapshot" - -def create_new_release_for_version(release_version:Version, assets:Dict, latest_release: Optional[GitRelease], )->None: + +def create_new_release_for_version( + release_version: Version, assets: Dict, latest_release: Optional[GitRelease] +) -> None: gh = get_github_client() release_tag = str(release_version) repo = gh.get_repo(_GH_SNAPSHOT_REPO) - release_body = _diff_snapshot_requirements(assets[_SNAP_REQ_NAME], latest_release=latest_release) - created_release = repo.create_git_release(tag=release_tag, name='Snapshot Release', message=release_body) + release_body = _diff_snapshot_requirements( + assets[_SNAP_REQ_NAME], latest_release=latest_release + ) + if not release_body: + raise Exception("New snapshot does not contain any new changes") + created_release = repo.create_git_release( + tag=release_tag, name="Snapshot Release", message=release_body + ) try: for asset_name, asset_path in assets.items(): created_release.upload_asset(path=asset_path, name=asset_name) except Exception as e: created_release.delete_release() - raise e \ No newline at end of file + raise e + + +def add_assets_to_release(assets: Dict, latest_release: Optional[GitRelease]) -> None: + if not latest_release: + raise ValueError("Cannot update that which doth not exist!") + for asset_name, asset_path in assets.items(): + try: + latest_release.upload_asset(path=asset_path, name=asset_name) + except GithubException as e: + breakpoint() + if e.status == 422: + print("Asset already exists!") + else: + raise e diff --git a/release_creation/main.py b/release_creation/main.py index ccbbb43d..a0aa397b 100644 --- a/release_creation/main.py +++ b/release_creation/main.py @@ -1,18 +1,40 @@ +from enum import Enum import os +import argparse -from github_client.releases import create_new_release_for_version, get_latest_release +from github_client.releases import ( + create_new_release_for_version, + get_latest_snapshot_release, + add_assets_to_release, +) from snapshot.create import generate_snapshot BASE_DIR = os.path.dirname(os.path.realpath(__file__)) +class ReleaseOperations(str, Enum): + create = "create" + update = "update" + + def main(): - version = os.environ['INPUT_VERSION'] #e.g. v1.3.4 - latest_version, latest_release = get_latest_release(version) - target_version = latest_version.next_patch() - snapshot_assets = generate_snapshot(target_version) - create_new_release_for_version(target_version, snapshot_assets, latest_release) + parser = argparse.ArgumentParser() + parser.add_argument("--operation", required=True, type=ReleaseOperations) + parser.add_argument("--input-version", required=True, type=str) # e.g. v1.3.4 + args = parser.parse_args() + version = args.input_version + operation = args.operation + latest_version, latest_release = get_latest_snapshot_release(version) + if operation == ReleaseOperations.create: + target_version = latest_version.next_patch() + target_version.prerelease = latest_version.prerelease + target_version.build = latest_version.build + snapshot_assets = generate_snapshot(target_version) + create_new_release_for_version(target_version, snapshot_assets, latest_release) + elif operation == ReleaseOperations.update: + snapshot_assets = generate_snapshot(latest_version) + add_assets_to_release(assets=snapshot_assets, latest_release=latest_release) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py index f0eb68d7..586dd2bd 100644 --- a/release_creation/snapshot/create.py +++ b/release_creation/snapshot/create.py @@ -1,6 +1,7 @@ from typing import Dict, List, Optional, Tuple from semantic_version import Version import os +import platform import subprocess import shutil @@ -10,27 +11,53 @@ _SNAPSHOT_FILE = f"{_FILE_DIR}/snapshot.requirements.txt" -def _generate_download_command_args(major_version:Optional[int], minor_version:Optional[int], is_pre:bool = False) -> Tuple[List, str]: - download_command = [] +def _get_local_platform() -> str: + local_sys = platform.system() + if local_sys == "Linux": + return "linux" + elif local_sys == "Windows": + return "windows" + elif local_sys == "Darwin": + return "mac" + else: + raise ValueError(f"Unsupported system {local_sys}") + + +def _get_requirements_prefix( + major_version: Optional[int], minor_version: Optional[int], is_pre: bool = False +): suffix = "latest" if is_pre: - download_command.append("--pre") suffix = "pre" - requirements_prefix = f"v{major_version}.{minor_version}.{suffix}" - download_command.append(f"-r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt") - return download_command, requirements_prefix + return f"v{major_version}.{minor_version}.{suffix}" + + +def _generate_download_command_args( + requirements_prefix: str, is_pre: bool = False +) -> str: + download_command = "" + if is_pre: + download_command += " --pre" + download_command += f" -r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt" + return download_command def generate_snapshot(target_version: Version) -> Dict[str, str]: is_pre = True if target_version.prerelease else False - download_cmd, requirements_prefix = _generate_download_command_args(major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre) - subprocess.run(['sh',f"{_FILE_DIR}/download.sh", *download_cmd], check=True) - subprocess.run(['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix], check=True) + requirements_prefix = _get_requirements_prefix( + major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre + ) + archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{target_version}" + _os = _get_local_platform() + os_archive_path = f"{archive_path}-{_os}" + tmp_path = f"tmp/{_os}" + download_cmd = _generate_download_command_args(requirements_prefix=requirements_prefix, is_pre=is_pre) + subprocess.run(['sh',f"{_FILE_DIR}/download.sh", tmp_path, download_cmd], check=True) + subprocess.run(['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix, tmp_path], check=True) subprocess.run(['sh',f"{_FILE_DIR}/freeze.sh", _SNAPSHOT_FILE], check=True) - archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{str(target_version)}" - shutil.make_archive(archive_path, 'zip', 'tmp') + shutil.make_archive(os_archive_path, 'zip', 'tmp') assets = { - "snapshot_core_all_adapters": archive_path + ".zip", + f"snapshot_core_all_adapters_{_os}.zip": os_archive_path + ".zip", "snapshot_requirements": _SNAPSHOT_FILE } return assets \ No newline at end of file diff --git a/release_creation/snapshot/download.sh b/release_creation/snapshot/download.sh index 0cc2aa99..be7b2fee 100755 --- a/release_creation/snapshot/download.sh +++ b/release_creation/snapshot/download.sh @@ -1,5 +1,6 @@ python -m pip install --upgrade pip -rm -rf tmp -mkdir tmp -pip3 download --dest ./tmp \ - $1 \ No newline at end of file +rm -rf $1 +mkdir $1 +python -m pip download \ + --dest $1 \ + $2 \ No newline at end of file diff --git a/release_creation/snapshot/install.sh b/release_creation/snapshot/install.sh index 49f4fc71..17489ae8 100755 --- a/release_creation/snapshot/install.sh +++ b/release_creation/snapshot/install.sh @@ -3,5 +3,5 @@ mkdir target pip install -r $1/requirements/$2.requirements.txt \ --no-index \ --force-reinstall \ ---find-links ./tmp/ \ +--find-links ./$3 \ --target ./target \ No newline at end of file From a7453f8e96dbd552607079f76b366236559237a2 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 30 Nov 2022 16:35:15 -0800 Subject: [PATCH 07/18] fix conditional logic --- release_creation/github_client/releases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index 31fea0b4..31dcee77 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -32,7 +32,7 @@ def get_latest_snapshot_release(input_version: str) -> Tuple[Version, Optional[G match(target_match, r.tag_name) and target_version.build == release_version.build and target_version.prerelease == release_version.prerelease - and release_version > latest + and release_version >= latest ): latest = release_version latest_release = r From 319979271af5b07d31a18e6b7ed8df7456a6f9cd Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 30 Nov 2022 18:22:05 -0800 Subject: [PATCH 08/18] add prefer-binary --- release_creation/github_client/releases.py | 1 - release_creation/snapshot/create.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index 31dcee77..dbfa2fde 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -112,7 +112,6 @@ def add_assets_to_release(assets: Dict, latest_release: Optional[GitRelease]) -> try: latest_release.upload_asset(path=asset_path, name=asset_name) except GithubException as e: - breakpoint() if e.status == 422: print("Asset already exists!") else: diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py index 586dd2bd..e5c46919 100644 --- a/release_creation/snapshot/create.py +++ b/release_creation/snapshot/create.py @@ -38,6 +38,7 @@ def _generate_download_command_args( download_command = "" if is_pre: download_command += " --pre" + download_command += " --prefer-binary" download_command += f" -r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt" return download_command From 8dc4974499ad81e95d2f3f3ca0fa86952091c937 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 5 Dec 2022 17:13:20 -0800 Subject: [PATCH 09/18] fix comparison logic --- release_creation/github_client/releases.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index dbfa2fde..f0b8a993 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -2,7 +2,7 @@ import os import requests from typing import Dict, List, Optional, Set, Tuple -from semantic_version import Version, match +from semantic_version import Version from github import Github from github.GithubException import GithubException from github.GitRelease import GitRelease @@ -17,10 +17,9 @@ def get_github_client() -> Github: return Github(_GH_ACCESS_TOKEN) -def get_latest_snapshot_release(input_version: str) -> Tuple[Version, Optional[GitRelease]]: +def get_latest_snapshot_release(input_version: str) -> Tuple[ Version, Optional[GitRelease]]: gh = get_github_client() target_version = Version.coerce(input_version) - target_match = f"~{target_version.major}.{target_version.minor}" latest = copy.copy(target_version) latest.patch = 0 repo = gh.get_repo(_GH_SNAPSHOT_REPO) @@ -29,10 +28,11 @@ def get_latest_snapshot_release(input_version: str) -> Tuple[Version, Optional[G for r in releases: release_version = Version.coerce(r.tag_name) if ( - match(target_match, r.tag_name) - and target_version.build == release_version.build - and target_version.prerelease == release_version.prerelease - and release_version >= latest + release_version.major == latest.major + and release_version.minor == latest.minor + and release_version.prerelease == latest.prerelease + and release_version.build == latest.build + and release_version.patch >= latest.patch # type: ignore ): latest = release_version latest_release = r From 6f46dce896480ca15ded83e06c791e140dc421f2 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 6 Dec 2022 10:11:36 -0800 Subject: [PATCH 10/18] break out snapshots by py versions --- release_creation/github_client/releases.py | 2 +- release_creation/snapshot/create.py | 29 ++++++++++++------- release_creation/snapshot/download.sh | 4 +++ release_creation/snapshot/freeze.sh | 2 +- release_creation/snapshot/install.sh | 2 +- .../requirements/v1.0.latest.requirements.txt | 10 +++++++ 6 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 release_creation/snapshot/requirements/v1.0.latest.requirements.txt diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index f0b8a993..796efeb0 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -36,7 +36,7 @@ def get_latest_snapshot_release(input_version: str) -> Tuple[ Version, Optional[ ): latest = release_version latest_release = r - return latest, latest_release + return latest, latest_release def _get_local_snapshot_reqs(snapshot_req_path: str) -> List[str]: diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py index e5c46919..e4fff18a 100644 --- a/release_creation/snapshot/create.py +++ b/release_creation/snapshot/create.py @@ -9,7 +9,7 @@ _OUTPUT_ARCHIVE_FILE_BASE = "dbt-core-all-adapters-snapshot" _FILE_DIR = os.path.dirname(os.path.realpath(__file__)) _SNAPSHOT_FILE = f"{_FILE_DIR}/snapshot.requirements.txt" - +_PYTHON_VERSIONS = ['3.8.11','3.9.1','3.10.0'] def _get_local_platform() -> str: local_sys = platform.system() @@ -51,14 +51,21 @@ def generate_snapshot(target_version: Version) -> Dict[str, str]: archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{target_version}" _os = _get_local_platform() os_archive_path = f"{archive_path}-{_os}" - tmp_path = f"tmp/{_os}" - download_cmd = _generate_download_command_args(requirements_prefix=requirements_prefix, is_pre=is_pre) - subprocess.run(['sh',f"{_FILE_DIR}/download.sh", tmp_path, download_cmd], check=True) - subprocess.run(['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix, tmp_path], check=True) - subprocess.run(['sh',f"{_FILE_DIR}/freeze.sh", _SNAPSHOT_FILE], check=True) - shutil.make_archive(os_archive_path, 'zip', 'tmp') - assets = { - f"snapshot_core_all_adapters_{_os}.zip": os_archive_path + ".zip", - "snapshot_requirements": _SNAPSHOT_FILE - } + base_tmp_path = f"tmp/{_os}/" + assets = {} + for py_version in _PYTHON_VERSIONS: + py_major_minor = ".".join(py_version.split(".")[:-1]) + py_version_tmp_path = f"{base_tmp_path}{py_major_minor}" + py_version_archive_path = os_archive_path + f"-{py_major_minor}" + download_cmd = _generate_download_command_args(requirements_prefix=requirements_prefix, is_pre=is_pre) + subprocess.run( + ['sh',f"{_FILE_DIR}/download.sh", py_version_tmp_path, download_cmd, py_version], + check=True) + subprocess.run( + ['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix, py_version_tmp_path, py_version], + check=True) + shutil.make_archive(py_version_archive_path, 'zip', py_version_tmp_path) + assets[f"snapshot_core_all_adapters_{_os}_{py_major_minor}.zip"] = py_version_archive_path + ".zip" + subprocess.run(['sh',f"{_FILE_DIR}/freeze.sh", _SNAPSHOT_FILE, py_version], check=True) + assets["snapshot_requirements"] = _SNAPSHOT_FILE return assets \ No newline at end of file diff --git a/release_creation/snapshot/download.sh b/release_creation/snapshot/download.sh index be7b2fee..23c54ed0 100755 --- a/release_creation/snapshot/download.sh +++ b/release_creation/snapshot/download.sh @@ -1,3 +1,7 @@ +set -e +pyenv install $3 --skip-existing +pyenv virtualenv $3 snapshot-env -f +pyenv local snapshot-env python -m pip install --upgrade pip rm -rf $1 mkdir $1 diff --git a/release_creation/snapshot/freeze.sh b/release_creation/snapshot/freeze.sh index 4a9df7cf..c177a123 100755 --- a/release_creation/snapshot/freeze.sh +++ b/release_creation/snapshot/freeze.sh @@ -1 +1 @@ -pip freeze --path ./target > $1 \ No newline at end of file +pip freeze --path ./target/$2 > $1 \ No newline at end of file diff --git a/release_creation/snapshot/install.sh b/release_creation/snapshot/install.sh index 17489ae8..3c8da60a 100755 --- a/release_creation/snapshot/install.sh +++ b/release_creation/snapshot/install.sh @@ -4,4 +4,4 @@ pip install -r $1/requirements/$2.requirements.txt \ --no-index \ --force-reinstall \ --find-links ./$3 \ ---target ./target \ No newline at end of file +--target ./target/$4 \ No newline at end of file diff --git a/release_creation/snapshot/requirements/v1.0.latest.requirements.txt b/release_creation/snapshot/requirements/v1.0.latest.requirements.txt new file mode 100644 index 00000000..f004a413 --- /dev/null +++ b/release_creation/snapshot/requirements/v1.0.latest.requirements.txt @@ -0,0 +1,10 @@ +dbt-core~=1.0.0 --no-binary dbt-postgres +dbt-snowflake~=1.0.0 +dbt-bigquery~=1.0.0 +dbt-redshift~=1.0.0 +dbt-postgres~=1.0.0 +dbt-spark[PyHive,ODBC]~=1.0.0 +dbt-databricks==1.0.0 +dbt-rpc~=0.1.1 +grpcio-status~=1.47.0 +pyasn1-modules~=0.2.1 \ No newline at end of file From 11179123b04b23cb2f75125a94a6f41d3a67ef95 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 6 Dec 2022 11:32:17 -0800 Subject: [PATCH 11/18] use venv instead pyenv-virtualenv --- .gitignore | 4 +++- release_creation/snapshot/download.sh | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index e25d4b97..023a315d 100644 --- a/.gitignore +++ b/.gitignore @@ -97,4 +97,6 @@ dmypy.json # Project Specific tmp/ dbt-core*.zip -snapshot*.txt \ No newline at end of file +snapshot*.txt +.snapshot-env* +.DS_Store \ No newline at end of file diff --git a/release_creation/snapshot/download.sh b/release_creation/snapshot/download.sh index 23c54ed0..1e1b36e9 100755 --- a/release_creation/snapshot/download.sh +++ b/release_creation/snapshot/download.sh @@ -1,10 +1,13 @@ set -e pyenv install $3 --skip-existing -pyenv virtualenv $3 snapshot-env -f -pyenv local snapshot-env +install_env=".snapshot-env-$3" +pyenv local $3 +python -mvenv $install_env +source $install_env/bin/activate python -m pip install --upgrade pip rm -rf $1 mkdir $1 python -m pip download \ --dest $1 \ - $2 \ No newline at end of file + $2 +source deactivate \ No newline at end of file From 3da412b8edd07819d19a4d004e62985c839c409e Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 8 Dec 2022 17:10:33 -0800 Subject: [PATCH 12/18] remove OS specific code --- release_creation/github_client/releases.py | 5 +- release_creation/main.py | 2 + release_creation/snapshot/create.py | 52 +++++++++++-------- release_creation/snapshot/download.sh | 21 ++++---- release_creation/snapshot/download_no_deps.sh | 12 +++++ release_creation/snapshot/install.sh | 4 +- .../requirements/v1.0.latest.requirements.txt | 7 +-- .../requirements/v1.2.latest.requirements.txt | 3 +- .../requirements/v1.2.pre.requirements.txt | 3 +- .../requirements/v1.3.latest.requirements.txt | 3 +- 10 files changed, 72 insertions(+), 40 deletions(-) create mode 100755 release_creation/snapshot/download_no_deps.sh diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index 796efeb0..1ddf4ca7 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -89,8 +89,11 @@ def create_new_release_for_version( gh = get_github_client() release_tag = str(release_version) repo = gh.get_repo(_GH_SNAPSHOT_REPO) + reqs_files = [x for x in assets if _SNAP_REQ_NAME in x] + print(assets) + print(reqs_files) release_body = _diff_snapshot_requirements( - assets[_SNAP_REQ_NAME], latest_release=latest_release + assets[reqs_files[0]], latest_release=latest_release ) if not release_body: raise Exception("New snapshot does not contain any new changes") diff --git a/release_creation/main.py b/release_creation/main.py index a0aa397b..79482fe5 100644 --- a/release_creation/main.py +++ b/release_creation/main.py @@ -30,6 +30,8 @@ def main(): target_version.prerelease = latest_version.prerelease target_version.build = latest_version.build snapshot_assets = generate_snapshot(target_version) + print(f"Attempting to create new release for target version: {target_version}") + print(snapshot_assets) create_new_release_for_version(target_version, snapshot_assets, latest_release) elif operation == ReleaseOperations.update: snapshot_assets = generate_snapshot(latest_version) diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py index e4fff18a..e1cec882 100644 --- a/release_creation/snapshot/create.py +++ b/release_creation/snapshot/create.py @@ -1,17 +1,15 @@ from typing import Dict, List, Optional, Tuple from semantic_version import Version -import os +import os, platform import platform import subprocess import shutil - _OUTPUT_ARCHIVE_FILE_BASE = "dbt-core-all-adapters-snapshot" _FILE_DIR = os.path.dirname(os.path.realpath(__file__)) -_SNAPSHOT_FILE = f"{_FILE_DIR}/snapshot.requirements.txt" -_PYTHON_VERSIONS = ['3.8.11','3.9.1','3.10.0'] -def _get_local_platform() -> str: + +def _get_local_os() -> str: local_sys = platform.system() if local_sys == "Linux": return "linux" @@ -22,6 +20,12 @@ def _get_local_platform() -> str: else: raise ValueError(f"Unsupported system {local_sys}") +def _get_extra_platforms_for_os(_os: str) -> List[str]: + if _os == "mac": + return ['macosx_10_9_x86_64', 'macosx_11_0_arm64'] + else: + return ['manylinux_2_17_x86_64','manylinux2014_x86_64'] + def _get_requirements_prefix( major_version: Optional[int], minor_version: Optional[int], is_pre: bool = False @@ -38,7 +42,6 @@ def _generate_download_command_args( download_command = "" if is_pre: download_command += " --pre" - download_command += " --prefer-binary" download_command += f" -r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt" return download_command @@ -49,23 +52,30 @@ def generate_snapshot(target_version: Version) -> Dict[str, str]: major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre ) archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{target_version}" - _os = _get_local_platform() + _os = _get_local_os() os_archive_path = f"{archive_path}-{_os}" base_tmp_path = f"tmp/{_os}/" assets = {} - for py_version in _PYTHON_VERSIONS: - py_major_minor = ".".join(py_version.split(".")[:-1]) - py_version_tmp_path = f"{base_tmp_path}{py_major_minor}" - py_version_archive_path = os_archive_path + f"-{py_major_minor}" - download_cmd = _generate_download_command_args(requirements_prefix=requirements_prefix, is_pre=is_pre) - subprocess.run( - ['sh',f"{_FILE_DIR}/download.sh", py_version_tmp_path, download_cmd, py_version], - check=True) + py_version = platform.python_version() + py_major_minor = ".".join(py_version.split(".")[:-1]) + requirements_file = f"{_FILE_DIR}/snapshot.requirements.{py_major_minor}.txt" + py_version_tmp_path = f"{base_tmp_path}{py_major_minor}" + py_version_archive_path = os_archive_path + f"-{py_major_minor}" + download_cmd = _generate_download_command_args(requirements_prefix=requirements_prefix, is_pre=is_pre) + subprocess.run( + ['sh',f"{_FILE_DIR}/download.sh", py_version_tmp_path, download_cmd, py_version], + check=True) + subprocess.run( + ['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix, py_version_tmp_path, py_version], + check=True) + shutil.make_archive(py_version_archive_path, 'zip', py_version_tmp_path) + assets[f"snapshot_core_all_adapters_{_os}_{py_major_minor}.zip"] = py_version_archive_path + ".zip" + subprocess.run(['sh',f"{_FILE_DIR}/freeze.sh", requirements_file, py_version], check=True) + assets[f"snapshot_requirements_{py_major_minor}.txt"] = requirements_file + extra_platforms = _get_extra_platforms_for_os(_os) + for extra_platform in extra_platforms: subprocess.run( - ['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix, py_version_tmp_path, py_version], - check=True) - shutil.make_archive(py_version_archive_path, 'zip', py_version_tmp_path) - assets[f"snapshot_core_all_adapters_{_os}_{py_major_minor}.zip"] = py_version_archive_path + ".zip" - subprocess.run(['sh',f"{_FILE_DIR}/freeze.sh", _SNAPSHOT_FILE, py_version], check=True) - assets["snapshot_requirements"] = _SNAPSHOT_FILE + ['sh',f"{_FILE_DIR}/download_no_deps.sh", + py_version_tmp_path, extra_platform, requirements_file], + check=True) return assets \ No newline at end of file diff --git a/release_creation/snapshot/download.sh b/release_creation/snapshot/download.sh index 1e1b36e9..ec0c31ec 100755 --- a/release_creation/snapshot/download.sh +++ b/release_creation/snapshot/download.sh @@ -1,13 +1,12 @@ set -e -pyenv install $3 --skip-existing -install_env=".snapshot-env-$3" -pyenv local $3 -python -mvenv $install_env -source $install_env/bin/activate -python -m pip install --upgrade pip -rm -rf $1 -mkdir $1 +base_dir=$1 + +rm -rf $base_dir +mkdir -p $base_dir + python -m pip download \ - --dest $1 \ - $2 -source deactivate \ No newline at end of file + --progress-bar off \ + --prefer-binary \ + --dest $base_dir \ + --no-cache-dir \ + $2 \ No newline at end of file diff --git a/release_creation/snapshot/download_no_deps.sh b/release_creation/snapshot/download_no_deps.sh new file mode 100755 index 00000000..47d10fc7 --- /dev/null +++ b/release_creation/snapshot/download_no_deps.sh @@ -0,0 +1,12 @@ +final_dest=$1 +platform=$2 +requirements_file=$3 +staging="download-no-deps-staging" + +pip download -r $requirements_file \ + --dest $staging \ + --progress-bar off \ + --platform $platform \ + --no-deps + +cp -a $staging/. $final_dest/ \ No newline at end of file diff --git a/release_creation/snapshot/install.sh b/release_creation/snapshot/install.sh index 3c8da60a..cd61e69c 100755 --- a/release_creation/snapshot/install.sh +++ b/release_creation/snapshot/install.sh @@ -1,7 +1,9 @@ rm -rf target mkdir target + pip install -r $1/requirements/$2.requirements.txt \ --no-index \ --force-reinstall \ --find-links ./$3 \ ---target ./target/$4 \ No newline at end of file +--target ./target/$4 +# dbt --version \ No newline at end of file diff --git a/release_creation/snapshot/requirements/v1.0.latest.requirements.txt b/release_creation/snapshot/requirements/v1.0.latest.requirements.txt index f004a413..59d9444f 100644 --- a/release_creation/snapshot/requirements/v1.0.latest.requirements.txt +++ b/release_creation/snapshot/requirements/v1.0.latest.requirements.txt @@ -1,10 +1,11 @@ -dbt-core~=1.0.0 --no-binary dbt-postgres +dbt-core~=1.0.0 dbt-snowflake~=1.0.0 dbt-bigquery~=1.0.0 dbt-redshift~=1.0.0 dbt-postgres~=1.0.0 dbt-spark[PyHive,ODBC]~=1.0.0 -dbt-databricks==1.0.0 +dbt-databricks~=1.0.0 dbt-rpc~=0.1.1 grpcio-status~=1.47.0 -pyasn1-modules~=0.2.1 \ No newline at end of file +pyasn1-modules~=0.2.1 +pyodbc~=4.0.32 \ No newline at end of file diff --git a/release_creation/snapshot/requirements/v1.2.latest.requirements.txt b/release_creation/snapshot/requirements/v1.2.latest.requirements.txt index 6e6eeac4..e67aa71f 100644 --- a/release_creation/snapshot/requirements/v1.2.latest.requirements.txt +++ b/release_creation/snapshot/requirements/v1.2.latest.requirements.txt @@ -7,4 +7,5 @@ dbt-spark[PyHive,ODBC]~=1.2.0 dbt-databricks==1.2.3 dbt-rpc~=0.1.1 grpcio-status~=1.47.0 -pyasn1-modules~=0.2.1 \ No newline at end of file +pyasn1-modules~=0.2.1 +pyodbc==4.0.32 --no-binary pyodbc \ No newline at end of file diff --git a/release_creation/snapshot/requirements/v1.2.pre.requirements.txt b/release_creation/snapshot/requirements/v1.2.pre.requirements.txt index 4f2612a2..5a26ac97 100644 --- a/release_creation/snapshot/requirements/v1.2.pre.requirements.txt +++ b/release_creation/snapshot/requirements/v1.2.pre.requirements.txt @@ -7,4 +7,5 @@ dbt-spark[PyHive,ODbC]~=1.2.0b1 dbt-databricks==1.2.3 dbt-rpc~=0.1.1 grpcio-status~=1.47.0 -pyasn1-modules~=0.2.1 \ No newline at end of file +pyasn1-modules~=0.2.1 +pyodbc==4.0.32 --no-binary pyodbc \ No newline at end of file diff --git a/release_creation/snapshot/requirements/v1.3.latest.requirements.txt b/release_creation/snapshot/requirements/v1.3.latest.requirements.txt index 23dd9f71..1daa43ca 100644 --- a/release_creation/snapshot/requirements/v1.3.latest.requirements.txt +++ b/release_creation/snapshot/requirements/v1.3.latest.requirements.txt @@ -7,4 +7,5 @@ dbt-spark[PyHive,ODBC]~=1.3.0 dbt-databricks==1.3.0 dbt-rpc~=0.1.1 grpcio-status~=1.47.0 -pyasn1-modules~=0.2.1 \ No newline at end of file +pyasn1-modules~=0.2.1 +pyodbc==4.0.32 --no-binary pyodbc \ No newline at end of file From 4f5ed517abf02b4386ee09bd3504eaea0be76ae3 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 8 Dec 2022 17:10:53 -0800 Subject: [PATCH 13/18] remove dbt version for now --- release_creation/snapshot/install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/release_creation/snapshot/install.sh b/release_creation/snapshot/install.sh index cd61e69c..72a56cbb 100755 --- a/release_creation/snapshot/install.sh +++ b/release_creation/snapshot/install.sh @@ -6,4 +6,3 @@ pip install -r $1/requirements/$2.requirements.txt \ --force-reinstall \ --find-links ./$3 \ --target ./target/$4 -# dbt --version \ No newline at end of file From 7baa29f4a64f253824b3ed4a1b82ad8ea8da1040 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 13 Dec 2022 11:53:57 -0800 Subject: [PATCH 14/18] clean up and code clarity fixes --- release_creation/main.py | 4 ++-- release_creation/snapshot/create.py | 9 +++++---- release_creation/snapshot/download.sh | 3 ++- release_creation/snapshot/download_no_deps.sh | 1 + release_creation/snapshot/freeze.sh | 1 + release_creation/snapshot/install.sh | 1 + requirements.txt | 5 +++-- 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/release_creation/main.py b/release_creation/main.py index 79482fe5..7d0f2f34 100644 --- a/release_creation/main.py +++ b/release_creation/main.py @@ -1,4 +1,4 @@ -from enum import Enum +from strenum import StrEnum import os import argparse @@ -12,7 +12,7 @@ BASE_DIR = os.path.dirname(os.path.realpath(__file__)) -class ReleaseOperations(str, Enum): +class ReleaseOperations(StrEnum): create = "create" update = "update" diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py index e1cec882..a95d0c81 100644 --- a/release_creation/snapshot/create.py +++ b/release_creation/snapshot/create.py @@ -39,11 +39,12 @@ def _get_requirements_prefix( def _generate_download_command_args( requirements_prefix: str, is_pre: bool = False ) -> str: - download_command = "" + download_args = [] if is_pre: - download_command += " --pre" - download_command += f" -r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt" - return download_command + download_args.append(" --pre") + download_args.append( + f" -r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt") + return "".join(download_args) def generate_snapshot(target_version: Version) -> Dict[str, str]: diff --git a/release_creation/snapshot/download.sh b/release_creation/snapshot/download.sh index ec0c31ec..930c3a48 100755 --- a/release_creation/snapshot/download.sh +++ b/release_creation/snapshot/download.sh @@ -1,5 +1,6 @@ set -e base_dir=$1 +download_args=$2 rm -rf $base_dir mkdir -p $base_dir @@ -9,4 +10,4 @@ python -m pip download \ --prefer-binary \ --dest $base_dir \ --no-cache-dir \ - $2 \ No newline at end of file + $download_args \ No newline at end of file diff --git a/release_creation/snapshot/download_no_deps.sh b/release_creation/snapshot/download_no_deps.sh index 47d10fc7..23f5d04e 100755 --- a/release_creation/snapshot/download_no_deps.sh +++ b/release_creation/snapshot/download_no_deps.sh @@ -1,3 +1,4 @@ +set -e final_dest=$1 platform=$2 requirements_file=$3 diff --git a/release_creation/snapshot/freeze.sh b/release_creation/snapshot/freeze.sh index c177a123..ecdd68fe 100755 --- a/release_creation/snapshot/freeze.sh +++ b/release_creation/snapshot/freeze.sh @@ -1 +1,2 @@ +set -e pip freeze --path ./target/$2 > $1 \ No newline at end of file diff --git a/release_creation/snapshot/install.sh b/release_creation/snapshot/install.sh index 72a56cbb..25b7bdda 100755 --- a/release_creation/snapshot/install.sh +++ b/release_creation/snapshot/install.sh @@ -1,3 +1,4 @@ +set -e rm -rf target mkdir target diff --git a/requirements.txt b/requirements.txt index bd6dc358..e6e01f75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -semantic-version -PyGithub \ No newline at end of file +StrEnum +semantic-version~=2.10.0 +PyGithub~=1.0 \ No newline at end of file From aecf5655075dc443a518e8ef93ad45293b98fcc6 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 13 Dec 2022 12:24:43 -0800 Subject: [PATCH 15/18] use a less generic exception and add logging --- release_creation/github_client/releases.py | 2 +- release_creation/main.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index 1ddf4ca7..cb5f8d59 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -96,7 +96,7 @@ def create_new_release_for_version( assets[reqs_files[0]], latest_release=latest_release ) if not release_body: - raise Exception("New snapshot does not contain any new changes") + raise RuntimeError("New snapshot does not contain any new changes") created_release = repo.create_git_release( tag=release_tag, name="Snapshot Release", message=release_body ) diff --git a/release_creation/main.py b/release_creation/main.py index 7d0f2f34..6b66163b 100644 --- a/release_creation/main.py +++ b/release_creation/main.py @@ -1,3 +1,4 @@ +import logging from strenum import StrEnum import os import argparse @@ -10,7 +11,7 @@ from snapshot.create import generate_snapshot BASE_DIR = os.path.dirname(os.path.realpath(__file__)) - +logger = logging.getLogger(__name__) class ReleaseOperations(StrEnum): create = "create" @@ -30,8 +31,8 @@ def main(): target_version.prerelease = latest_version.prerelease target_version.build = latest_version.build snapshot_assets = generate_snapshot(target_version) - print(f"Attempting to create new release for target version: {target_version}") - print(snapshot_assets) + logger.info(f"Attempting to create new release for target version: {target_version}") + logger.info(f"release assets {[]}") create_new_release_for_version(target_version, snapshot_assets, latest_release) elif operation == ReleaseOperations.update: snapshot_assets = generate_snapshot(latest_version) From 2ef0bd5a24c5093cf7899b160e96d234968161fa Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 13 Dec 2022 13:03:30 -0800 Subject: [PATCH 16/18] add doc strings to primary functions --- release_creation/github_client/releases.py | 21 +++++++++++++++++++++ release_creation/snapshot/create.py | 11 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index cb5f8d59..296cdd4c 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -18,6 +18,15 @@ def get_github_client() -> Github: def get_latest_snapshot_release(input_version: str) -> Tuple[ Version, Optional[GitRelease]]: + """Retrieve the latest release matching the major.minor and release stage + semantic version if it exists. Ignores the patch version. + + Args: + input_version (str): semantic version (1.0.0.0rc, 2.3.5) to match against + + Returns: + Tuple[ Version, Optional[GitRelease]]: _description_ + """ gh = get_github_client() target_version = Version.coerce(input_version) latest = copy.copy(target_version) @@ -86,6 +95,18 @@ def _diff_snapshot_requirements( def create_new_release_for_version( release_version: Version, assets: Dict, latest_release: Optional[GitRelease] ) -> None: + """Given an input version it creates a matching Github Release and attaches the assets + as a ReleaseAsset + + Args: + release_version (Version): semantic version to be used when creating the release + assets (Dict): assets to be added to the created release where a key is the asset name + latest_release (Optional[GitRelease]): supply if there is a prior release to be diffed against + + Raises: + RuntimeError: _description_ + e: _description_ + """ gh = get_github_client() release_tag = str(release_version) repo = gh.get_repo(_GH_SNAPSHOT_REPO) diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py index a95d0c81..b76c4d58 100644 --- a/release_creation/snapshot/create.py +++ b/release_creation/snapshot/create.py @@ -48,6 +48,17 @@ def _generate_download_command_args( def generate_snapshot(target_version: Version) -> Dict[str, str]: + """creates a zip archive of the python dependencies for the provided + semantic version + + Args: + target_version (Version): the input version to use when determining + the requirements to download. + + Returns: + Dict[str, str]: dict of generated snapshot assets, key is it's name + and the value is the path to the file. + """ is_pre = True if target_version.prerelease else False requirements_prefix = _get_requirements_prefix( major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre From 8a9aca7aae41170932d047bcda83062ef06a81bf Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 15 Dec 2022 12:27:09 -0800 Subject: [PATCH 17/18] code cleanup and swap print for logging --- release_creation/github_client/releases.py | 28 ++++++++++++---------- release_creation/main.py | 10 +++++++- release_creation/snapshot/download.sh | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/release_creation/github_client/releases.py b/release_creation/github_client/releases.py index 296cdd4c..c7c56f3b 100644 --- a/release_creation/github_client/releases.py +++ b/release_creation/github_client/releases.py @@ -1,4 +1,5 @@ import copy +import logging import os import requests from typing import Dict, List, Optional, Set, Tuple @@ -12,6 +13,7 @@ _GH_ACCESS_TOKEN = os.environ["GH_ACCESS_TOKEN"] _SNAP_REQ_NAME = "snapshot_requirements" +logger = logging.getLogger(__name__) def get_github_client() -> Github: return Github(_GH_ACCESS_TOKEN) @@ -25,27 +27,28 @@ def get_latest_snapshot_release(input_version: str) -> Tuple[ Version, Optional[ input_version (str): semantic version (1.0.0.0rc, 2.3.5) to match against Returns: - Tuple[ Version, Optional[GitRelease]]: _description_ + Tuple[ Version, Optional[GitRelease]]: A tuple of the latest release tag + and the latest release itself. """ gh = get_github_client() target_version = Version.coerce(input_version) - latest = copy.copy(target_version) - latest.patch = 0 + latest_version = copy.copy(target_version) + latest_version.patch = 0 repo = gh.get_repo(_GH_SNAPSHOT_REPO) releases = repo.get_releases() latest_release = None for r in releases: release_version = Version.coerce(r.tag_name) if ( - release_version.major == latest.major - and release_version.minor == latest.minor - and release_version.prerelease == latest.prerelease - and release_version.build == latest.build - and release_version.patch >= latest.patch # type: ignore + release_version.major == latest_version.major + and release_version.minor == latest_version.minor + and release_version.prerelease == latest_version.prerelease + and release_version.build == latest_version.build + and release_version.patch >= latest_version.patch # type: ignore ): - latest = release_version + latest_version = release_version latest_release = r - return latest, latest_release + return latest_version, latest_release def _get_local_snapshot_reqs(snapshot_req_path: str) -> List[str]: @@ -111,8 +114,7 @@ def create_new_release_for_version( release_tag = str(release_version) repo = gh.get_repo(_GH_SNAPSHOT_REPO) reqs_files = [x for x in assets if _SNAP_REQ_NAME in x] - print(assets) - print(reqs_files) + release_body = _diff_snapshot_requirements( assets[reqs_files[0]], latest_release=latest_release ) @@ -137,6 +139,6 @@ def add_assets_to_release(assets: Dict, latest_release: Optional[GitRelease]) -> latest_release.upload_asset(path=asset_path, name=asset_name) except GithubException as e: if e.status == 422: - print("Asset already exists!") + logger.warning("Asset already exists!") else: raise e diff --git a/release_creation/main.py b/release_creation/main.py index 6b66163b..779ef33b 100644 --- a/release_creation/main.py +++ b/release_creation/main.py @@ -19,9 +19,17 @@ class ReleaseOperations(StrEnum): def main(): + """ + Implements two workflows: + * Create: Generate a net new release for a major.minor version which corresponds to core. + * Update: Add release assets to an existing release. + + Input version is a string that corresponds to the semver standard, + see https://semver.org/ for more info. + """ parser = argparse.ArgumentParser() parser.add_argument("--operation", required=True, type=ReleaseOperations) - parser.add_argument("--input-version", required=True, type=str) # e.g. v1.3.4 + parser.add_argument("--input-version", required=True, type=str) # e.g. 1.3.4 args = parser.parse_args() version = args.input_version operation = args.operation diff --git a/release_creation/snapshot/download.sh b/release_creation/snapshot/download.sh index 930c3a48..facc6fcd 100755 --- a/release_creation/snapshot/download.sh +++ b/release_creation/snapshot/download.sh @@ -10,4 +10,4 @@ python -m pip download \ --prefer-binary \ --dest $base_dir \ --no-cache-dir \ - $download_args \ No newline at end of file + $download_args From b1d4ce733c6458258d345ad050df51ba63690ef1 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 15 Dec 2022 17:52:53 -0800 Subject: [PATCH 18/18] Add comments and clarification --- release_creation/snapshot/create.py | 34 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/release_creation/snapshot/create.py b/release_creation/snapshot/create.py index b76c4d58..1580ff54 100644 --- a/release_creation/snapshot/create.py +++ b/release_creation/snapshot/create.py @@ -41,10 +41,10 @@ def _generate_download_command_args( ) -> str: download_args = [] if is_pre: - download_args.append(" --pre") + download_args.append("--pre") download_args.append( - f" -r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt") - return "".join(download_args) + f"-r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt") + return " ".join(download_args) def generate_snapshot(target_version: Version) -> Dict[str, str]: @@ -63,31 +63,43 @@ def generate_snapshot(target_version: Version) -> Dict[str, str]: requirements_prefix = _get_requirements_prefix( major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre ) + # Setup confiuration variables archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{target_version}" - _os = _get_local_os() - os_archive_path = f"{archive_path}-{_os}" - base_tmp_path = f"tmp/{_os}/" - assets = {} + local_os = _get_local_os() + os_archive_path = f"{archive_path}-{local_os}" + base_tmp_path = f"tmp/{local_os}/" py_version = platform.python_version() py_major_minor = ".".join(py_version.split(".")[:-1]) requirements_file = f"{_FILE_DIR}/snapshot.requirements.{py_major_minor}.txt" py_version_tmp_path = f"{base_tmp_path}{py_major_minor}" py_version_archive_path = os_archive_path + f"-{py_major_minor}" + download_cmd = _generate_download_command_args(requirements_prefix=requirements_prefix, is_pre=is_pre) + # Download pip dependencies subprocess.run( ['sh',f"{_FILE_DIR}/download.sh", py_version_tmp_path, download_cmd, py_version], check=True) + # Check install subprocess.run( ['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix, py_version_tmp_path, py_version], check=True) - shutil.make_archive(py_version_archive_path, 'zip', py_version_tmp_path) - assets[f"snapshot_core_all_adapters_{_os}_{py_major_minor}.zip"] = py_version_archive_path + ".zip" + # Freeze complete requirements (i.e. including transitive dependencies) subprocess.run(['sh',f"{_FILE_DIR}/freeze.sh", requirements_file, py_version], check=True) - assets[f"snapshot_requirements_{py_major_minor}.txt"] = requirements_file - extra_platforms = _get_extra_platforms_for_os(_os) + + # Use the complete requirements to do a no-deps download (doesn't check system compatibility) + # This allows us to download requirements for platform architectures other than the local + extra_platforms = _get_extra_platforms_for_os(local_os) for extra_platform in extra_platforms: subprocess.run( ['sh',f"{_FILE_DIR}/download_no_deps.sh", py_version_tmp_path, extra_platform, requirements_file], check=True) + + # Generate a Zip archive of required packages + shutil.make_archive(py_version_archive_path, 'zip', py_version_tmp_path) + + assets = {} + assets[f"snapshot_core_all_adapters_{local_os}_{py_major_minor}.zip"] = py_version_archive_path + ".zip" + assets[f"snapshot_requirements_{py_major_minor}.txt"] = requirements_file + return assets \ No newline at end of file