Merge pull request #1 from dbt-labs/addPyScripts

Generate Releases

.gitignore

@@ -51,36 +51,9 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
 # PyBuilder
 target/
 
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
 # pyenv
 .python-version
 
@@ -94,13 +67,6 @@ ipython_config.py
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
 # Environments
 .env
 .venv
@@ -127,3 +93,10 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Project Specific
+tmp/
+dbt-core*.zip
+snapshot*.txt
+.snapshot-env*
+.DS_Store

release_creation/github_client/releases.py

@@ -0,0 +1,144 @@
+import copy
+import logging
+import os
+import requests
+from typing import Dict, List, Optional, Set, Tuple
+from semantic_version import Version
+from github import Github
+from github.GithubException import GithubException
+from github.GitRelease import GitRelease
+from github.GitReleaseAsset import GitReleaseAsset
+
+_GH_SNAPSHOT_REPO = "dbt-labs/dbt-core-snapshots"
+_GH_ACCESS_TOKEN = os.environ["GH_ACCESS_TOKEN"]
+_SNAP_REQ_NAME = "snapshot_requirements"
+
+logger = logging.getLogger(__name__)
+
+def get_github_client() -> Github:
+    return Github(_GH_ACCESS_TOKEN)
+
+
+def get_latest_snapshot_release(input_version: str) -> Tuple[ Version, Optional[GitRelease]]:
+    """Retrieve the latest release matching the major.minor and release stage
+       semantic version if it exists. Ignores the patch version. 
+
+    Args:
+        input_version (str): semantic version (1.0.0.0rc, 2.3.5) to match against 
+
+    Returns:
+        Tuple[ Version, Optional[GitRelease]]: A tuple of the latest release tag 
+        and the latest release itself.
+    """
+    gh = get_github_client()
+    target_version = Version.coerce(input_version)
+    latest_version = copy.copy(target_version)
+    latest_version.patch = 0
+    repo = gh.get_repo(_GH_SNAPSHOT_REPO)
+    releases = repo.get_releases()
+    latest_release = None
+    for r in releases:
+        release_version = Version.coerce(r.tag_name)
+        if (
+            release_version.major == latest_version.major
+            and release_version.minor == latest_version.minor
+            and release_version.prerelease == latest_version.prerelease
+            and release_version.build == latest_version.build
+            and release_version.patch >= latest_version.patch  # type: ignore
+        ):
+            latest_version = release_version
+            latest_release = r
+    return latest_version, latest_release    
+
+
+def _get_local_snapshot_reqs(snapshot_req_path: str) -> List[str]:
+    with open(snapshot_req_path) as f:
+        reqs = f.read()
+    return reqs.split()
+
+
+def _get_gh_release_asset(release_asset: GitReleaseAsset) -> List[str]:
+    resp = requests.get(release_asset.browser_download_url)
+    resp.raise_for_status()
+    return resp.content.decode("utf-8").split()
+
+
+def _compare_reqs(snapshot_req: List[str], release_req: List[str]) -> Tuple[Set[str], Set[str]]:
+    snapshot_req_set = set(snapshot_req)
+    release_req_set = set(release_req)
+    added_req = snapshot_req_set - release_req_set
+    removed_req = release_req_set - snapshot_req_set
+    return added_req, removed_req
+
+
+def _diff_snapshot_requirements(
+    snapshot_req_path: str, latest_release: Optional[GitRelease]
+) -> str:
+    # Scenarios being handled:
+    # 1. No change - raise exception
+    # 2. No prior patch version - Creat major.minor.0 snapshot
+    # 3. New changes - generate diff
+    if latest_release:
+        diff_result = ""
+        release_reqs = [
+            _asset for _asset in latest_release.get_assets() if _SNAP_REQ_NAME in _asset.name
+        ]
+        snapshot_req = _get_local_snapshot_reqs(snapshot_req_path=snapshot_req_path)
+        release_req = _get_gh_release_asset(release_reqs[0])
+        added, removed = _compare_reqs(snapshot_req=snapshot_req, release_req=release_req)
+        if added:
+            diff_result += "Added:\n* " + "\n* ".join(added) + "\n___\n"
+        if removed:
+            diff_result += "\nRemoved:\n* " + "\n* ".join(removed) + "\n"
+        return diff_result
+    else:
+        return "No prior snapshot"
+
+
+def create_new_release_for_version(
+    release_version: Version, assets: Dict, latest_release: Optional[GitRelease]
+) -> None:
+    """Given an input version it creates a matching Github Release and attaches the assets
+       as a ReleaseAsset
+
+    Args:
+        release_version (Version): semantic version to be used when creating the release
+        assets (Dict): assets to be added to the created release where a key is the asset name
+        latest_release (Optional[GitRelease]): supply if there is a prior release to be diffed against
+
+    Raises:
+        RuntimeError: _description_
+        e: _description_
+    """
+    gh = get_github_client()
+    release_tag = str(release_version)
+    repo = gh.get_repo(_GH_SNAPSHOT_REPO)
+    reqs_files = [x for x in assets if _SNAP_REQ_NAME in x]
+
+    release_body = _diff_snapshot_requirements(
+        assets[reqs_files[0]], latest_release=latest_release
+    )
+    if not release_body:
+        raise RuntimeError("New snapshot does not contain any new changes")
+    created_release = repo.create_git_release(
+        tag=release_tag, name="Snapshot Release", message=release_body
+    )
+    try:
+        for asset_name, asset_path in assets.items():
+            created_release.upload_asset(path=asset_path, name=asset_name)
+    except Exception as e:
+        created_release.delete_release()
+        raise e
+
+
+def add_assets_to_release(assets: Dict, latest_release: Optional[GitRelease]) -> None:
+    if not latest_release:
+        raise ValueError("Cannot update that which doth not exist!")
+    for asset_name, asset_path in assets.items():
+        try:
+            latest_release.upload_asset(path=asset_path, name=asset_name)
+        except GithubException as e:
+            if e.status == 422:
+                logger.warning("Asset already exists!")
+            else:
+                raise e

release_creation/main.py

@@ -0,0 +1,51 @@
+import logging
+from strenum import StrEnum
+import os
+import argparse
+
+from github_client.releases import (
+    create_new_release_for_version,
+    get_latest_snapshot_release,
+    add_assets_to_release,
+)
+from snapshot.create import generate_snapshot
+
+BASE_DIR = os.path.dirname(os.path.realpath(__file__))
+logger = logging.getLogger(__name__)
+
+class ReleaseOperations(StrEnum):
+    create = "create"
+    update = "update"
+
+
+def main():
+    """
+    Implements two workflows:
+    * Create: Generate a net new release for a major.minor version which corresponds to core.
+    * Update: Add release assets to an existing release.
+
+    Input version is a string that corresponds to the semver standard, 
+    see https://semver.org/ for more info. 
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--operation", required=True, type=ReleaseOperations)
+    parser.add_argument("--input-version", required=True, type=str)  # e.g. 1.3.4
+    args = parser.parse_args()
+    version = args.input_version
+    operation = args.operation
+    latest_version, latest_release = get_latest_snapshot_release(version)
+    if operation == ReleaseOperations.create:
+        target_version = latest_version.next_patch()
+        target_version.prerelease = latest_version.prerelease
+        target_version.build = latest_version.build
+        snapshot_assets = generate_snapshot(target_version)
+        logger.info(f"Attempting to create new release for target version: {target_version}")
+        logger.info(f"release assets {[]}")
+        create_new_release_for_version(target_version, snapshot_assets, latest_release)
+    elif operation == ReleaseOperations.update:
+        snapshot_assets = generate_snapshot(latest_version)
+        add_assets_to_release(assets=snapshot_assets, latest_release=latest_release)
+
+
+if __name__ == "__main__":
+    main()

release_creation/snapshot/create.py

@@ -0,0 +1,105 @@
+from typing import Dict, List, Optional, Tuple
+from semantic_version import Version
+import os, platform
+import platform
+import subprocess
+import shutil
+
+_OUTPUT_ARCHIVE_FILE_BASE = "dbt-core-all-adapters-snapshot"
+_FILE_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def _get_local_os() -> str:
+    local_sys = platform.system()
+    if local_sys == "Linux":
+        return "linux"
+    elif local_sys == "Windows":
+        return "windows"
+    elif local_sys == "Darwin":
+         return "mac"
+    else:
+        raise ValueError(f"Unsupported system {local_sys}")
+
+def _get_extra_platforms_for_os(_os: str) -> List[str]:
+    if _os == "mac":
+        return ['macosx_10_9_x86_64', 'macosx_11_0_arm64']
+    else:
+        return ['manylinux_2_17_x86_64','manylinux2014_x86_64']
+
+
+def _get_requirements_prefix(
+    major_version: Optional[int], minor_version: Optional[int], is_pre: bool = False
+):
+    suffix = "latest"
+    if is_pre:
+        suffix = "pre"
+    return f"v{major_version}.{minor_version}.{suffix}"
+
+
+def _generate_download_command_args(
+    requirements_prefix: str, is_pre: bool = False
+) -> str:
+    download_args = []
+    if is_pre:
+        download_args.append("--pre")
+    download_args.append(
+        f"-r {_FILE_DIR}/requirements/{requirements_prefix}.requirements.txt")
+    return " ".join(download_args)
+
+
+def generate_snapshot(target_version: Version) -> Dict[str, str]:
+    """creates a zip archive of the python dependencies for the provided 
+       semantic version
+
+    Args:
+        target_version (Version): the input version to use when determining 
+        the requirements to download.
+
+    Returns:
+        Dict[str, str]: dict of generated snapshot assets, key is it's name 
+                        and the value is the path to the file.
+    """
+    is_pre = True if target_version.prerelease else False
+    requirements_prefix = _get_requirements_prefix(
+        major_version=target_version.major, minor_version=target_version.minor, is_pre=is_pre
+    )
+    # Setup confiuration variables
+    archive_path = f"{_OUTPUT_ARCHIVE_FILE_BASE}-{target_version}"
+    local_os = _get_local_os()
+    os_archive_path = f"{archive_path}-{local_os}"
+    base_tmp_path = f"tmp/{local_os}/"
+    py_version = platform.python_version()
+    py_major_minor = ".".join(py_version.split(".")[:-1])
+    requirements_file = f"{_FILE_DIR}/snapshot.requirements.{py_major_minor}.txt"
+    py_version_tmp_path = f"{base_tmp_path}{py_major_minor}"
+    py_version_archive_path = os_archive_path + f"-{py_major_minor}"
+
+    download_cmd = _generate_download_command_args(requirements_prefix=requirements_prefix, is_pre=is_pre)
+    # Download pip dependencies
+    subprocess.run(
+        ['sh',f"{_FILE_DIR}/download.sh", py_version_tmp_path, download_cmd, py_version], 
+        check=True)
+    # Check install
+    subprocess.run(
+        ['sh',f"{_FILE_DIR}/install.sh", _FILE_DIR, requirements_prefix, py_version_tmp_path, py_version], 
+        check=True)
+    # Freeze complete requirements (i.e. including transitive dependencies)
+    subprocess.run(['sh',f"{_FILE_DIR}/freeze.sh", requirements_file, py_version], check=True)
+
+    # Use the complete requirements to do a no-deps download (doesn't check system compatibility)
+    # This allows us to download requirements for platform architectures other than the local
+    extra_platforms = _get_extra_platforms_for_os(local_os)
+    for extra_platform in extra_platforms:
+        subprocess.run(
+            ['sh',f"{_FILE_DIR}/download_no_deps.sh", 
+            py_version_tmp_path, extra_platform, requirements_file],
+                check=True)
+
+    # Generate a Zip archive of required packages
+    shutil.make_archive(py_version_archive_path, 'zip', py_version_tmp_path)
+
+    assets = {}
+    assets[f"snapshot_core_all_adapters_{local_os}_{py_major_minor}.zip"] = py_version_archive_path + ".zip"
+    assets[f"snapshot_requirements_{py_major_minor}.txt"] = requirements_file
+
+    return assets

release_creation/snapshot/download.sh

@@ -0,0 +1,13 @@
+set -e
+base_dir=$1
+download_args=$2
+
+rm -rf $base_dir
+mkdir  -p $base_dir
+
+python -m pip download \
+ --progress-bar off \
+ --prefer-binary \
+ --dest $base_dir \
+ --no-cache-dir \
+ $download_args

release_creation/snapshot/download_no_deps.sh

@@ -0,0 +1,13 @@
+set -e
+final_dest=$1
+platform=$2
+requirements_file=$3
+staging="download-no-deps-staging"
+
+pip download -r $requirements_file \
+ --dest $staging \
+ --progress-bar off \
+ --platform $platform \
+ --no-deps
+
+cp -a $staging/. $final_dest/

release_creation/snapshot/freeze.sh

@@ -0,0 +1,2 @@
+set -e
+pip freeze --path ./target/$2 > $1