diff --git a/.github/workflows/test-check-binary.yml b/.github/workflows/test-check-binary.yml deleted file mode 100644 index e29ee7ba9..000000000 --- a/.github/workflows/test-check-binary.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Test check_binary - -on: - push: - branches: - - main - pull_request: - paths: - - .github/workflows/test-check-binary.yml - - check_binary.sh - - test/smoke_test/smoke_test.py - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true - -jobs: - check_binary_linux_cpu: - name: Test check_binary.sh for Linux CPU - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: "pytorch/builder" - docker-image: python:3.11 - script: | - pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - DESIRED_PYTHON=3.11 DESIRED_CUDA=cpu PACKAGE_TYPE=manywheel ./check_binary.sh - - check_binary_linux_cuda: - name: Test check_binary.sh for Linux CUDA - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: "pytorch/builder" - runner: linux.4xlarge.nvidia.gpu - docker-image: python:3.11 - script: | - pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu121 - DESIRED_PYTHON=3.11 DESIRED_CUDA=cu121 PACKAGE_TYPE=manywheel ./check_binary.sh diff --git a/.github/workflows/validate-repackaged-binary-sizes.yml b/.github/workflows/validate-repackaged-binary-sizes.yml deleted file mode 100644 index cb1a6a73e..000000000 --- a/.github/workflows/validate-repackaged-binary-sizes.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Validate manywheel binaries - -# This workflow validates the size of the manywheel binaries after repackaging for PyPi -# Specify the direct URLs to the binaries (from https://download.pytorch.org/whl/test/torch/) in the matrix -# along with the python version. -# -# The workflow will: -# * download the binaries, -# * run release/pypi/prep_binary_for_pypi.sh -# * run smoke tests on the repackaged binaries -# * display the size before and after repackaging as the workflow annotation -# * optionally upload the repackaged binaries as artifacts (for debug or promotion) - -on: - pull_request: - paths: - - .github/workflows/validate-repackaged-binary-sizes.yml - - release/pypi/prep_binary_for_pypi.sh - -jobs: - validate-binary-size: - strategy: - fail-fast: false - matrix: - whl: - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp312-cp312-linux_x86_64.whl - python: "3.12" - artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp311-cp311-linux_x86_64.whl - python: "3.11" # python version to use for smoke tests - upload_artifact: false # upload the repackaged binary as an artifact - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp310-cp310-linux_x86_64.whl - python: "3.10" - artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp39-cp39-linux_x86_64.whl - python: "3.9" - artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp38-cp38-linux_x86_64.whl - python: "3.8" - artifact: false - - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: linux.g5.4xlarge.nvidia.gpu - job-name: "Validate binary size" - upload-artifact: ${{ matrix.whl.upload_artifact == 'true' && 'repackaged-binary' || '' }} - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export MATRIX_GPU_ARCH_VERSION="12.1" - export MATRIX_GPU_ARCH_TYPE="cuda" - export MATRIX_CUDA_VER="12.1" - export DESIRED_PYTHON="${{ matrix.whl.python }}" - export MATRIX_PACKAGE_TYPE="wheel" - export TARGET_OS="linux" - - # install zip - sudo yum install zip -y - - # install patchelf - chmod a+x common/install_patchelf.sh - sudo common/install_patchelf.sh - - # download torch whl - wget ${{ matrix.whl.url }} - FILENAME=$(ls -1 *.whl | head -n 1) - SIZE_BEFORE=$(du -h $FILENAME | cut -f1) - - # repackage into manywheel - release/pypi/prep_binary_for_pypi.sh $FILENAME - - NEW_FILENAME=$(ls -1 *.whl | head -n 1) - echo "::notice:: $FILENAME before: $SIZE_BEFORE after: $(du -h $NEW_FILENAME | cut -f1)" - - # cp to ${RUNNER_ARTIFACT_DIR} - cp $NEW_FILENAME ${RUNNER_ARTIFACT_DIR}/ - - # create conda env - conda create -y -n $ENV_NAME python=$DESIRED_PYTHON - conda activate $ENV_NAME - - # install torch - pip install numpy pillow $NEW_FILENAME - - # run smoke test - python ./test/smoke_test/smoke_test.py --package=torchonly diff --git a/release/README.md b/release/README.md deleted file mode 100644 index a80144b17..000000000 --- a/release/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# PyTorch Release Scripts - -These are a collection of scripts that are to be used for release activities. - -> NOTE: All scripts should do no actual work unless the `DRY_RUN` environment variable is set -> to `disabled`. -> The basic idea being that there should be no potential to do anything dangerous unless -> `DRY_RUN` is explicitly set to `disabled`. - -## Requirements to actually run these scripts -* AWS access to pytorch account (META Employees: `bunnylol cloud pytorch`) -* Access to upload conda packages to the [`pytorch`](https://anaconda.org/pytorch) conda channel -* Access to the PyPI repositories (like [torch](https://pypi.org/project/torch)) - -## Promote pypi to staging - -Following steps needed in order to promote pypi to staging: -1. Edit `release_versions.sh` and set correct version -2. Run promote script : `./pypi/promote_pypi_to_staging.sh` -3. Edit and run `../analytics/validate_pypi_staging.py` to perform initial prevalidation of binaries for pypi promotion -4. Manually inspect and spot check binaries staged for pypi promotion by logging into s3 and downloading packages - -## Promote - -These are scripts related to promotion of release candidates to GA channels, these -can actually be used to promote pytorch, libtorch, and related domain libraries. - -> NOTE: Currently the script requires some knowledge on when to comment things out / comment things in - -> TODO: Make the script not rely on commenting things out / commenting this in - -### Usage - -```bash -./promote.sh -``` - -## Restoring backups - -All release candidates from `pytorch/pytorch` are currently backed up -to `s3://pytorch-backup/${TAG_NAME}` and can be restored to the test channels with the -`restore-backup.sh` script. - -Which backup to restore from is dictated by the `RESTORE_FROM` environment variable. - -### Usage -```bash -RESTORE_FROM=v1.5.0-rc5 ./restore-backup.sh -``` diff --git a/release/cut-release-branch.sh b/release/cut-release-branch.sh deleted file mode 100644 index 468dbfb18..000000000 --- a/release/cut-release-branch.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -: ' -So you are looking to cut a release branch? Well you came -to the right script. - -This script can be used to cut any branch on any repository - -For `pytorch/pytorch` usage would be like: -> DRY_RUN=disabled cut-release-branch.sh - -For `pytorch/builder` or domains usage would be like: -> DRY_RUN=disabled GIT_BRANCH_TO_CUT_FROM=main RELEASE_VERSION=1.11 cut-release-branch.sh -' - -set -eou pipefail - -GIT_TOP_DIR=$(git rev-parse --show-toplevel) -GIT_REMOTE=${GIT_REMOTE:-origin} -GIT_BRANCH_TO_CUT_FROM=${GIT_BRANCH_TO_CUT_FROM:-viable/strict} - -# should output something like 1.11 -RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} - -DRY_RUN_FLAG="--dry-run" -if [[ ${DRY_RUN:-enabled} == "disabled" ]]; then - DRY_RUN_FLAG="" -fi - - -( - set -x - git fetch --all - git checkout "${GIT_REMOTE}/${GIT_BRANCH_TO_CUT_FROM}" -) - -for branch in "release/${RELEASE_VERSION}" "orig/release/${RELEASE_VERSION}"; do - if git rev-parse --verify "${branch}" >/dev/null 2>/dev/null; then - echo "+ Branch ${branch} already exists, skipping..." - continue - else - ( - set -x - git checkout "${GIT_REMOTE}/${GIT_BRANCH_TO_CUT_FROM}" - git checkout -b "${branch}" - git push "${GIT_REMOTE}" "${branch}" - ) - fi -done diff --git a/release/promote.sh b/release/promote.sh deleted file mode 100644 index 8b7939783..000000000 --- a/release/promote.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/release_versions.sh" - -# Make sure to update these versions when doing a release first -PYTORCH_VERSION=${PYTORCH_VERSION:-2.3.0} -TORCHVISION_VERSION=${TORCHVISION_VERSION:-0.18.0} -TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-2.3.0} -TORCHTEXT_VERSION=${TORCHTEXT_VERSION:-0.18.0} -TORCHREC_VERSION=${TORCHREC_VERSION:-0.8.0} -TENSORRT_VERSION=${TENSORRT_VERSION:-2.2.0} -FBGEMMGPU_VERSION=${FBGEMMGPU_VERSION:-1.0.0} - -DRY_RUN=${DRY_RUN:-enabled} - -promote_s3() { - local package_name - package_name=$1 - local package_type - package_type=$2 - local promote_version - promote_version=$3 - - echo "=-=-=-= Promoting ${package_name}'s v${promote_version} ${package_type} packages' =-=-=-=" - ( - set -x - TEST_PYTORCH_PROMOTE_VERSION="${promote_version}" \ - PACKAGE_NAME="${package_name}" \ - PACKAGE_TYPE="${package_type}" \ - TEST_WITHOUT_GIT_TAG=1 \ - DRY_RUN="${DRY_RUN}" ${DIR}/promote/s3_to_s3.sh - ) - echo -} - -promote_conda() { - local package_name - package_name=$1 - local package_type - package_type=$2 - local promote_version - promote_version=$3 - echo "=-=-=-= Promoting ${package_name}'s v${promote_version} ${package_type} packages' =-=-=-=" - ( - ANACONDA="echo + anaconda" - if [[ "${DRY_RUN:-enabled}" = "disabled" ]]; then - ANACONDA="anaconda" - set -x - else - echo "DRY_RUN enabled not actually doing work" - fi - ${ANACONDA} copy --to-owner ${PYTORCH_CONDA_TO:-pytorch} ${PYTORCH_CONDA_FROM:-pytorch-test}/${package_name}/${promote_version} - ) - echo -} - -promote_pypi() { - local package_name - package_name=$1 - local promote_version - promote_version=$2 - echo "=-=-=-= Promoting ${package_name}'s v${promote_version} to pypi' =-=-=-=" - ( - set -x - TEST_PYTORCH_PROMOTE_VERSION="${promote_version}" \ - PACKAGE_NAME="${package_name}" \ - TEST_WITHOUT_GIT_TAG=1 \ - DRY_RUN="${DRY_RUN}" ${DIR}/promote/wheel_to_pypi.sh - ) - echo -} - -# Promote s3 dependencies -# promote_s3 "certifi" whl "2022.12.7" -# promote_s3 "charset_normalizer" whl "2.1.1" -# promote_s3 "cmake" whl "3.25" -# promote_s3 "colorama" whl "0.4.6" -# promote_s3 "triton" whl "2.0.0" -# promote_s3 "pytorch_triton_rocm" whl "2.0.1" -# promote_s3 "tqdm" whl "4.64.1" -# promote_s3 "Pillow" whl "9.3.0" -# for python 3.8-3.11 -# promote_s3 "numpy" whl "1.24.1" -# for python 3.7 older pytorch versions -# promote_s3 "numpy" whl "1.21.6" -# promote_s3 "urllib3" whl "1.26.13" -# promote_s3 "lit" whl "15.0.7" -# promote_s3 "sympy" whl "1.11.1" -# promote_s3 "typing_extensions" whl "4.4.0" -# promote_s3 "filelock" whl "3.9.0" -# promote_s3 "mpmath" whl "1.2.1" -# promote_s3 "MarkupSafe" whl "2.1.2" -# promote_s3 "Jinja2" whl "3.1.2" -# promote_s3 "idna" whl "3.4" -# promote_s3 "networkx" whl "3.0" -# promote_s3 "packaging" whl "22.0" -# promote_s3 "requests" whl "2.28.1" - -# promote_s3 torch whl "${PYTORCH_VERSION}" -# promote_s3 torchvision whl "${TORCHVISION_VERSION}" -# promote_s3 torchaudio whl "${TORCHAUDIO_VERSION}" -# promote_s3 torchtext whl "${TORCHTEXT_VERSION}" -# promote_s3 torchrec whl "${TORCHREC_VERSION}" -# promote_s3 fbgemm-gpu whl "${FBGEMMGPU_VERSION}" -# promote_s3 "libtorch-*" libtorch "${PYTORCH_VERSION}" -# promote_s3 "torch_tensorrt" whl "${TENSORRT_VERSION}" - -# promote_conda torchtriton conda "2.1.0" -# promote_conda pytorch-cuda conda "11.8" -# promote_conda pytorch-cuda conda "12.1" - -# promote_conda pytorch conda "${PYTORCH_VERSION}" -# promote_conda torchvision conda "${TORCHVISION_VERSION}" -# promote_conda torchaudio conda "${TORCHAUDIO_VERSION}" -# promote_conda torchtext conda "${TORCHTEXT_VERSION}" - -# Uncomment these to promote to pypi -LINUX_VERSION_SUFFIX="%2Bcu102" -WIN_VERSION_SUFFIX="%2Bcpu" -# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torch "${PYTORCH_VERSION}" -# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torch "${PYTORCH_VERSION}" -# PLATFORM="win_amd64" VERSION_SUFFIX="${WIN_VERSION_SUFFIX}" promote_pypi torch "${PYTORCH_VERSION}" -# PLATFORM="macosx_11_0" VERSION_SUFFIX="" promote_pypi torch "${PYTORCH_VERSION}" # m1 mac - -# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torchvision "${TORCHVISION_VERSION}" -# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torchvision "${TORCHVISION_VERSION}" -# PLATFORM="win_amd64" VERSION_SUFFIX="${WIN_VERSION_SUFFIX}" promote_pypi torchvision "${TORCHVISION_VERSION}" -# PLATFORM="macosx_11_0" VERSION_SUFFIX="" promote_pypi torchvision "${TORCHVISION_VERSION}" - -# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torchaudio "${TORCHAUDIO_VERSION}" -# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torchaudio "${TORCHAUDIO_VERSION}" -# PLATFORM="win_amd64" VERSION_SUFFIX="${WIN_VERSION_SUFFIX}" promote_pypi torchaudio "${TORCHAUDIO_VERSION}" -# PLATFORM="macosx_11_0" VERSION_SUFFIX="" promote_pypi torchaudio "${TORCHAUDIO_VERSION}" diff --git a/release/promote/common_utils.sh b/release/promote/common_utils.sh deleted file mode 100644 index e50ccb110..000000000 --- a/release/promote/common_utils.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -exit_if_not_on_git_tag() { - # Have an override for debugging purposes - if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]] ;then - >&2 echo "+ WARN: Continuing without being on a git tag" - exit 0 - fi - # Exit if we're not currently on a git tag - if ! git describe --tags --exact >/dev/null 2>/dev/null; then - >&2 echo "- ERROR: Attempting to promote on a non-git tag, must have tagged current commit locally first" - exit 1 - fi - # Exit if we're currently on an RC - if git describe --tags | grep "-rc" >/dev/null 2>/dev/null; then - >&2 echo "- ERROR: Attempting to promote on a non GA git tag, current tag must be a GA tag" - >&2 echo " Example: v1.5.0" - exit 1 - fi -} - -get_pytorch_version() { - if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]];then - if [[ -z "${TEST_PYTORCH_PROMOTE_VERSION-}" ]]; then - >&2 echo "- ERROR: Specified TEST_WITHOUT_GIT_TAG without specifying TEST_PYTORCH_PROMOTE_VERSION" - >&2 echo "- TEST_PYTORCH_PROMOTE_VERSION must be specified" - exit 1 - else - echo "${TEST_PYTORCH_PROMOTE_VERSION}" - exit 0 - fi - fi - exit_if_not_on_git_tag - # Echo git tag, strip leading v - git describe --tags | sed -e 's/^v//' -} - -aws_promote() { - package_name=$1 - pytorch_version=$(get_pytorch_version) - # Dry run by default - DRY_RUN=${DRY_RUN:-enabled} - DRY_RUN_FLAG="--dryrun" - if [[ $DRY_RUN = "disabled" ]]; then - DRY_RUN_FLAG="" - fi - AWS=${AWS:-aws} - ( - set -x - ${AWS} s3 cp ${DRY_RUN_FLAG} \ - --only-show-errors \ - --acl public-read \ - --recursive \ - --exclude '*' \ - --include "*${package_name}-${pytorch_version}*" \ - "${PYTORCH_S3_FROM/\/$//}" \ - "${PYTORCH_S3_TO/\/$//}" - ) - # ^ We grep for package_name-.*pytorch_version to avoid any situations where domain libraries have - # the same version on our S3 buckets -} diff --git a/release/promote/conda_to_conda.sh b/release/promote/conda_to_conda.sh deleted file mode 100755 index c890f9c18..000000000 --- a/release/promote/conda_to_conda.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/common_utils.sh" - - -# Allow for users to pass PACKAGE_NAME -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-pytorch} -PYTORCH_CONDA_FROM=${PYTORCH_CONDA_FROM:-pytorch-test} -PYTORCH_CONDA_TO=${PYTORCH_CONDA_TO:-pytorch} -CONDA_PLATFORMS="linux-64 osx-64 win-64 noarch" - -pytorch_version="$(get_pytorch_version)" - -tmp_dir="$(mktemp -d)" -pushd "${tmp_dir}" -trap 'rm -rf ${tmp_dir}' EXIT - -conda_search() { - conda search -q "${PYTORCH_CONDA_FROM}::${PACKAGE_NAME}==${pytorch_version}" -c "${PYTORCH_CONDA_FROM}" --platform "${platform}" \ - | grep -e "^${PACKAGE_NAME}" \ - | awk -F ' *' '{print $3}' \ - | xargs -I % echo "https://anaconda.org/${PYTORCH_CONDA_FROM}/${PACKAGE_NAME}/${pytorch_version}/download/${platform}/${PACKAGE_NAME}-${pytorch_version}-%.tar.bz2" -} - -pkgs_to_download=() -for platform in ${CONDA_PLATFORMS}; do - pkgs_to_download+=($(\ - conda_search 2>/dev/null || true - )) - # Create directory where packages will eventually be downloaded - mkdir -p "${platform}" -done - -my_curl() { - local dl_url=$1 - local start=$(date +%s) - # downloads should be distinguished by platform which should be the second - # to last field in the url, this is to avoid clobbering same named files - # for different platforms - dl_dir=$(echo "${dl_url}" | rev | cut -d'/' -f 2 | rev) - dl_name=$(echo "${dl_url}" | rev | cut -d'/' -f 1 | rev) - curl -fsSL -o "${dl_dir}/${dl_name}" "${dl_url}" - local end=$(date +%s) - local diff=$(( end - start )) - echo "+ ${dl_url} took ${diff}s" -} -export -f my_curl - -# Download all packages in parallel -printf '%s\n' "${pkgs_to_download[@]}" \ - | xargs -P 10 -I % bash -c '(declare -t my_curl); my_curl %' - -# dry run by default -DRY_RUN=${DRY_RUN:-enabled} -ANACONDA="true anaconda" -if [[ $DRY_RUN = "disabled" ]]; then - ANACONDA="anaconda" -fi -( - # We use --skip here to avoid re-uploading files we've already uploaded - set -x - ${ANACONDA} upload --skip -u ${PYTORCH_CONDA_TO} $(find . -name '*.bz2') -) - -popd diff --git a/release/promote/prep_binary_for_pypi.sh b/release/promote/prep_binary_for_pypi.sh deleted file mode 100755 index bb088f554..000000000 --- a/release/promote/prep_binary_for_pypi.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Preps binaries for publishing to pypi by removing the -# version suffix we normally add for all binaries -# (outside of default ones, CUDA 10.2 currently) - -# Usage is: -# $ prep_binary_for_pypy.sh - -# Will output a whl in your current directory - -set -eou pipefail -shopt -s globstar - -OUTPUT_DIR=${OUTPUT_DIR:-$(pwd)} - -tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir}' EXIT - -for whl_file in "$@"; do - whl_file=$(realpath "${whl_file}") - whl_dir="${tmp_dir}/$(basename "${whl_file}")_unzipped" - mkdir -pv "${whl_dir}" - ( - set -x - unzip -q "${whl_file}" -d "${whl_dir}" - ) - version_with_suffix=$(grep '^Version:' "${whl_dir}"/*/METADATA | cut -d' ' -f2) - version_with_suffix_escaped=${version_with_suffix/+/%2B} - # Remove all suffixed +bleh versions - version_no_suffix=${version_with_suffix/+*/} - new_whl_file=${OUTPUT_DIR}/$(basename "${whl_file/${version_with_suffix_escaped}/${version_no_suffix}}") - dist_info_folder=$(find "${whl_dir}" -type d -name '*.dist-info' | head -1) - basename_dist_info_folder=$(basename "${dist_info_folder}") - dirname_dist_info_folder=$(dirname "${dist_info_folder}") - ( - set -x - find "${dist_info_folder}" -type f -exec sed -i "s!${version_with_suffix}!${version_no_suffix}!" {} \; - # Moves distinfo from one with a version suffix to one without - # Example: torch-1.8.0+cpu.dist-info => torch-1.8.0.dist-info - mv "${dist_info_folder}" "${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}" - cd "${whl_dir}" - zip -qr "${new_whl_file}" . - ) -done diff --git a/release/promote/s3_to_s3.sh b/release/promote/s3_to_s3.sh deleted file mode 100755 index 21495ac3b..000000000 --- a/release/promote/s3_to_s3.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/common_utils.sh" - -# Allow for users to pass PACKAGE_NAME -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-torch} -PACKAGE_TYPE=${PACKAGE_TYPE:-whl} - -PYTORCH_S3_BUCKET=${PYTORCH_S3_BUCKET:-s3://pytorch} -FROM=${FROM:-test} -PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${FROM}} -TO=${TO:-} -PYTORCH_S3_TO=${PYTORCH_S3_TO:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${TO}} - -aws_promote "${PACKAGE_NAME}" diff --git a/release/promote/wheel_to_pypi.sh b/release/promote/wheel_to_pypi.sh deleted file mode 100755 index 1e9b7411f..000000000 --- a/release/promote/wheel_to_pypi.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/common_utils.sh" - -# Allow for users to pass PACKAGE_NAME -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-torch} - -pytorch_version="$(get_pytorch_version)" -# Refers to the specific package we'd like to promote -# i.e. VERSION_SUFFIX='%2Bcu102' -# torch-1.8.0+cu102 -> torch-1.8.0 -VERSION_SUFFIX=${VERSION_SUFFIX:-} -# Refers to the specific platofmr we'd like to promote -# i.e. PLATFORM=linux_x86_64 -# For domains like torchaudio / torchtext this is to be left blank -PLATFORM=${PLATFORM:-} - -pkgs_to_promote=$(\ - curl -fsSL https://download.pytorch.org/whl/torch_stable.html \ - | grep "${PACKAGE_NAME}-${pytorch_version}${VERSION_SUFFIX}-" \ - | grep "${PLATFORM}" \ - | cut -d '"' -f2 -) - -tmp_dir="$(mktemp -d)" -output_tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir} ${output_tmp_dir}' EXIT -pushd "${output_tmp_dir}" - -# Dry run by default -DRY_RUN=${DRY_RUN:-enabled} -# On dry run just echo the commands that are meant to be run -TWINE_UPLOAD="echo twine upload" -if [[ $DRY_RUN = "disabled" ]]; then - TWINE_UPLOAD="twine upload" -fi - -for pkg in ${pkgs_to_promote}; do - pkg_basename="$(basename "${pkg}")" - # Don't attempt to change if manylinux2014 - if [[ "${pkg}" != *manylinux2014* ]]; then - pkg_basename="$(basename "${pkg//linux/manylinux1}")" - fi - orig_pkg="${tmp_dir}/${pkg_basename}" - ( - set -x - # Download package, sub out linux for manylinux1 - curl -fsSL -o "${orig_pkg}" "https://download.pytorch.org/whl/${pkg}" - ) - - if [[ -n "${VERSION_SUFFIX}" ]]; then - OUTPUT_DIR="${output_tmp_dir}" ${DIR}/prep_binary_for_pypi.sh "${orig_pkg}" - else - mv "${orig_pkg}" "${output_tmp_dir}/" - fi - - ( - set -x - ${TWINE_UPLOAD} \ - --disable-progress-bar \ - --non-interactive \ - ./*.whl - rm -rf ./*.whl - ) -done diff --git a/release/pypi/prep_binary_for_pypi.sh b/release/pypi/prep_binary_for_pypi.sh deleted file mode 100755 index 154b22852..000000000 --- a/release/pypi/prep_binary_for_pypi.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -# Preps binaries for publishing to pypi by removing the -# version suffix we normally add for all binaries -# (outside of default ones, CUDA 10.2 currently) - -# Usage is: -# $ prep_binary_for_pypy.sh - -# Will output a whl in your current directory - -set -eou pipefail -shopt -s globstar - -# Function copied from manywheel/build_common.sh -make_wheel_record() { - FPATH=$1 - if echo $FPATH | grep RECORD >/dev/null 2>&1; then - # if the RECORD file, then - echo "$FPATH,," - else - HASH=$(openssl dgst -sha256 -binary $FPATH | openssl base64 | sed -e 's/+/-/g' | sed -e 's/\//_/g' | sed -e 's/=//g') - FSIZE=$(ls -nl $FPATH | awk '{print $5}') - echo "$FPATH,sha256=$HASH,$FSIZE" - fi -} - -OUTPUT_DIR=${OUTPUT_DIR:-$(pwd)} - -tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir}' EXIT - -DEBUG=${DEBUG:-} - -for whl_file in "$@"; do - whl_file=$(realpath "${whl_file}") - whl_dir="${tmp_dir}/$(basename "${whl_file}")_unzipped" - mkdir -pv "${whl_dir}" - ( - set -x - unzip -q "${whl_file}" -d "${whl_dir}" - ) - version_with_suffix=$(grep '^Version:' "${whl_dir}"/*/METADATA | cut -d' ' -f2 | tr -d "[:space:]") - version_with_suffix_escaped=${version_with_suffix/+/%2B} - - # Remove all suffixed +bleh versions - version_no_suffix=${version_with_suffix/+*/} - new_whl_file=${OUTPUT_DIR}/$(basename "${whl_file/${version_with_suffix_escaped}/${version_no_suffix}}") - dist_info_folder=$(find "${whl_dir}" -type d -name '*.dist-info' | head -1) - basename_dist_info_folder=$(basename "${dist_info_folder}") - dirname_dist_info_folder=$(dirname "${dist_info_folder}") - ( - set -x - - find "${dist_info_folder}" -type f -exec sed -i "s!${version_with_suffix}!${version_no_suffix}!" {} \; - # Moves distinfo from one with a version suffix to one without - # Example: torch-1.8.0+cpu.dist-info => torch-1.8.0.dist-info - mv "${dist_info_folder}" "${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}" - cd "${whl_dir}" - - ( - set +x - # copied from manywheel/build_common.sh - # regenerate the RECORD file with new hashes - record_file="${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}/RECORD" - if [[ -e $record_file ]]; then - echo "Generating new record file $record_file" - : > "$record_file" - # generate records for folders in wheel - find * -type f | while read fname; do - make_wheel_record "$fname" >>"$record_file" - done - fi - ) - - rm -rf "${new_whl_file}" - zip -qr9 "${new_whl_file}" . - ) -done diff --git a/release/pypi/promote_pypi_to_production.sh b/release/pypi/promote_pypi_to_production.sh deleted file mode 100644 index 4da59ac84..000000000 --- a/release/pypi/promote_pypi_to_production.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/../release_versions.sh" - -PYTORCH_DIR=${PYTORCH_DIR:-~/pytorch} - -DRY_RUN=${DRY_RUN:-enabled} - -TWINE_UPLOAD="true twine upload" -if [[ ${DRY_RUN:-enabled} = "disabled" ]]; then - TWINE_UPLOAD="twine upload" -fi - -promote_staging_binaries() { - local package_name - package_name=$1 - local promote_version - promote_version=$2 - - ( - TMP_DIR=$(mktemp -d) - trap 'rm -rf ${TMP_DIR}' EXIT - pushd "${TMP_DIR}" - set -x - aws s3 sync "s3://pytorch-backup/${package_name}-${promote_version}-pypi-staging/" "${TMP_DIR}/" - ${TWINE_UPLOAD} --skip-existing "${TMP_DIR}/*.whl" - ) -} - -promote_staging_binaries torch "${PYTORCH_VERSION}" -promote_staging_binaries torchvision "${TORCHVISION_VERSION}" -promote_staging_binaries torchaudio "${TORCHAUDIO_VERSION}" - -promote_staging_binaries executorch "${EXECUTORCH_VERSION}" -#promote_staging_binaries torchtext "${TORCHTEXT_VERSION}" -#promote_staging_binaries torchdata "${TORCHDATA_VERSION}" - -#promote_staging_binaries torchao "${TORCHAO_VERSION}" -#promote_staging_binaries torchtune "${TORCHTUNE_VERSION}" diff --git a/release/pypi/promote_pypi_to_staging.sh b/release/pypi/promote_pypi_to_staging.sh deleted file mode 100644 index 72f04e3da..000000000 --- a/release/pypi/promote_pypi_to_staging.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/../release_versions.sh" - -DRY_RUN=${DRY_RUN:-enabled} - -upload_pypi_to_staging() { - local package_name - package_name=$1 - local promote_version - promote_version=$2 - echo "=-=-=-= Promoting ${package_name}'s v${promote_version} to pypi staging' =-=-=-=" - ( - set -x - PACKAGE_VERSION="${promote_version}" PACKAGE_NAME="${package_name}" DRY_RUN="${DRY_RUN}" bash ${DIR}/upload_pypi_to_staging.sh - ) - echo -} - -# Uncomment these to promote to pypi -LINUX_VERSION_SUFFIX="%2Bcu124" -CPU_VERSION_SUFFIX="%2Bcpu" -MACOS_X86_64="macosx_.*_x86_64" -MACOS_ARM64="macosx_.*_arm64" - -PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" ARCH="cu124" upload_pypi_to_staging torch "${PYTORCH_VERSION}" -PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" upload_pypi_to_staging torch "${PYTORCH_VERSION}" -PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torch "${PYTORCH_VERSION}" -PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torch "${PYTORCH_VERSION}" - -PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" ARCH="cu124" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}" -PLATFORM="linux_aarch64" VERSION_SUFFIX="" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}" -PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}" -PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}" - -PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" ARCH="cu124" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}" -PLATFORM="linux_aarch64" VERSION_SUFFIX="" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}" -PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}" -PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}" - -#PLATFORM="manylinux2014_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" upload_pypi_to_staging torchao "${TORCHAO_VERSION}" -#PLATFORM="" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchtune "${TORCHTUNE_VERSION}" - -#PLATFORM="linux_x86" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging executorch "${EXECUTORCH_VERSION}" -#PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging executorch "${EXECUTORCH_VERSION}" - -#PLATFORM="linux_x86" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}" -#PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}" -#PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}" - -#PLATFORM="linux_x86_64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}" -#PLATFORM="win_amd64" VERSION_SUFFIX="" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}" -#PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}" diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh deleted file mode 100644 index 732c049ba..000000000 --- a/release/pypi/upload_pypi_to_staging.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# Allow for users to pass PACKAGE_NAME - -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-torch} - -PACKAGE_VERSION=${PACKAGE_VERSION:-1.0.0} -# Refers to the specific package we'd like to promote -# i.e. VERSION_SUFFIX='%2Bcu102' -# torch-1.8.0+cu102 -> torch-1.8.0 -VERSION_SUFFIX=${VERSION_SUFFIX:-} -# Refers to the specific platofmr we'd like to promote -# i.e. PLATFORM=linux_x86_64 -# For domains like torchaudio / torchtext this is to be left blank -PLATFORM=${PLATFORM:-} -# Refers to the specific architecture we'd like to promote -# i.e. cpu, cu121, cu124 -ARCH==${ARCH:-cpu} - -pkgs_to_promote=$(\ - curl -fsSL "https://download.pytorch.org/whl/test/${ARCH}/${PACKAGE_NAME}/index.html" \ - | grep "${PACKAGE_NAME}-${PACKAGE_VERSION}${VERSION_SUFFIX}-" \ - | grep "${PLATFORM}" \ - | cut -d '"' -f2 -) - -tmp_dir="$(mktemp -d)" -output_tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir} ${output_tmp_dir}' EXIT -pushd "${output_tmp_dir}" - -# Dry run by default -DRY_RUN=${DRY_RUN:-enabled} -# On dry run just echo the commands that are meant to be run -DRY_RUN_FLAG="--dryrun" -if [[ $DRY_RUN = "disabled" ]]; then - DRY_RUN_FLAG="" -fi - -for pkg in ${pkgs_to_promote}; do - pkg_basename="$(basename "${pkg}")" - - if [[ "${pkg}" != *aarch64* && "${pkg}" != *torchao* ]]; then - # sub out linux for manylinux1 - pkg_basename="$(basename "${pkg//linux/manylinux1}")" - elif [[ "${pkg}" == *manylinux_2_17_aarch64* ]]; then - # strip manylinux_2_17 from core filename - pkg_basename="$(basename "${pkg//manylinux_2_17_aarch64./}")" - elif [[ "${pkg}" == *linux_aarch64* ]]; then - # domains change linux_aarch64 to manylinux2014_aarch64 - pkg_basename="$(basename "${pkg//linux_aarch64/manylinux2014_aarch64}")" - fi - - orig_pkg="${tmp_dir}/${pkg_basename}" - ( - set -x - curl -fSL -o "${orig_pkg}" "https://download.pytorch.org${pkg}" - ) - - if [[ -n "${VERSION_SUFFIX}" ]]; then - OUTPUT_DIR="${output_tmp_dir}" bash "${DIR}/prep_binary_for_pypi.sh" "${orig_pkg}" - else - mv "${orig_pkg}" "${output_tmp_dir}/" - fi - - ( - set -x - aws s3 cp ${DRY_RUN_FLAG} *.whl "s3://pytorch-backup/${PACKAGE_NAME}-${PACKAGE_VERSION}-pypi-staging/" - rm -rf ./*.whl - ) -done diff --git a/release/release_versions.sh b/release/release_versions.sh deleted file mode 100644 index 32e8c47d8..000000000 --- a/release/release_versions.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# Make sure to update these versions when doing a release first -PYTORCH_VERSION=${PYTORCH_VERSION:-2.3.1} -TORCHVISION_VERSION=${TORCHVISION_VERSION:-0.18.1} -TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-2.3.1} -TORCHTEXT_VERSION=${TORCHTEXT_VERSION:-0.18.0} -TORCHREC_VERSION=${TORCHREC_VERSION:-0.8.0} -TENSORRT_VERSION=${TENSORRT_VERSION:-2.4.0} -EXECUTORCH_VERSION=${EXECUTORCH_VERSION:-0.3.0} -TORCHAO_VERSION=${TORCHAO_VERSION:-0.4.0} -TORCHTUNE_VERSION=${TORCHTUNE_VERSION:-0.2.1} -FBGEMMGPU_VERSION=${FBGEMMGPU_VERSION:-1.0.0} diff --git a/release/restore-backup.sh b/release/restore-backup.sh deleted file mode 100755 index 6cf79549d..000000000 --- a/release/restore-backup.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/promote/common_utils.sh" - -if [[ -z "${RESTORE_FROM:-}" ]]; then - echo "ERROR: RESTORE_FROM environment variable must be specified" - echo " example: RESTORE_FROM=v1.6.0-rc3 ${0}" - exit 1 -fi - -DRY_RUN=${DRY_RUN:-enabled} - -PYTORCH_S3_BACKUP_BUCKET=${PYTORCH_S3_BACKUP_BUCKET:-s3://pytorch-backup/${RESTORE_FROM}} -PYTORCH_S3_TEST_BUCKET=${PYTORCH_S3_TEST_BUCKET:-s3://pytorch/} -PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BACKUP_BUCKET}} -PYTORCH_S3_TO=${PYTORCH_S3_TO:-s3://pytorch/} - -restore_wheels() { - aws_promote torch whl -} - -restore_libtorch() { - aws_promote libtorch-* libtorch -} - -ANACONDA="true anaconda" -if [[ ${DRY_RUN} = "disabled" ]]; then - ANACONDA="anaconda" -fi -PYTORCH_CONDA_TO=${PYTORCH_CONDA_TO:-pytorch-test} - -upload_conda() { - local pkg - pkg=${1} - ( - set -x - ${ANACONDA} upload --skip -u "${PYTORCH_CONDA_TO}" "${pkg}" - ) -} - -export -f upload_conda - -restore_conda() { - TMP_DIR="$(mktemp -d)" - trap 'rm -rf ${TMP_DIR}' EXIT - ( - set -x - aws s3 cp --recursive "${PYTORCH_S3_BACKUP_BUCKET}/conda" "${TMP_DIR}/" - ) - export ANACONDA - export PYTORCH_CONDA_TO - # Should upload all bz2 packages in parallel for quick restoration - find "${TMP_DIR}" -name '*.bz2' -type f \ - | xargs -P 10 -I % bash -c "(declare -t upload_conda); upload_conda %" -} - - -restore_wheels -restore_libtorch -restore_conda