Skip to content

Commit

Permalink
Add build-locally.py
Browse files Browse the repository at this point in the history
  • Loading branch information
isuruf committed Jan 22, 2022
1 parent 5e85931 commit 6362bc4
Show file tree
Hide file tree
Showing 9 changed files with 179 additions and 22 deletions.
12 changes: 4 additions & 8 deletions .azure-pipelines/azure-pipelines-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,9 @@ jobs:
mkdir -p build_artifacts/linux-64/
export CI=azure
export CONFIG=linux64
export CONFIG=linux64_cuda102
export IMAGE_NAME=quay.io/condaforge/linux-anvil-cos7-cuda:10.2
export AZURE=True
export CF_CUDA_VERSION="10.2"
.scripts/run_docker_build.sh
displayName: Run docker build for CUDA 10.2
Expand All @@ -79,10 +78,9 @@ jobs:
mkdir -p build_artifacts/linux-64/
export CI=azure
export CONFIG=linux64
export CONFIG=linux64_cuda110
export IMAGE_NAME=quay.io/condaforge/linux-anvil-cuda:11.0
export AZURE=True
export CF_CUDA_VERSION="11.0"
.scripts/run_docker_build.sh
displayName: Run docker build for CUDA 11.0
Expand All @@ -108,10 +106,9 @@ jobs:
mkdir -p build_artifacts/linux-64/
export CI=azure
export CONFIG=linux64
export CONFIG=linux64_cuda111
export IMAGE_NAME=quay.io/condaforge/linux-anvil-cuda:11.1
export AZURE=True
export CF_CUDA_VERSION="11.1"
.scripts/run_docker_build.sh
displayName: Run docker build for CUDA 11.1
Expand All @@ -137,10 +134,9 @@ jobs:
mkdir -p build_artifacts/linux-64/
export CI=azure
export CONFIG=linux64
export CONFIG=linux64_cuda112
export IMAGE_NAME=quay.io/condaforge/linux-anvil-cuda:11.2
export AZURE=True
export CF_CUDA_VERSION="11.2"
.scripts/run_docker_build.sh
displayName: Run docker build for CUDA 11.2
Expand Down
9 changes: 7 additions & 2 deletions .ci_support/build_all.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@ def get_host_platform():
return "win"


def get_config_name(arch):
platform = get_host_platform()
return os.environ.get("CONFIG", "{}{}".format(platform, arch))



def build_all(recipes_dir, arch):
folders = os.listdir(recipes_dir)
old_comp_folders = []
Expand All @@ -35,8 +41,7 @@ def build_all(recipes_dir, arch):

platform = get_host_platform()
script_dir = os.path.dirname(os.path.realpath(__file__))
variant_config_file = os.path.join(script_dir, '{}{}.yaml'.format(
platform, arch))
variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch)))

found_cuda = False
found_centos7 = False
Expand Down
6 changes: 1 addition & 5 deletions .ci_support/linux64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,4 @@ channel_sources:
docker_image:
- quay.io/condaforge/linux-anvil-comp7
cuda_compiler_version:
- None # [os.environ.get('CF_CUDA_VERSION', 'None') == 'None']
- 10.2 # [os.environ.get('CF_CUDA_VERSION', 'None') == '10.2']
- 11.0 # [os.environ.get('CF_CUDA_VERSION', 'None') == '11.0']
- 11.1 # [os.environ.get('CF_CUDA_VERSION', 'None') == '11.1']
- 11.2 # [os.environ.get('CF_CUDA_VERSION', 'None') == '11.2']
- None
14 changes: 14 additions & 0 deletions .ci_support/linux64_cuda102.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
c_compiler:
- gcc
cxx_compiler:
- gxx
fortran_compiler:
- gfortran
target_platform:
- linux-64
channel_sources:
- conda-forge
docker_image:
- quay.io/condaforge/linux-anvil-cos7-cuda:10.2
cuda_compiler_version:
- 10.2
14 changes: 14 additions & 0 deletions .ci_support/linux64_cuda110.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
c_compiler:
- gcc
cxx_compiler:
- gxx
fortran_compiler:
- gfortran
target_platform:
- linux-64
channel_sources:
- conda-forge
docker_image:
- quay.io/condaforge/linux-anvil-cuda:11.0
cuda_compiler_version:
- 11.0
14 changes: 14 additions & 0 deletions .ci_support/linux64_cuda111.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
c_compiler:
- gcc
cxx_compiler:
- gxx
fortran_compiler:
- gfortran
target_platform:
- linux-64
channel_sources:
- conda-forge
docker_image:
- quay.io/condaforge/linux-anvil-cuda:11.1
cuda_compiler_version:
- 11.1
14 changes: 14 additions & 0 deletions .ci_support/linux64_cuda112.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
c_compiler:
- gcc
cxx_compiler:
- gxx
fortran_compiler:
- gfortran
target_platform:
- linux-64
channel_sources:
- conda-forge
docker_image:
- quay.io/condaforge/linux-anvil-cuda:11.2
cuda_compiler_version:
- 11.2
17 changes: 10 additions & 7 deletions .scripts/run_docker_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,17 @@ if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then
HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u)
fi

if [ -z "${IMAGE_NAME}" ]; then
if [ -z "${DOCKER_IMAGE}" ]; then
SHYAML_INSTALLED="$(shyaml -h || echo NO)"
if [ "${SHYAML_INSTALLED}" == "NO" ]; then
echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Falling back to quay.io/condaforge/linux-anvil-comp7"
IMAGE_NAME="quay.io/condaforge/linux-anvil-comp7"
echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Trying to parse with coreutils"
DOCKER_IMAGE=$(cat .ci_support/${CONFIG}.yaml | grep '^docker_image:$' -A 1 | tail -n 1 | cut -b 3-)
if [ "${DOCKER_IMAGE}" = "" ]; then
echo "No docker_image entry found in ${CONFIG}. Falling back to quay.io/condaforge/linux-anvil-comp7"
DOCKER_IMAGE="quay.io/condaforge/linux-anvil-comp7"
fi
else
IMAGE_NAME="$(cat "${REPO_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 quay.io/condaforge/linux-anvil-comp7 )"
DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 quay.io/condaforge/linux-anvil-comp7 )"
fi
fi

Expand All @@ -40,16 +44,15 @@ if [ "${AZURE}" == "True" ]; then
DOCKER_RUN_ARGS=""
fi

docker pull "${DOCKER_IMAGE}"
docker run ${DOCKER_RUN_ARGS} \
--pull always \
-v "${REPO_ROOT}:/home/conda/staged-recipes" \
-e HOST_USER_ID=${HOST_USER_ID} \
-e AZURE=${AZURE} \
-e CONFIG \
-e CI \
-e CF_CUDA_VERSION \
-e DEFAULT_LINUX_VERSION \
$IMAGE_NAME \
"${DOCKER_IMAGE}" \
bash \
"/home/conda/staged-recipes/${PROVIDER_DIR}/build_steps.sh"

Expand Down
101 changes: 101 additions & 0 deletions build-locally.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
#
# This file has been generated by conda-smithy in order to build the recipe
# locally.
#
import os
import glob
import subprocess
from argparse import ArgumentParser
import platform


def setup_environment(ns):
os.environ["CONFIG"] = ns.config
os.environ["UPLOAD_PACKAGES"] = "False"
os.environ["IS_PR_BUILD"] = "True"
if ns.debug:
os.environ["BUILD_WITH_CONDA_DEBUG"] = "1"
if ns.output_id:
os.environ["BUILD_OUTPUT_ID"] = ns.output_id
if "MINIFORGE_HOME" not in os.environ:
os.environ["MINIFORGE_HOME"] = os.path.join(
os.path.dirname(__file__), "miniforge3"
)
if "OSX_SDK_DIR" not in os.environ:
os.environ["OSX_SDK_DIR"] = os.path.join(
os.path.dirname(__file__), "SDKs"
)


def run_docker_build(ns):
script = ".scripts/run_docker_build.sh"
subprocess.check_call([script])


def run_osx_build(ns):
script = ".scripts/run_osx_build.sh"
subprocess.check_call([script])


def verify_config(ns):
valid_configs = {
os.path.basename(f)[:-5] for f in glob.glob(".ci_support/*.yaml")
}
print(f"valid configs are {valid_configs}")
if ns.config in valid_configs:
print("Using " + ns.config + " configuration")
return
elif len(valid_configs) == 1:
ns.config = valid_configs.pop()
print("Found " + ns.config + " configuration")
elif ns.config is None:
print("config not selected, please choose from the following:\n")
selections = list(enumerate(sorted(valid_configs), 1))
for i, c in selections:
print(f"{i}. {c}")
s = input("\n> ")
idx = int(s) - 1
ns.config = selections[idx][1]
print(f"selected {ns.config}")
else:
raise ValueError("config " + ns.config + " is not valid")
# Remove the following, as implemented
if ns.config.startswith("win"):
raise ValueError(
f"only Linux/macOS configs currently supported, got {ns.config}"
)
elif ns.config.startswith("osx") and platform.system() == "Darwin":
if "OSX_SDK_DIR" not in os.environ:
raise RuntimeError(
"Need OSX_SDK_DIR env variable set. Run 'export OSX_SDK_DIR=/opt'"
"to download the SDK automatically to '/opt/MacOSX<ver>.sdk'"
)


def main(args=None):
p = ArgumentParser("build-locally")
p.add_argument("config", default=None, nargs="?")
p.add_argument(
"--debug",
action="store_true",
help="Setup debug environment using `conda debug`",
)
p.add_argument(
"--output-id", help="If running debug, specify the output to setup."
)

ns = p.parse_args(args=args)
verify_config(ns)
setup_environment(ns)

if ns.config.startswith("linux") or (
ns.config.startswith("osx") and platform.system() == "Linux"
):
run_docker_build(ns)
elif ns.config.startswith("osx"):
run_osx_build(ns)


if __name__ == "__main__":
main()

0 comments on commit 6362bc4

Please sign in to comment.