From cd70115ac79dde6247ba81428aca8d9ce54a30eb Mon Sep 17 00:00:00 2001 From: Dale Roberts Date: Wed, 17 Jan 2024 12:20:51 +1100 Subject: [PATCH 1/4] Big update --- .github/workflows/build.yml | 25 +++++++ .github/workflows/build_and_test.yml | 71 ------------------- .github/workflows/deploy.yml | 22 +++--- .github/workflows/get_changed_env.yml | 41 +++++++++++ .github/workflows/manual_trigger.yml | 54 ++++++++++++++ .github/workflows/pull_request.yml | 71 +++++++++++++++++++ .github/workflows/push_to_main.yml | 35 +++++++++ .github/workflows/test.yml | 24 +++++++ environments/analysis3/build_inner.sh | 8 +++ environments/analysis3/config.sh | 23 ++++++ environments/analysis3/deploy.sh | 18 +++++ .../analysis3}/environment.yml | 1 + .../analysis3}/testconfig.yml | 14 +--- modules/common_v3 | 6 +- scripts/build.sh | 66 +++++++++++------ scripts/condaenv.sh | 4 +- scripts/deploy.sh | 44 ++++-------- scripts/dev_prep.sh | 4 ++ scripts/functions.sh | 22 +++++- scripts/initialise.sh | 27 +++---- scripts/install_config.sh | 21 +++--- scripts/launcher.sh | 41 ++++++++++- scripts/launcher_conf.sh | 2 +- scripts/test.sh | 43 +++++++---- scripts/test_environment.py | 10 +-- 25 files changed, 488 insertions(+), 209 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/build_and_test.yml create mode 100644 .github/workflows/get_changed_env.yml create mode 100644 .github/workflows/manual_trigger.yml create mode 100644 .github/workflows/pull_request.yml create mode 100644 .github/workflows/push_to_main.yml create mode 100644 .github/workflows/test.yml create mode 100644 environments/analysis3/build_inner.sh create mode 100644 environments/analysis3/config.sh create mode 100644 environments/analysis3/deploy.sh rename {scripts => environments/analysis3}/environment.yml (99%) rename {scripts => environments/analysis3}/testconfig.yml (81%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..9c00e03 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,25 @@ +name: Build conda env +on: + workflow_call: + inputs: + environment: + required: true + type: string +jobs: + build: + name: Build environment + runs-on: ubuntu-latest + steps: + - name: Build environment + uses: appleboy/ssh-action@v0.1.7 + with: + host: gadi.nci.org.au + username: ${{secrets.GADI_USER}} + key: ${{secrets.DEPLOY_KEY}} + command_timeout: 120m + script: | + export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts + eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh ) + eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh ) + cd $JOB_LOG_DIR + qsub -N build_${{ inputs.environment }} -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR,CONDA_ENVIRONMENT=${{ inputs.environment }} -P kr06 -q copyq -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/build.sh diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml deleted file mode 100644 index a007d5a..0000000 --- a/.github/workflows/build_and_test.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Build and test conda env -on: pull_request -jobs: - build: - runs-on: ubuntu-latest - container: - image: quay.io/singularity/singularity:v3.11.4 - options: --privileged - steps: - - name: Checkout repository - ### Latest at time of writing - uses: actions/checkout@v3.3.0 - - name: Check if container definition has changed - id: changed-container-def - uses: tj-actions/changed-files@v41 - with: - files_yaml: | - containerdef: - - container/container.def - - name: Build container if definition has changed - if: steps.changed-container-def.outputs.containerdef_any_changed == 'true' - run: | - sudo -E singularity build container/base.sif container/container.def - - name: Sync repository to Gadi - ### Latest at time of writing - uses: up9cloud/action-rsync@v1.3 - env: - HOST: gadi.nci.org.au - TARGET: ${{secrets.GADI_REPO_PATH}} - KEY: ${{secrets.DEPLOY_KEY}} - USER: ${{secrets.GADI_USER}} - - name: Create Admin dirs on Gadi - uses: appleboy/ssh-action@v0.1.7 - with: - host: gadi.nci.org.au - username: ${{secrets.GADI_USER}} - key: ${{secrets.DEPLOY_KEY}} - script: | - source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh - source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh - mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR - set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR - - name: Build environment - uses: appleboy/ssh-action@v0.1.7 - with: - host: gadi.nci.org.au - username: ${{secrets.GADI_USER}} - key: ${{secrets.DEPLOY_KEY}} - command_timeout: 120m - script: | - export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts - eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh ) - eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh ) - cd $JOB_LOG_DIR - qsub -N build_conda -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR -P kr06 -q copyq -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/build.sh - test: - needs: build - runs-on: ubuntu-latest - steps: - - name: Test environment - uses: appleboy/ssh-action@v0.1.7 - with: - host: gadi.nci.org.au - username: ${{secrets.GADI_USER}} - key: ${{secrets.DEPLOY_KEY}} - script: | - export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts - eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh ) - eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh ) - cd $JOB_LOG_DIR - qsub -N test_conda -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR -P kr06 -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/test.sh \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6a305f1..2a77cbe 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,23 +1,16 @@ name: Deploy conda env on: - push: - branches: main + workflow_call: + inputs: + environment: + required: true + type: string jobs: deploy: + name: Deploy environment runs-on: ubuntu-latest steps: - - name: Checkout repository - ### Latest at time of writing - uses: actions/checkout@v3.3.0 - - name: Sync repository to Gadi - ### Latest at time of writing - uses: up9cloud/action-rsync@v1.3 - env: - HOST: gadi.nci.org.au - TARGET: ${{secrets.GADI_REPO_PATH}} - KEY: ${{secrets.DEPLOY_KEY}} - USER: ${{secrets.GADI_USER}} - - name: Deploy conda environment + - name: Deploy environment uses: appleboy/ssh-action@v0.1.7 with: host: gadi.nci.org.au @@ -25,6 +18,7 @@ jobs: key: ${{secrets.DEPLOY_KEY}} script: | export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts + export CONDA_ENVIRONMENT=${{ inputs.environment }} eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh ) eval $( grep BUILD_STAGE_DIR $SCRIPT_DIR/install_config.sh ) "${SCRIPT_DIR}"/deploy.sh diff --git a/.github/workflows/get_changed_env.yml b/.github/workflows/get_changed_env.yml new file mode 100644 index 0000000..71a0ef9 --- /dev/null +++ b/.github/workflows/get_changed_env.yml @@ -0,0 +1,41 @@ +name: Get changed conda envs +on: + workflow_call: + outputs: + matrix: + value: ${{ jobs.generate_matrix.outputs.matrix }} +jobs: + generate_matrix: + name: Determine changed environments + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.diff.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - name: Check changed environments + id: diff + run: | + # See https://github.community/t/check-pushed-file-changes-with-git-diff-tree-in-github-actions/17220/10 + ### https://stackoverflow.com/questions/59977364/github-actions-how-use-strategy-matrix-with-script + if [ $GITHUB_BASE_REF ]; then + # Pull Request + git fetch origin $GITHUB_BASE_REF --depth=1 + export DIFF=$( git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA ) + echo "Diff between origin/$GITHUB_BASE_REF and $GITHUB_SHA" + else + # Push + git fetch origin ${{ github.event.before }} --depth=1 + export DIFF=$( git diff --name-only ${{ github.event.before }} $GITHUB_SHA ) + echo "Diff between ${{ github.event.before }} and $GITHUB_SHA" + fi + json="{\"include\":[" + for line in $DIFF; do + if [[ $line =~ environments ]]; then + env_name=$( basename ${line%/*} ) + if ! [[ $json =~ $env_name ]]; then + json="$json{\"environment\":\"$env_name\"}," + fi + fi + done + ### https://github.com/actions/runner/issues/2947 + echo "matrix=$( echo "${json%,}]}" | jq -r 'tostring' )" >> $GITHUB_OUTPUT diff --git a/.github/workflows/manual_trigger.yml b/.github/workflows/manual_trigger.yml new file mode 100644 index 0000000..71a800a --- /dev/null +++ b/.github/workflows/manual_trigger.yml @@ -0,0 +1,54 @@ +name: Force environment update +on: + workflow_dispatch: + inputs: + environment: + description: "Environment to update" + required: true + type: string + +jobs: + setup: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Sync repository to Gadi + uses: up9cloud/action-rsync@v1.3 + env: + HOST: gadi.nci.org.au + TARGET: ${{secrets.GADI_REPO_PATH}} + KEY: ${{secrets.DEPLOY_KEY}} + USER: ${{secrets.GADI_USER}} + - name: Create Admin dirs on Gadi + uses: appleboy/ssh-action@v0.1.7 + with: + host: gadi.nci.org.au + username: ${{secrets.GADI_USER}} + key: ${{secrets.DEPLOY_KEY}} + script: | + source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh + source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh + mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR + set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR + + build: + needs: setup + uses: ./.github/workflows/build.yml + with: + environment: ${{ inputs.environment }} + secrets: inherit + + test: + needs: build + uses: ./.github/workflows/test.yml + with: + environment: ${{ inputs.environment }} + secrets: inherit + + deploy: + needs: test + uses: ./.github/workflows/deploy.yml + with: + environment: ${{ inputs.environment }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..5b6f8b5 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,71 @@ +name: Build and test conda env +on: pull_request +jobs: + generate_matrix: + uses: ./.github/workflows/get_changed_env.yml + + setup: + runs-on: ubuntu-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }} + container: + image: quay.io/singularity/singularity:v3.11.4 + options: --privileged + steps: + - name: Checkout repository + ### Latest at time of writing + uses: actions/checkout@v4 + - name: Check if container definition has changed + id: changed-container-def + uses: tj-actions/changed-files@v41 + with: + files_yaml: | + containerdef: + - container/container.def + - name: Build container if definition has changed + if: steps.changed-container-def.outputs.containerdef_any_changed == 'true' + run: | + echo ${{ needs.generate_matrix.outputs.matrix }} + sudo -E singularity build container/base.sif container/container.def + - name: Sync repository to Gadi + ### Latest at time of writing + uses: up9cloud/action-rsync@v1.3 + env: + HOST: gadi.nci.org.au + TARGET: ${{secrets.GADI_REPO_PATH}} + KEY: ${{secrets.DEPLOY_KEY}} + USER: ${{secrets.GADI_USER}} + - name: Create Admin dirs on Gadi + uses: appleboy/ssh-action@v0.1.7 + with: + host: gadi.nci.org.au + username: ${{secrets.GADI_USER}} + key: ${{secrets.DEPLOY_KEY}} + script: | + source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh + source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh + mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR + set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR + + build: + needs: [ generate_matrix, setup ] + uses: ./.github/workflows/build.yml + with: + environment: ${{ matrix.environment }} + secrets: inherit + if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.matrix)}} + max-parallel: 1 + fail-fast: false + test: + needs: [ generate_matrix, build ] + uses: ./.github/workflows/test.yml + with: + environment: ${{ matrix.environment }} + secrets: inherit + if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.matrix)}} + max-parallel: 1 + fail-fast: false \ No newline at end of file diff --git a/.github/workflows/push_to_main.yml b/.github/workflows/push_to_main.yml new file mode 100644 index 0000000..49a7e3c --- /dev/null +++ b/.github/workflows/push_to_main.yml @@ -0,0 +1,35 @@ +name: Deploy conda env +on: + push: + branches: main +jobs: + generate_matrix: + uses: ./.github/workflows/get_changed_env.yml + + setup: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + ### Latest at time of writing + uses: actions/checkout@v4 + - name: Sync repository to Gadi + ### Latest at time of writing + uses: up9cloud/action-rsync@v1.3 + env: + HOST: gadi.nci.org.au + TARGET: ${{secrets.GADI_REPO_PATH}} + KEY: ${{secrets.DEPLOY_KEY}} + USER: ${{secrets.GADI_USER}} + + deploy: + needs: [ generate_matrix, setup ] + uses: ./.github/workflows/deploy.yml + with: + environment: ${{ matrix.environment }} + secrets: inherit + if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.matrix)}} + max-parallel: 1 + fail-fast: false + \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c6a65cc --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Test conda env +on: + workflow_call: + inputs: + environment: + required: true + type: string +jobs: + test: + name: Test environment + runs-on: ubuntu-latest + steps: + - name: Test environment + uses: appleboy/ssh-action@v0.1.7 + with: + host: gadi.nci.org.au + username: ${{secrets.GADI_USER}} + key: ${{secrets.DEPLOY_KEY}} + script: | + export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts + eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh ) + eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh ) + cd $JOB_LOG_DIR + qsub -N test_${{ inputs.environment }} -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR,CONDA_ENVIRONMENT=${{ inputs.environment }} -P kr06 -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/test.sh \ No newline at end of file diff --git a/environments/analysis3/build_inner.sh b/environments/analysis3/build_inner.sh new file mode 100644 index 0000000..32e9d5f --- /dev/null +++ b/environments/analysis3/build_inner.sh @@ -0,0 +1,8 @@ +### Custom install inner to build jupyter lab extensions + +set +u +eval "$( ${MAMBA} shell hook --shell bash)" +micromamba activate "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" +set -u + +jupyter lab build \ No newline at end of file diff --git a/environments/analysis3/config.sh b/environments/analysis3/config.sh new file mode 100644 index 0000000..fa9179d --- /dev/null +++ b/environments/analysis3/config.sh @@ -0,0 +1,23 @@ +### config.sh MUST provide the following: +### $ENVIRONMENT +### $FULLENV +### +### Arrays (can be empty) +### rpms_to_remove +### replace_from_apps +### outside_commands_to_include +### outside_files_to_copy + +### Optional config for custom deploy script +export VERSION_TO_MODIFY=24.01 +export STABLE_VERSION=23.10 +export UNSTABLE_VERSION=24.01 + +### Version settings +export ENVIRONMENT=analysis3 +export FULLENV="${ENVIRONMENT}-${VERSION_TO_MODIFY}" + +declare -a rpms_to_remove=( "openssh-clients" "openssh-server" "openssh" ) +declare -a replace_from_apps=( "openmpi/4.1.5" "ucx/1.14.0" ) +declare -a outside_commands_to_include=( "pbs_tmrsh" "ssh" ) +declare -a outside_files_to_copy=( "/g/data/hh5/public/apps/nci-intake-catalogue/catalogue_new.yaml" ) \ No newline at end of file diff --git a/environments/analysis3/deploy.sh b/environments/analysis3/deploy.sh new file mode 100644 index 0000000..b466cbc --- /dev/null +++ b/environments/analysis3/deploy.sh @@ -0,0 +1,18 @@ +### Update stable/unstable if necessary +CURRENT_STABLE=$( get_aliased_module "${MODULE_NAME}"/analysis "${CONDA_MODULE_PATH}" ) +NEXT_STABLE="${ENVIRONMENT}-${STABLE_VERSION}" +CURRENT_UNSTABLE=$( get_aliased_module "${MODULE_NAME}"/analysis3-unstable "${CONDA_MODULE_PATH}" ) +NEXT_UNSTABLE="${ENVIRONMENT}-${UNSTABLE_VERSION}" + +if ! [[ "${CURRENT_STABLE}" == "${MODULE_NAME}/${NEXT_STABLE}" ]]; then + echo "Updating stable environment to ${NEXT_STABLE}" + write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}" + symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}" "${NEXT_STABLE}" + symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}".d "${NEXT_STABLE}".d +fi +if ! [[ "${CURRENT_UNSTABLE}" == "${MODULE_NAME}/${NEXT_UNSTABLE}" ]]; then + echo "Updating unstable environment to ${NEXT_UNSTABLE}" + write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}" + symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}"-unstable "${NEXT_UNSTABLE}" + symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}"-unstable.d "${NEXT_UNSTABLE}".d +fi \ No newline at end of file diff --git a/scripts/environment.yml b/environments/analysis3/environment.yml similarity index 99% rename from scripts/environment.yml rename to environments/analysis3/environment.yml index fc1d07f..ccbbfc1 100644 --- a/scripts/environment.yml +++ b/environments/analysis3/environment.yml @@ -3,6 +3,7 @@ channels: - coecms - conda-forge - accessnri +- anaconda variables: CARTOPY_USER_BACKGROUNDS: /g/data/hh5/public/apps/cartopy-data/backgrounds dependencies: diff --git a/scripts/testconfig.yml b/environments/analysis3/testconfig.yml similarity index 81% rename from scripts/testconfig.yml rename to environments/analysis3/testconfig.yml index 4622167..b87af49 100644 --- a/scripts/testconfig.yml +++ b/environments/analysis3/testconfig.yml @@ -48,6 +48,7 @@ skip: - attrdict ### Deprecated but still erroneously bought in by wavespectra - not used by anything else - skimage.future.graph ### Thanks for raising a module error to tell me that you've moved this - numba.core.rvsdg_frontend ### Not supported in Python3.10 +- xgboost.testing ### Has an 'pytest.importorskip' call when imported, causing the whole thing to report skipped # Preload these modules before testing to avoid weird python issues preload: @@ -75,19 +76,6 @@ exception: - metpy.plots # While Issue #21 unresolved - httpx - sanic -- tensorflow.estimator -- tensorflow._api.v2.compat.v1.estimator -- tensorflow._api.v2.compat.v1.compat.v1.estimator -- tensorflow._api.v2.compat.v1.compat.v2.estimator -- tensorflow._api.v2.compat.v2.estimator -- tensorflow._api.v2.compat.v2.compat.v1.estimator -- tensorflow._api.v2.compat.v2.compat.v2.estimator -- tensorflow.compat.v1.estimator -- tensorflow.compat.v1.compat.v1.estimator -- tensorflow.compat.v1.compat.v2.estimator -- tensorflow.compat.v2.estimator -- tensorflow.compat.v2.compat.v1.estimator -- tensorflow.compat.v2.compat.v2.estimator - pyparsing.diagrams - pysal.explore.esda - pysal.explore.segregation diff --git a/modules/common_v3 b/modules/common_v3 index d86f348..04a58a2 100644 --- a/modules/common_v3 +++ b/modules/common_v3 @@ -6,7 +6,6 @@ set package __CONDA_INSTALL_BASENAME__ # Name of this module's environment set condaenv [lindex [split [module-info name] {/}] 1] set basedir "$prefix/$package/envs/$condaenv" -set rootdir [ string map [ list /envs/$condaenv {} ] $basedir ] set mymod [file normalize [info script]] set mydir [file dirname $mymod] @@ -17,7 +16,6 @@ set launcher $myscripts/launcher.sh module load singularity -prepend-path PATH $rootdir/condabin prepend-path CONTAINER_OVERLAY_PATH $overlay_path prepend-path SINGULARITYENV_PREPEND_PATH $basedir/condabin @@ -37,4 +35,6 @@ if { [ file exists $mydir/.$condaenv ] } { } ### Extra env to get other things working -setenv OMPI_MCA_orte_launch_agent $myscripts/orted \ No newline at end of file +setenv OMPI_MCA_orte_launch_agent $myscripts/orted +setenv MAMBA_ROOT_PREFIX $prefix/$package +setenv CONDA_EXE $prefix/$package/bin/micromamba \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh index 59658ec..1cc3954 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash +if [[ ! "${CONDA_ENVIRONMENT}" ]]; then + echo "Error! CONDA_ENVIRONMENT must be defined" + exit 1 +fi + set -eu [[ "${SCRIPT_DIR}" ]] && cd "${SCRIPT_DIR}" @@ -13,14 +18,23 @@ export ENV_INSTALLATION_PATH="${CONDA_TEMP_PATH}"/squashfs-root/opt/conda/"${FUL ### Derived installation paths export CONDA_INSTALLATION_PATH=${CONDA_INSTALLATION_PATH:-${CONDA_BASE}/./${APPS_SUBDIR}/${CONDA_INSTALL_BASENAME}} -export MAMBA="${CONDA_INSTALLATION_PATH}"/condabin/mamba +export MAMBA="${CONDA_INSTALLATION_PATH}"/bin/micromamba +export MAMBA_ROOT_PREFIX="${CONDA_INSTALLATION_PATH}" + +export ENV_FILE="${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/environment.yml +if ! [[ -e "${ENV_FILE}" ]]; then + echo "Error! Environment file for ${CONDA_ENVIRONMENT} not present!" + exit 1 +fi + +initialise_tmp_dirs .conda .mamba micromamba function inner() { - source "${CONDA_INSTALLATION_PATH}"/etc/profile.d/conda.sh ### Create the environment if [[ "${1}" == "--install" ]]; then - ${MAMBA} env create -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" -f environment.yml + ### Use --relocate-prefix to prevent micromamba helpfully resolving symlinks... + ${MAMBA} create -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" --relocate-prefix "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" -f "${ENV_FILE}" -y if [[ $? -ne 0 ]]; then echo "Error installing new environment" exit 1 @@ -28,8 +42,10 @@ function inner() { elif [[ "${1}" == "--update" ]]; then cat "${CONDA_INSTALLATION_PATH}"/envs/${FULLENV}/conda-meta/history >> "${CONDA_INSTALLATION_PATH}"/envs/${FULLENV}/conda-meta/history.log echo > "${CONDA_INSTALLATION_PATH}"/envs/${FULLENV}/conda-meta/history - conda env export -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" > deployed.old.yml - ${MAMBA} env update -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" -f environment.yml + ${MAMBA} env export -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" > deployed."${CONDA_ENVIRONMENT}".old.yml + ### micromamba forces this to be done in 2 steps - install for new packages, and update to check for updates + ${MAMBA} install -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" --relocate-prefix "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" -f "${ENV_FILE}" -y + ${MAMBA} update -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" --relocate-prefix "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" -f "${ENV_FILE}" -y if [[ $? -ne 0 ]]; then echo "Error updating new environment" exit 1 @@ -39,9 +55,9 @@ function inner() { rm -rf "${CONDA_SCRIPT_PATH}"/"${FULLENV}".d/{bin,overrides} fi - conda env export -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" > deployed.yml + ${MAMBA} env export -p "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" > deployed."${CONDA_ENVIRONMENT}".yml - if [[ "${1}" == "--update" ]] && diff -q deployed.yml deployed.old.yml; then + if [[ "${1}" == "--update" ]] && diff -q deployed."${CONDA_ENVIRONMENT}".yml deployed."${CONDA_ENVIRONMENT}".old.yml; then echo "No changes detected in the environment, discarding update" exit 0 fi @@ -70,7 +86,7 @@ function inner() { fn="${i//$apps_subdir\//}" [[ -e $fn ]] && rm $fn [[ "${fn}" != "${fn%/*}" ]] && mkdir -p "${fn%/*}" - ln -s "${i}" "${fn}" + ln -sf "${i}" "${fn}" done popd fi @@ -107,13 +123,11 @@ function inner() { done popd - set +u - conda activate "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" - set -u - - jupyter lab build + if [[ -e "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/build_inner.sh ]]; then + source "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/build_inner.sh + fi - conda clean -a -f -y + ${MAMBA} clean -a -f -y } @@ -148,7 +162,9 @@ if [[ -e "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}.sqsh" ]]; then rm "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/"${FULLENV}" export DO_UPDATE="--update" else - mkdir -p "${ENV_INSTALLATION_PATH}" + ### conda-meta subdirectory must be present to trick micromamba into + ### thinking that the directory we're making is a conda directory + mkdir -p "${ENV_INSTALLATION_PATH}/conda-meta" export DO_UPDATE="--install" fi @@ -181,9 +197,9 @@ if [[ -e "${CONTAINER_PATH}" ]]; then cp "${CONTAINER_PATH}" "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}" fi -if [[ "${DO_UPDATE}" == "--update" ]] && diff -q deployed.yml deployed.old.yml; then +if [[ "${DO_UPDATE}" == "--update" ]] && diff -q deployed."${CONDA_ENVIRONMENT}".yml deployed."${CONDA_ENVIRONMENT}".old.yml; then echo "No changes detected in the environment, discarding update" - cp deployed.yml deployed.old.yml "${BUILD_STAGE_DIR}"/ + cp deployed."${CONDA_ENVIRONMENT}".yml deployed."${CONDA_ENVIRONMENT}".old.yml "${BUILD_STAGE_DIR}"/ exit 0 fi @@ -203,22 +219,26 @@ cp "${FULLENV}".sqsh "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp set_apps_perms "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp popd -### Can't use ${CONDA_SCRIPT_PATH} or "${CONDA_INSTALLATION_PATH}" due to the need to string match on those paths -### which they won't with the '/./' part required for arcane rsync magic -construct_module_insert "${SINGULARITY_BINARY_PATH}" "${OVERLAY_BASE}" "${my_container}" "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp "${SCRIPT_DIR}"/condaenv.sh "${CONDA_INSTALLATION_PATH}" "${CONDA_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/"${FULLENV}" "${CONDA_BASE}"/"${SCRIPT_SUBDIR}"/"${FULLENV}".d/bin "${CONDA_OUTER_BASE}"/"${MODULE_SUBDIR}"/"${MODULE_NAME}"/."${FULLENV}" +if [[ -e "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/build_outer.sh ]]; then + source "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/build_outer.sh +fi rm "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/"${FULLENV}" ln -s /opt/conda/"${FULLENV}" "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/ +### Can't use ${CONDA_SCRIPT_PATH} or "${CONDA_INSTALLATION_PATH}" due to the need to string match on those paths +### which they won't with the '/./' part required for arcane rsync magic +construct_module_insert "${SINGULARITY_BINARY_PATH}" "${OVERLAY_BASE}" "${my_container}" "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp "${SCRIPT_DIR}"/condaenv.sh "${CONDA_INSTALLATION_PATH}" /opt/conda/"${FULLENV}" "${CONDA_BASE}"/"${SCRIPT_SUBDIR}"/"${FULLENV}".d/bin "${CONDA_OUTER_BASE}"/"${MODULE_SUBDIR}"/"${MODULE_NAME}"/."${FULLENV}" + ### Set permissions on base environment set_apps_perms "${CONDA_OUTER_BASE}" ### Archive base env pushd "${CONDA_OUTER_BASE}" ### WARNING: Non-standard tar extension: --acls -tar --acls -cf "${BUILD_STAGE_DIR}"/conda_base.tar "${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}" "${MODULE_SUBDIR}" "${SCRIPT_SUBDIR}" +tar --acls -cf "${BUILD_STAGE_DIR}"/conda_base."${CONDA_ENVIRONMENT}".tar "${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}" "${MODULE_SUBDIR}" "${SCRIPT_SUBDIR}" popd -cp deployed.yml "${BUILD_STAGE_DIR}"/ +cp deployed."${CONDA_ENVIRONMENT}".yml "${BUILD_STAGE_DIR}"/ ### || true so the script doesn't report failed if its doing a fresh install. -[[ "${DO_UPDATE}" == "--update" ]] && cp deployed.old.yml "${BUILD_STAGE_DIR}"/ || true +[[ "${DO_UPDATE}" == "--update" ]] && cp deployed."${CONDA_ENVIRONMENT}".old.yml "${BUILD_STAGE_DIR}"/ || true diff --git a/scripts/condaenv.sh b/scripts/condaenv.sh index 88eabd3..ac50eca 100755 --- a/scripts/condaenv.sh +++ b/scripts/condaenv.sh @@ -1,7 +1,7 @@ #!/bin/bash # Prints the environment variables set by 'conda activate $1', for processing by the modules export PATH=/usr/bin:/bin -source "${1}"/etc/profile.d/conda.sh -conda activate "${2}" +eval "$( "${1}"/bin/micromamba shell hook -s bash )" +micromamba activate "${2}" /bin/env alias \ No newline at end of file diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d8ac8d2..be5ff94 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +if [[ ! "${CONDA_ENVIRONMENT}" ]]; then + echo "Error! CONDA_ENVIRONMENT must be defined" + exit 1 +fi set -eu [[ "${SCRIPT_DIR}" ]] && cd "${SCRIPT_DIR}" @@ -12,16 +16,16 @@ source functions.sh export CONDA_INSTALLATION_PATH=${CONDA_INSTALLATION_PATH:-${CONDA_BASE}/./${APPS_SUBDIR}/${CONDA_INSTALL_BASENAME}} ### Do not package conda_base.tar or the updated env if there was no change -if diff -q "${BUILD_STAGE_DIR}"/deployed.yml "${BUILD_STAGE_DIR}"/deployed.old.yml; then +if diff -q "${BUILD_STAGE_DIR}"/deployed."${CONDA_ENVIRONMENT}".yml "${BUILD_STAGE_DIR}"/deployed."${CONDA_ENVIRONMENT}".old.yml; then echo "No changes detected in the environment, not deploying" - rm -f "${BUILD_STAGE_DIR}"/deployed.yml "${BUILD_STAGE_DIR}"/deployed.old.yml + rm -f "${BUILD_STAGE_DIR}"/deployed."${CONDA_ENVIRONMENT}".yml "${BUILD_STAGE_DIR}"/deployed."${CONDA_ENVIRONMENT}".old.yml exit fi mkdir -p "${CONDA_TEMP_PATH}" pushd "${CONDA_TEMP_PATH}" ### WARNING: Non-standard tar extension: --acls -tar --acls -xf "${BUILD_STAGE_DIR}"/conda_base.tar +tar --acls -xf "${BUILD_STAGE_DIR}"/conda_base."${CONDA_ENVIRONMENT}".tar popd ### rsync --archive attempts to set permissions on @@ -30,36 +34,18 @@ set +e echo "Sync across any changes in the base conda environment" rsync --archive --verbose --partial --progress --one-file-system --itemize-changes --hard-links --acls --relative -- "${CONDA_TEMP_PATH}"/./"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}" "${CONDA_TEMP_PATH}"/./"${MODULE_SUBDIR}" "${CONDA_TEMP_PATH}"/./"${SCRIPT_SUBDIR}" "${CONDA_BASE}" -echo "Make sure anything deleted from the base conda environment is also deleted in the prod copy" -rsync --archive --verbose --partial --progress --one-file-system --itemize-changes --hard-links --acls --relative --delete --exclude=*.sqsh -- "${CONDA_TEMP_PATH}"/./"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}" "${CONDA_BASE}" - -echo "Make sure anything deleted from the scripts directory is also deleted from the prod copy" -rsync --archive --verbose --partial --progress --one-file-system --itemize-changes --hard-links --acls --relative --delete -- "${CONDA_TEMP_PATH}"/./"${SCRIPT_SUBDIR}" "${CONDA_BASE}" +echo "Make sure anything deleted from this environments scripts directory is also deleted from the prod copy" +rsync --archive --verbose --partial --progress --one-file-system --itemize-changes --hard-links --acls --relative --delete -- "${CONDA_TEMP_PATH}"/./"${SCRIPT_SUBDIR}"/"${CONDA_ENVIRONMENT}".d "${CONDA_BASE}" set -e [[ -e "${CONDA_INSTALLATION_PATH}"/envs/"${FULLENV}".sqsh ]] && cp "${CONDA_INSTALLATION_PATH}"/envs/"${FULLENV}".sqsh "${ADMIN_DIR}"/"${FULLENV}".sqsh.bak mv "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp "${CONDA_INSTALLATION_PATH}"/envs/"${FULLENV}".sqsh -### Update stable/unstable if necessary -CURRENT_STABLE=$( get_aliased_module "${MODULE_NAME}"/analysis "${CONDA_MODULE_PATH}" ) -NEXT_STABLE="${ENVIRONMENT}-${STABLE_VERSION}" -CURRENT_UNSTABLE=$( get_aliased_module "${MODULE_NAME}"/analysis3-unstable "${CONDA_MODULE_PATH}" ) -NEXT_UNSTABLE="${ENVIRONMENT}-${UNSTABLE_VERSION}" - -if ! [[ "${CURRENT_STABLE}" == "${MODULE_NAME}/${NEXT_STABLE}" ]]; then - echo "Updating stable environment to ${NEXT_STABLE}" - write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}" - symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}" "${NEXT_STABLE}" - symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}".d "${NEXT_STABLE}".d -fi -if ! [[ "${CURRENT_UNSTABLE}" == "${MODULE_NAME}/${NEXT_UNSTABLE}" ]]; then - echo "Updating unstable environment to ${NEXT_UNSTABLE}" - write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}" - symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}"-unstable "${NEXT_UNSTABLE}" - symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}"-unstable.d "${NEXT_UNSTABLE}".d -fi - ### Overwrite existing conda_base tarball -mv "${BUILD_STAGE_DIR}"/conda_base.tar "${ADMIN_DIR}" +mv "${BUILD_STAGE_DIR}"/conda_base."${CONDA_ENVIRONMENT}".tar "${ADMIN_DIR}" ### Remove staging artefacts -rm -f "${BUILD_STAGE_DIR}"/deployed.yml "${BUILD_STAGE_DIR}"/deployed.old.yml "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp \ No newline at end of file +rm -f "${BUILD_STAGE_DIR}"/deployed."${CONDA_ENVIRONMENT}".yml "${BUILD_STAGE_DIR}"/deployed."${CONDA_ENVIRONMENT}".old.yml "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp + +if [[ -e "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/deploy.sh ]]; then + source "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/deploy.sh +fi diff --git a/scripts/dev_prep.sh b/scripts/dev_prep.sh index 0de36a1..88b9067 100644 --- a/scripts/dev_prep.sh +++ b/scripts/dev_prep.sh @@ -107,6 +107,10 @@ if [[ ! "${PBS_JOBFS}" ]]; then echo "Must be run inside a PBS job" && return fi +if [[ ! "${CONDA_ENVIRONMENT}" ]]; then + echo "Error! CONDA_ENVIRONMENT must be defined" && return +fi + _parse_jobfs() { ### PBS_NCI_JOBFS var takes the form of nnnnnnXb multiplier=1 diff --git a/scripts/functions.sh b/scripts/functions.sh index 3ac96b7..7dd368a 100644 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -117,11 +117,14 @@ function construct_module_insert() { script_path="${8}" module_path="${9}" + declare -a discard_paths=( "/bin" "/usr/bin" "/condabin" ) + declare -a discard_vars=( "MODULEPATH" "_" "PWD" "SHLVL" ) + while read line; do key="${line%%=*}" value="${line#*=}" ### Skip these environment variables - in_array "MODULEPATH" "_" "PWD" "SHLVL" "${key}" && continue + in_array "${discard_vars[@]}" "${key}" && continue ### Prepend to these variables if [[ $key =~ .PATH$ ]]; then echo prepend-path $key $value @@ -131,7 +134,7 @@ function construct_module_insert() { ### Treat path specially - remove system paths and retain order elif [[ "${key}" == "PATH" ]]; then while IFS= read -r -d: entry; do - in_array "/bin" "/usr/bin" "${entry}" && continue + in_array "${discard_paths[@]}" "${entry}" && continue if [[ $entry =~ $condaenv ]]; then echo prepend-path PATH $script_path else @@ -197,4 +200,19 @@ function copy_and_replace_if_changed() { mv "${out}" "${final_out}" fi +} + +function initialise_tmp_dirs() { + + if [[ "${PBS_JOBFS}" ]]; then + relink_cmds="" + for dir in "$@"; do + relink_cmds="${relink_cmds}rm ~/${dir}; ln -s $( readlink ~/${dir} ) ~/${dir}; " + rm ~/"${dir}" + mkdir -p "${PBS_JOBFS}"/"${dir}" + ln -s "${PBS_JOBFS}"/"${dir}" ~ + ### Race condition + done + trap "${relink_cmds}" EXIT + fi } \ No newline at end of file diff --git a/scripts/initialise.sh b/scripts/initialise.sh index 8c80d84..27709ef 100755 --- a/scripts/initialise.sh +++ b/scripts/initialise.sh @@ -15,11 +15,12 @@ export CONDA_INSTALLATION_PATH="${CONDA_INSTALLATION_PATH:-$CONDA_BASE/$APPS_SUB function inner() { - mkdir -p "${CONDA_INSTALLATION_PATH%/*}" - bash Miniconda3-py39_4.12.0-Linux-x86_64.sh -b -p "${CONDA_INSTALLATION_PATH}" - - . "${CONDA_INSTALLATION_PATH}"/etc/profile.d/conda.sh - conda install mamba -y + mkdir -p "${CONDA_INSTALLATION_PATH}" + pushd "${CONDA_INSTALLATION_PATH}" + #curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba + ### Modified micromamba for compatibility with nb_conda_kernels + curl -Ls https://dsroberts.github.io/mamba/latest | tar -xvj bin/micromamba + popd mkdir -p "${CONDA_SCRIPT_PATH}"/overrides cp "${SCRIPT_DIR}"/launcher.sh "${CONDA_SCRIPT_PATH}" @@ -30,8 +31,6 @@ function inner() { copy_and_replace "${SCRIPT_DIR}"/../modules/common_v3 "${CONDA_MODULE_PATH}"/.common_v3 CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME SCRIPT_SUBDIR copy_and_replace "${SCRIPT_DIR}"/launcher_conf.sh "${CONDA_SCRIPT_PATH}"/launcher_conf.sh CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME - conda clean -a -f -y - } if [[ $# -gt 0 ]]; then @@ -41,27 +40,19 @@ if [[ $# -gt 0 ]]; then fi fi -wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh - mkdir -p "${OVERLAY_BASE}" "${SINGULARITY_BINARY_PATH}" -s exec --bind /etc,/half-root,/local,/ram,/run,/system,/usr,/var/lib/sss,/var/run/munge,/var/lib/rpm,"${OVERLAY_BASE}":/g "${CONTAINER_PATH}" $( realpath $0 ) --inner ### Copy in container +mkdir -p "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/ cp "${CONTAINER_PATH}" "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/ ### Set permissions set_apps_perms "${CONDA_OUTER_BASE}" -#rsync --archive --verbose --partial --progress --one-file-system --hard-links --acls --relative -- "${CONDA_OUTER_BASE}"/./"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}" "${CONDA_OUTER_BASE}"/./"${SCRIPT_SUBDIR}" "${CONDA_OUTER_BASE}"/./"${MODULE_SUBDIR}" "${BUILD_STAGE_DIR}" - -### To be made by jenkins -#mkdir -p "${ADMIN_DIR}" -#chgrp "${APPS_OWNERS_GROUP}" "${ADMIN_DIR}" -#chmod g=u+s,o= "${ADMIN_DIR}" - +### Create necessary directories: +mkdir -p "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/ pushd "${CONDA_OUTER_BASE}" ### WARNING: Non-standard tar extension: --acls tar --acls -cf "${BUILD_STAGE_DIR}"/conda_base.tar "${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}" "${SCRIPT_SUBDIR}" "${MODULE_SUBDIR}" popd - -rm -f Miniconda3-py39_4.12.0-Linux-x86_64.sh diff --git a/scripts/install_config.sh b/scripts/install_config.sh index dd87ccc..4d3f3e2 100644 --- a/scripts/install_config.sh +++ b/scripts/install_config.sh @@ -20,22 +20,19 @@ export BUILD_STAGE_DIR="${ADMIN_DIR}"/staging export APPS_USERS_GROUP=hh5 export APPS_OWNERS_GROUP=hh5_w -### Version settings -export ENVIRONMENT=analysis3 -export VERSION_TO_MODIFY=23.10 -export STABLE_VERSION=23.07 -export UNSTABLE_VERSION=23.10 -export FULLENV="${ENVIRONMENT}-${VERSION_TO_MODIFY}" - ### Other settings export TEST_OUT_FILE=test_results.xml export PYTHONNOUSERSITE=true export CONTAINER_PATH="${SCRIPT_DIR}"/../container/base.sif export SINGULARITY_BINARY_PATH="/opt/singularity/bin/singularity" -declare -a rpms_to_remove=( "openssh-clients" "openssh-server" "openssh" ) -declare -a replace_from_apps=( "openmpi/4.1.5" "ucx/1.14.0" ) -declare -a outside_commands_to_include=( "pbs_tmrsh" "ssh" ) -declare -a outside_files_to_copy=( "/g/data/hh5/public/apps/nci-intake-catalogue/catalogue_new.yaml" ) - declare -a bind_dirs=( "/etc" "/half-root" "/local" "/ram" "/run" "/system" "/usr" "/var/lib/sss" "/var/lib/rpm" "/var/run/munge" "/sys/fs/cgroup" "/iointensive" ) + +if [[ "${CONDA_ENVIRONMENT}" ]]; then + if [[ -e "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/config.sh ]]; then + source "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/config.sh + else + echo "ERROR! ${CONDA_ENVIRONMENT} config file missing!" + exit 1 + fi +fi \ No newline at end of file diff --git a/scripts/launcher.sh b/scripts/launcher.sh index 06bea13..4a5cb1c 100755 --- a/scripts/launcher.sh +++ b/scripts/launcher.sh @@ -11,8 +11,21 @@ function in_array() { return 1 } -wrapper_bin=$( realpath "${0%/*}" ) +function debug_print() { + echo "$@" 1>&2 +} + +if [[ "${CMS_CONDA_DEBUG_SCRIPTS}" ]]; then + debug=debug_print +else + debug=true +fi + +wrapper_path=$( realpath "${0}" ) +wrapper_bin=${wrapper_path%/*} +$debug "wrapper_bin = " "${wrapper_bin}" conf_file="${wrapper_bin}"/launcher_conf.sh +$debug "conf_file = " "${conf_file}" source "${conf_file}" @@ -23,25 +36,30 @@ while [[ $# -gt 0 ]]; do "--cms_singularity_overlay_path_override") ### Sometimes we do not want to use the 'correct' container export CONTAINER_OVERLAY_PATH_OVERRIDE=1 + $debug "cms_singularity_overlay_path_override=1" shift ;; "--cms_singularity_overlay_path") ### From time to time we need to manually specify an overlay filesystem, handle that here: export CONTAINER_OVERLAY_PATH="${2}" + $debug "cms_singularity_overlay_path="${CONTAINER_OVERLAY_PATH} shift 2 ;; "--cms_singularity_in_container_path") ### Set path manually export PATH="${2}" + $debug "cms_singularity_in_container_path="${PATH} shift 2 ;; "--cms_singularity_launcher_override") ### Override the launcher script name export LAUNCHER_SCRIPT="${2}" + $debug "cms_singularity_launcher_override="${LAUNCHER_SCRIPT} shift 2 ;; "--cms_singularity_singularity_path") export SINGULARITY_BINARY_PATH="${2}" + $debug "cms_singularity_singularity_path="${SINGULARITY_BINARY_PATH} shift 2 ;; *) @@ -51,6 +69,8 @@ while [[ $# -gt 0 ]]; do esac done +$debug "PROG_ARGS =" "${PROG_ARGS[@]}" + if ! [[ "${SINGULARITY_BINARY_PATH}" ]]; then module load singularity export SINGULARITY_BINARY_PATH=$( type -p singularity ) @@ -67,6 +87,8 @@ else cmd_to_run+=( "${PROG_ARGS[@]}" ) fi +$debug "cmd_to_run = " "${cmd_to_run[@]}" + ### Handle the case where we've been invoked directly. Make sure the container ### we need is on path, and that CONDA_BASE is set so that the right thing ### runs in the container. If we haven't been directly invoked, this does @@ -77,8 +99,13 @@ fi ### somewhere else (e.g. jobfs), the one on gdata will be mounted but not used. myenv=$( basename "${wrapper_bin%/*}" ".d" ) if ! [[ "${CONTAINER_OVERLAY_PATH_OVERRIDE}" ]]; then - [[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh: ]] || export CONTAINER_OVERLAY_PATH="${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh:${CONTAINER_OVERLAY_PATH} + if ! [[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh: ]]; then + [[ -r "${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh ]] && export CONTAINER_OVERLAY_PATH="${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh:${CONTAINER_OVERLAY_PATH} + fi fi + +$debug "CONTAINER_OVERLAY_PATH after override check = " ${CONTAINER_OVERLAY_PATH} + export CONDA_BASE="${CONDA_BASE_ENV_PATH}/envs/${myenv}" if ! [[ -x "${SINGULARITY_BINARY_PATH}" ]]; then @@ -90,16 +117,19 @@ if ! [[ -x "${SINGULARITY_BINARY_PATH}" ]]; then ### bin directory of the active conda env, so just reset the path to that but keep the ### original argv[0] so virtual envs work. cmd_to_run[0]="${CONDA_BASE}/bin/${cmd_to_run[0]##*/}" + $debug "Short circuit detected, running: " "exec -a" "${0}" "${cmd_to_run[@]}" exec -a "${0}" "${cmd_to_run[@]}" fi ### Handle some functions separately. if [[ -e ${wrapper_bin}/../overrides/"${cmd_to_run[0]##*/}".sh ]]; then + $debug "Running override function: " ${wrapper_bin}/../overrides/"${cmd_to_run[0]##*/}".sh "${cmd_to_run[@]:1}" exec ${wrapper_bin}/../overrides/"${cmd_to_run[0]##*/}".sh "${cmd_to_run[@]:1}" fi ### Add some additional config for some functions if [[ -e "${wrapper_bin}"/../overrides/"${cmd_to_run[0]##*/}".config.sh ]]; then + $debug "Loading additional configuration: " "${wrapper_bin}"/../overrides/"${cmd_to_run[0]##*/}".config.sh . "${wrapper_bin}"/../overrides/"${cmd_to_run[0]##*/}".config.sh fi @@ -114,15 +144,22 @@ while IFS= read -r -d: i; do done<<<"${PATH%:}:" export SINGULARITYENV_PREPEND_PATH=${SINGULARITYENV_PREPEND_PATH#:*} +$debug "SINGULARITYENV_PREPEND_PATH= " ${SINGULARITYENV_PREPEND_PATH} + overlay_args="" while IFS= read -r -d: i; do overlay_args="${overlay_args}--overlay=${i} " done<<<"${CONTAINER_OVERLAY_PATH%:}:" +$debug "overlay_args= " ${overlay_args} + bind_str="" for bind_dir in "${bind_dirs[@]}"; do [[ -d "${bind_dir}" ]] && bind_str="${bind_str}${bind_dir}," done bind_str=${bind_str%,} +$debug "binding args= " ${bind_str} + +$debug "Singularity invocation: " "$SINGULARITY_BINARY_PATH" -s exec --bind "${bind_str}" ${overlay_args} "${CONTAINER_PATH}" "${cmd_to_run[@]}" "$SINGULARITY_BINARY_PATH" -s exec --bind "${bind_str}" ${overlay_args} "${CONTAINER_PATH}" "${cmd_to_run[@]}" diff --git a/scripts/launcher_conf.sh b/scripts/launcher_conf.sh index e1b7758..17f7040 100644 --- a/scripts/launcher_conf.sh +++ b/scripts/launcher_conf.sh @@ -2,7 +2,7 @@ export SINGULARITY_BINARY_PATH="/opt/singularity/bin/singularity" export CONTAINER_PATH="__CONDA_BASE__/__APPS_SUBDIR__/__CONDA_INSTALL_BASENAME__/etc/base.sif" if [[ "${CONDA_EXE}" ]]; then - export CONDA_BASE_ENV_PATH="${CONDA_EXE//\/bin\/conda/}" + export CONDA_BASE_ENV_PATH="${CONDA_EXE//\/bin\/micromamba/}" else export CONDA_BASE_ENV_PATH="__CONDA_BASE__/__APPS_SUBDIR__/__CONDA_INSTALL_BASENAME__" fi diff --git a/scripts/test.sh b/scripts/test.sh index 8555d00..b5ce387 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,8 +1,11 @@ #!/usr/bin/env bash +if [[ ! "${CONDA_ENVIRONMENT}" ]]; then + echo "Error! CONDA_ENVIRONMENT must be defined" + exit 1 +fi ### Not using set -eu in this script - we do not rely on the exit status ### to determine if the tests have failed. Success or otherwise of ### other commands in this script is immaterial - [[ "${SCRIPT_DIR}" ]] && cd "${SCRIPT_DIR}" source install_config.sh @@ -14,24 +17,32 @@ export CONDA_OUTER_BASE="${OVERLAY_BASE}"/"${CONDA_BASE#/*/}" export CONDA_INSTALLATION_PATH=${CONDA_INSTALLATION_PATH:-${CONDA_BASE}/./${APPS_SUBDIR}/${CONDA_INSTALL_BASENAME}} function inner() { - - source "${CONDA_INSTALLATION_PATH}"/etc/profile.d/conda.sh + if [[ -e "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/test_inner.sh ]]; then + source "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/test_inner.sh + fi + set +u - conda activate "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" + eval "$( "${CONDA_INSTALLATION_PATH}"/bin/micromamba shell hook --shell bash)" + micromamba activate "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" set -u ### For reasons I can't figure out, py.test hangs on exit due to not being able to ### clean up one of its threads when run in singularity. To get around this, background ### py.test and read its status from an output file, rather than using the exit status - rm -f "${TEST_OUT_FILE}" - py.test -s --junitxml "${TEST_OUT_FILE}" & - test_pid=$! - - while ! [[ -e "${TEST_OUT_FILE}" ]]; do sleep 5; done - - [[ -e /proc/"${test_pid}" ]] && kill -15 "${test_pid}" - wait + if command -v py.test; then + rm -f "${TEST_OUT_FILE}" + py.test -s --junitxml "${TEST_OUT_FILE}" & + test_pid=$! + + while ! [[ -e "${TEST_OUT_FILE}" ]]; do sleep 5; done + [[ -e /proc/"${test_pid}" ]] && kill -15 "${test_pid}" + wait + else + echo "py.test not present - skipping" + ### Make some fake test data + echo '' > "${TEST_OUT_FILE}" + fi } @@ -41,14 +52,14 @@ if [[ "${1}" == '--inner' ]]; then fi ### Do not package conda_base.tar or the updated env if there was no change -if diff -q "${BUILD_STAGE_DIR}"/deployed.yml "${BUILD_STAGE_DIR}"/deployed.old.yml; then +if diff -q "${BUILD_STAGE_DIR}"/deployed."${CONDA_ENVIRONMENT}".yml "${BUILD_STAGE_DIR}"/deployed."${CONDA_ENVIRONMENT}".old.yml; then echo "No changes detected in the environment, not performing tests" exit fi mkdir -p "${CONDA_OUTER_BASE}" pushd "${CONDA_OUTER_BASE}" -tar -xf "${BUILD_STAGE_DIR}"/conda_base.tar +tar -xf "${BUILD_STAGE_DIR}"/conda_base."${CONDA_ENVIRONMENT}".tar popd ### Copy in any files outside the conda directory tree that may be needed @@ -78,6 +89,10 @@ if [[ ! -e "${TEST_OUT_FILE}" ]]; then exit 1 fi +if [[ -e "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/test_outer.sh ]]; then + source "${SCRIPT_DIR}"/../environments/"${CONDA_ENVIRONMENT}"/test_outer.sh +fi + read errors failures < <( python3 -c 'import xml.etree.ElementTree as ET; import sys; t=ET.parse(sys.argv[1]); print(t.getroot().getchildren()[0].get("errors") + " " + t.getroot().getchildren()[0].get("failures"))' "${TEST_OUT_FILE}" ) if [[ "${errors}" -gt 0 ]] || [[ "${failures}" -gt 0 ]]; then diff --git a/scripts/test_environment.py b/scripts/test_environment.py index 224bc96..1e2630e 100644 --- a/scripts/test_environment.py +++ b/scripts/test_environment.py @@ -95,7 +95,7 @@ def import_config(filename): except IOError as exc: if exc.errno == errno.ENOENT: print('Warning: config file {0} not found!' - .format(config_fname)) + .format(filename)) config = {} else: raise @@ -119,7 +119,7 @@ def handle_error(name): def test_walk_packages(): global exception - config = import_config('testconfig.yml') + config = import_config('../environments/'+ os.environ['CONDA_ENVIRONMENT'] + '/testconfig.yml') for mod in config.get('preload',[]): try: if debug: print("Preloading {}".format(mod)) @@ -127,8 +127,8 @@ def test_walk_packages(): except: print("Error preloading {}".format(mod)) - exception = set(config.get('exception',None)) - skip = set(config.get('skip',None)) + exception = set(config.get('exception',[])) + skip = set(config.get('skip',[])) if debug: print('Exception: {}'.format(exception)) if debug: print('Skip: {}'.format(skip)) for importer, name, ispkg in my_walk_packages(onerror=handle_error,skip=skip): @@ -139,7 +139,7 @@ def test_walk_packages(): warnings.warn(UserWarning(warnstring)) def test_import(): - config = import_config('testconfig.yml') + config = import_config('../environments/'+ os.environ['CONDA_ENVIRONMENT'] + '/testconfig.yml') for info in iter_modules(): if info.ispkg and info.name not in config['skip']: From 1042c745b582a03ce4edcf93ecf395c34df31dbe Mon Sep 17 00:00:00 2001 From: Dale Roberts Date: Wed, 17 Jan 2024 13:37:09 +1100 Subject: [PATCH 2/4] Make arrays optional in environment config files --- container/container.def | 1 + environments/analysis3/config.sh | 3 +-- scripts/install_config.sh | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/container/container.def b/container/container.def index 4589633..f97cb0c 100644 --- a/container/container.def +++ b/container/container.def @@ -44,6 +44,7 @@ Bootstrap: scratch mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.04 mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.07 mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.10 + mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-24.01 %runscript /usr/bin/bash -l \ No newline at end of file diff --git a/environments/analysis3/config.sh b/environments/analysis3/config.sh index fa9179d..b568db6 100644 --- a/environments/analysis3/config.sh +++ b/environments/analysis3/config.sh @@ -1,8 +1,7 @@ ### config.sh MUST provide the following: -### $ENVIRONMENT ### $FULLENV ### -### Arrays (can be empty) +### Arrays used by the build system (optional, can be empty) ### rpms_to_remove ### replace_from_apps ### outside_commands_to_include diff --git a/scripts/install_config.sh b/scripts/install_config.sh index 4d3f3e2..bbc629e 100644 --- a/scripts/install_config.sh +++ b/scripts/install_config.sh @@ -35,4 +35,10 @@ if [[ "${CONDA_ENVIRONMENT}" ]]; then echo "ERROR! ${CONDA_ENVIRONMENT} config file missing!" exit 1 fi -fi \ No newline at end of file +fi + +### Define any undefined arrays +[[ -z ${rpms_to_remove+x} ]] && declare -a rpms_to_remove=() || true +[[ -z ${replace_from_apps+x} ]] && declare -a replace_from_apps=() || true +[[ -z ${outside_commands_to_include+x} ]] && declare -a outside_commands_to_include=() || true +[[ -z ${outside_files_to_copy+x} ]] && declare -a outside_files_to_copy=() || true From 1a09c2611720221cc093b063ec59016ebb3694f0 Mon Sep 17 00:00:00 2001 From: Dale Roberts Date: Tue, 30 Jan 2024 13:23:32 +1100 Subject: [PATCH 3/4] Add activate script --- scripts/initialise.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/initialise.sh b/scripts/initialise.sh index 27709ef..0cd157e 100755 --- a/scripts/initialise.sh +++ b/scripts/initialise.sh @@ -19,7 +19,7 @@ function inner() { pushd "${CONDA_INSTALLATION_PATH}" #curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba ### Modified micromamba for compatibility with nb_conda_kernels - curl -Ls https://dsroberts.github.io/mamba/latest | tar -xvj bin/micromamba + curl -Ls https://dsroberts.github.io/mamba/latest | tar -xvj bin/micromamba bin/activate popd mkdir -p "${CONDA_SCRIPT_PATH}"/overrides From 94838378c01b07ece388f7d98925c154ea5d8519 Mon Sep 17 00:00:00 2001 From: Dale Roberts Date: Wed, 31 Jan 2024 21:23:20 +1100 Subject: [PATCH 4/4] Set versions. Really kill tests --- environments/analysis3/config.sh | 4 ++-- scripts/test.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environments/analysis3/config.sh b/environments/analysis3/config.sh index b568db6..d02ece0 100644 --- a/environments/analysis3/config.sh +++ b/environments/analysis3/config.sh @@ -9,8 +9,8 @@ ### Optional config for custom deploy script export VERSION_TO_MODIFY=24.01 -export STABLE_VERSION=23.10 -export UNSTABLE_VERSION=24.01 +export STABLE_VERSION=23.07 +export UNSTABLE_VERSION=23.10 ### Version settings export ENVIRONMENT=analysis3 diff --git a/scripts/test.sh b/scripts/test.sh index b5ce387..429fb90 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -36,7 +36,7 @@ function inner() { test_pid=$! while ! [[ -e "${TEST_OUT_FILE}" ]]; do sleep 5; done - [[ -e /proc/"${test_pid}" ]] && kill -15 "${test_pid}" + [[ -e /proc/"${test_pid}" ]] && kill -9 "${test_pid}" wait else echo "py.test not present - skipping"