diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 7cfccba..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,220 +0,0 @@ ---- -workflow: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_PIPELINE_SOURCE == "web" -.rules: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_PIPELINE_SOURCE == "web" -.no-merge-request-rules: - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: never - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "web" - -stages: - - unit tests - - generate - - build base containers - - build cache - - build spacktainer containers - - cleanup - -variables: - SPACK_BRANCH: develop - -generate base pipeline: - image: ubuntu:22.04 - stage: generate - needs: [] - variables: - SINGULARITY_VERSION: 4.0.2 - S3CMD_VERSION: 2.3.0 - script: - - apt-get update && apt-get install -y ca-certificates git podman python3 python3-pip - skopeo - - pip install --upgrade pip setuptools - - pip install -e ./job_creator - - jc create-jobs --singularity-version ${SINGULARITY_VERSION} --s3cmd-version ${S3CMD_VERSION} - artifacts: - when: always - paths: - - generated_pipeline.yaml - - spacktainer.yaml - - merged_spack*yaml - - job_creator.log - - spacktainer - extends: .rules -base containers and pipeline generation: - stage: build base containers - needs: - - job: generate base pipeline - artifacts: true - trigger: - include: - - artifact: generated_pipeline.yaml - job: generate base pipeline - strategy: depend - extends: .rules -gather child artifacts: - stage: build cache - needs: [base containers and pipeline generation] - image: ubuntu:22.04 - script: - - apt-get update && apt-get install -y ca-certificates git python3 python3-pip - unzip - - pip install --upgrade pip setuptools - - pip install -e ./get_artifacts - - ga -P ${CI_PIPELINE_ID} -t ${GITLAB_API_TOKEN} - - for zipfile in spack.artifacts*zip; do - - unzip ${zipfile} -d ${zipfile%%.zip} - - rm -f ${zipfile} - - architecture=$(echo ${zipfile##spack.artifacts.} | sed 's/.zip//') - - echo ${architecture}=yes >> build.env - - done - - for zipfile in spacktainer.artifacts*zip; do - - unzip ${zipfile} -d ${zipfile%%.zip} - - rm -f ${zipfile} - - done - - cat build.env - artifacts: - when: always - paths: [spack.artifacts.*, spacktainer.artifacts.*] - reports: - dotenv: build.env - extends: .rules -populate buildcache for amd64: - stage: build cache - needs: - - job: gather child artifacts - artifacts: true - trigger: - include: - - artifact: spack.artifacts.amd64/spack_pipeline.yaml - job: gather child artifacts - strategy: depend - extends: .rules -populate buildcache for arm64: - rules: - - exists: - - container_definitions/arm64 - stage: build cache - needs: - - job: gather child artifacts - artifacts: true - trigger: - include: - - artifact: spack.artifacts.arm64/spack_pipeline.yaml - job: gather child artifacts - strategy: depend - extends: .rules -build spacktainers for amd64: - stage: build spacktainer containers - needs: - - job: gather child artifacts - artifacts: true - - populate buildcache for amd64 - trigger: - include: - - artifact: spacktainer.artifacts.amd64/artifacts.amd64/spacktainer_pipeline.yaml - job: gather child artifacts - strategy: depend - extends: .rules -build spacktainers for arm64: - rules: - - exists: - - container_definitions/arm64 - stage: build spacktainer containers - needs: - - job: gather child artifacts - artifacts: true - - populate buildcache for arm64 - trigger: - include: - - artifact: spacktainer.artifacts.arm64/artifacts.arm64/spacktainer_pipeline.yaml - job: gather child artifacts - strategy: depend - extends: .rules -test spackitor: - stage: unit tests - image: python:3.10-buster - script: - - pip install './spackitor[test]' - - coverage run -m pytest --junit-xml=unittests.xml -vs spackitor/tests - - coverage report - - coverage xml - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: [spackitor/**/*] - coverage: /TOTAL.*\s+(\d+%)$/ - artifacts: - when: always - reports: - junit: unittests.xml - coverage_report: - coverage_format: cobertura - path: coverage.xml -cleanup deleted branch containers from the bucket: - stage: cleanup - image: bbpgitlab.epfl.ch:5050/hpc/spacktainers/singularitah:latest - extends: .no-merge-request-rules - before_script: - - sed -i 's/^access_key.*/access_key='${AWS_INFRASTRUCTURE_ACCESS_KEY_ID}'/' /root/.s3cfg - - sed -i 's/^secret_key.*/secret_key='${AWS_INFRASTRUCTURE_SECRET_ACCESS_KEY}'/' - /root/.s3cfg - - let length=$(($(echo $(expr index ${HTTP_PROXY:7} :)) - 1)) - - PROXY_HOST=${HTTP_PROXY:7:${length}} - - PROXY_PORT=${HTTP_PROXY:$((7+${length}+1))} - - sed -i 's/^proxy_host.*/proxy_host='${PROXY_HOST}'/' /root/.s3cfg - - sed -i 's/^proxy_port.*/proxy_port='${PROXY_PORT}'/' /root/.s3cfg - - cat /root/.s3cfg - script: - - git fetch - - git branch -r | sed 's|origin/||' - - branches=($(git branch -r | sed 's|origin/||' | awk '{print $NF}')) - - echo "Existing branches are ${branches[*]}" - - for container in $(s3cmd ls s3://sboinfrastructureassets/containers/ | awk '{print - $4}'); do - - container_branch=$(echo $container | awk -F'__' '{print $3}') - - echo "Container ${container} was built with branch ${container_branch}" - - if [[ -z "${container_branch}" ]] ; then - - echo "no branch for ${container} - skipping" - - continue - - fi - - if [[ ! " ${branches[*]} " =~ " ${container_branch} " ]]; then - - echo "${container_branch} not found in existing branches" - - s3cmd rm ${container} - - else - - echo "The branch still exists, it can stay" - - fi - - done -cleanup deleted branch containers from bb5: - stage: cleanup - tags: [bb5_map] - variables: - CONTAINER_ROOT: /gpfs/bbp.cscs.ch/ssd/containers/hpc/spacktainers - extends: .no-merge-request-rules - script: - - module load unstable git - - git fetch - - git branch -r | sed 's|origin/||' - - branches=($(git branch -r | sed 's|origin/||' | awk '{print $NF}')) - - echo "Existing branches are ${branches[*]}" - - for container in $(ls ${CONTAINER_ROOT}); do - - container_branch=$(echo $container | awk -F'__' '{print $3}' | sed 's/.sif//') - - echo "Checking container ${container} with branch ${container_branch}" - - if [[ -z "${container_branch}" ]] ; then - - echo "no branch for ${container} - skipping" - - continue - - fi - - if [[ ! " ${branches[*]} " =~ " ${container_branch} " ]]; then - - echo "${container_branch} not found in existing branches" - - rm -f ${CONTAINER_ROOT}/${container} - - else - - echo "The branch still exists, it can stay" - - fi - - done diff --git a/README.md b/README.md index 09ee518..8ddbddf 100644 --- a/README.md +++ b/README.md @@ -15,29 +15,6 @@ The only files you should have to edit as an end-user are located in the `contai In both cases, the filename will be used as the name of your container. In case of a YAML file, the container version will be derived from the first package in your spec. In case of a def file, the version will be the same as the tag on docker hub. -## Adding extra files to your containers - -Create a folder under `spacktainer/files` to hold your container's files. Make sure to use your container's name to keep everything somewhat orderly. -In your container definition file, add a `spacktainer` section with a `files` key. This key holds a list of `source:target` filepairs (note that there is no space between source and target!) -Source is specified starting from the level below `spacktainer`; in the example below the folder structure would look like this: -``` -spacktainer/files -└── my-awesome-container - ├── some_folder - │   ├── brilliant_hack.patch - │   ├── readme.txt - │   ├── ugly_hack.patch - │   └── useless_but_if_we_delete_it_everything_breaks.jpg - └── script.sh - -``` - -``` -spack: - specs: - - my-awesome-package -``` - # Developer documentation ## Build Order @@ -66,26 +43,6 @@ Folders of note are: * container_definitions: this is where users will define their containers * runtime: base container that contains everything needed to run the spack-built environment -## job_creator - -The main entrypoints can be found, unsurprisingly, in the `__main__.py` file. This is where the `click` commands are defined. - -`architectures.py` contains the configuration for different architectures: what bucket should be used for the Spack package cache, which tag should be applied for the gitlab jobs, in which variables is the authentication defined, etc - -`ci_objects.py` contains helper object that can be used to define gitlab jobs and workflows. These will take care of architecture-specific behaviour (such as setting/unsetting the proxy, setting AWS variables, ...) - -`containers.py` holds everything related to generating container jobs: classes that define the base containers (former Spacktainerizer, Singularitah) as well as the spacktainers (formerly Spackah) and custom containers. It also contains the methods that use these classes and return a workflow with only the required jobs. - -`job_templates.py` holds job definition templates as python dictionaries. - -`logging_config.py` should be self-explanatory - -`packages.py` holds everything related to package-building jobs. Here you'll find the methods that generate the workflows for building the job that runs `spack ci generate` as well as the job that processes the output. - -`spack_template.py` contains the spack.yaml template that will be merged with the user's container config to generate the spack.yaml that will be used to build the container - -`utils.py` contains utility functions for reading/writing yaml, getting the multiarch job for a container, ... - ## Pulling images with Apptainer, Podman, or Sarus Make sure you have your AWS credentials set up. Then identify the image you want to run. diff --git a/container_definitions/amd64/multiscale-run/spack.yaml b/container_definitions/amd64/multiscale-run/spack.yaml index 1e52b19..d0968c1 100644 --- a/container_definitions/amd64/multiscale-run/spack.yaml +++ b/container_definitions/amd64/multiscale-run/spack.yaml @@ -1,6 +1,20 @@ ---- spack: - specs: [py-multiscale-run] + specs: + - "gmsh@4:" + - "py-astrovascpy@0.1.5:" + - "py-bluepysnap@2:" + - "py-libsonata@0.1.20:" + - "py-mpi4py@3:" + - "py-notebook@6:" + - "py-numpy@1.22:" + - "py-pandas@1.4:" + - "py-psutil@5.8:" + - "py-pyyaml@5:" + - "py-scipy@1.11.1:" + - "py-simpleeval@0.9.13:" + - "py-tqdm@4.65:" + - "py-trimesh@3:" + - "steps@5" packages: petsc: require: