diff --git a/.github/actions/cleanup-conda/action.yml b/.github/actions/cleanup-conda/action.yml index 7958b0a40f..336beec080 100644 --- a/.github/actions/cleanup-conda/action.yml +++ b/.github/actions/cleanup-conda/action.yml @@ -6,6 +6,7 @@ runs: steps: - name: Remove extra conda environments run: | + conda env list CONDA_REMOVE_NAMES=$(conda env list | awk '{print $1}' | tail -n +3 | grep "${{ env.conda-env-name-no-time }}" || true) if [ -z "$CONDA_REMOVE_NAMES" ]; then echo "No matching conda environments for ${{ env.conda-env-name-no-time }}. Skip removal." diff --git a/.github/actions/create-conda-env/action.yml b/.github/actions/create-conda-env/action.yml index b9fd606690..7faa2a70f4 100644 --- a/.github/actions/create-conda-env/action.yml +++ b/.github/actions/create-conda-env/action.yml @@ -5,24 +5,20 @@ runs: steps: - name: Create conda environments run: | + conda env list if conda env list | grep -q "envs/${{ env.conda-env-name-no-time }}"; then echo "Using pre-existing conda environments with prefix ${{ env.conda-env-name-no-time }}" else echo "Creating a conda environment for each toolchain with the toolchain installed" - conda activate base - conda-lock install --conda $(which conda) -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools ./conda-reqs/conda-lock-reqs/conda-requirements-riscv-tools-linux-64.conda-lock.yml - conda deactivate - - echo "Add extra toolchain collateral + CIRCT to RISC-V install area" - git submodule update --init ./tools/install-circt - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools - ./scripts/build-toolchain-extra.sh riscv-tools -p $CONDA_PREFIX/riscv-tools - ./tools/install-circt/bin/download-release-or-nightly-circt.sh \ - -f circt-full-shared-linux-x64.tar.gz \ - -i $CONDA_PREFIX \ - -v version-file \ - -x ./conda-reqs/circt.json \ - -g ${{ github.token }} - conda deactivate + ./build-setup.sh \ + --conda-env-name ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) \ + --github-token ${{ github.token }} \ + --verbose \ + --skip-submodules \ + --skip-ctags \ + --skip-precompile \ + --skip-firesim \ + --skip-marshal \ + --skip-clean fi shell: bash -leo pipefail {0} diff --git a/.github/actions/prepare-rtl/action.yml b/.github/actions/prepare-rtl/action.yml index 74a040371e..5366460687 100644 --- a/.github/actions/prepare-rtl/action.yml +++ b/.github/actions/prepare-rtl/action.yml @@ -13,10 +13,6 @@ inputs: description: type of build required: false default: "sim" - toolchain: - description: toolchain to use - required: false - default: "riscv-tools" runs: using: "composite" @@ -32,7 +28,7 @@ runs: - name: Run RTL build if not cached run: | - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }} + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) if [[ "${{ steps.rtl-build-id.outputs.cache-hit }}" != 'true' ]]; then echo "Cache miss on ${{ inputs.group-key }}-${{ github.sha }}" ./.github/scripts/${{ inputs.build-script }} ${{ inputs.group-key }} ${{ inputs.build-type }} diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 1dd0ce89c1..fbfd4649de 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -12,17 +12,13 @@ inputs: description: rtl build script to use required: false default: "run-tests.sh" - toolchain: - description: toolchain to use - required: false - default: "riscv-tools" runs: using: "composite" steps: - name: Init submodules (since only the RTL is cached) run: | - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }} + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) ./scripts/init-submodules-no-riscv-tools.sh shell: bash -leo pipefail {0} @@ -31,10 +27,9 @@ runs: uses: ./.github/actions/prepare-rtl with: group-key: ${{ inputs.group-key }} - toolchain: ${{ inputs.toolchain }} - name: Run RTL tests run: | - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }} + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) ./.github/scripts/${{ inputs.run-script }} ${{ inputs.project-key }} shell: bash -leo pipefail {0} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f55b10dc8f..1708dd63fc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,28 +1,31 @@ version: 2 +# update weekly (to avoid spam) updates: - # submit a PR for bumping submodules daily (for selected submodules). + # bump submodules. # updates a submodule to the latest commit on the branch given in .gitmodules. # if branch not given, then it defaults to the main/master branch. - package-ecosystem: gitsubmodule schedule: - interval: "daily" + interval: "weekly" directory: / labels: - "dependencies" - "changelog:omit" - # ignore certain submodules - ignore: - - dependency-name: "generators/gemmini" - - dependency-name: "generators/rocket-chip" - - dependency-name: "generators/rocket-chip-blocks" - - dependency-name: "generators/rocket-chip-inclusive-cache" + # allow certain submodules + allow: + - dependency-name: "sims/firesim" + - dependency-name: "software/baremetal-ide" + - dependency-name: "software/firemarshal" + - dependency-name: "toolchains/riscv-tools/riscv-isa-sim" + - dependency-name: "toolchains/riscv-tools/riscv-openocd" + - dependency-name: "toolchains/riscv-tools/riscv-pk" + - dependency-name: "toolchains/riscv-tools/riscv-spike-devices" - dependency-name: "toolchains/riscv-tools/riscv-tests" - - dependency-name: "toolchains/riscv-tools/riscv-tools-feedstock" # submit a PR for bumping GH-A dependencies - package-ecosystem: github-actions schedule: - interval: "daily" + interval: "weekly" directory: / labels: - "dependencies" diff --git a/.github/workflows/chipyard-run-tests.yml b/.github/workflows/chipyard-run-tests.yml index c64deac7b3..ae4cfca439 100644 --- a/.github/workflows/chipyard-run-tests.yml +++ b/.github/workflows/chipyard-run-tests.yml @@ -114,7 +114,7 @@ jobs: uses: ./.github/actions/create-conda-env - name: Check commits of each submodule run: | - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) .github/scripts/check-commit.sh documentation-check: @@ -136,7 +136,7 @@ jobs: uses: ./.github/actions/create-conda-env - name: Check that documentation builds with no warnings/errors run: | - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) make -C docs html - name: Show error log from sphinx if failed if: ${{ failure() }} @@ -172,7 +172,7 @@ jobs: restore-keys: ${{ steps.genkey.outputs.extra-tests-cache-key }} - name: Build extra tests run: | - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) .github/scripts/build-extra-tests.sh prepare-chipyard-cores: @@ -351,14 +351,14 @@ jobs: uses: ./.github/actions/create-conda-env - name: Build Gemmini FireMarshal run: | - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) cd ${{ github.workspace }} && ./scripts/init-submodules-no-riscv-tools.sh cd ${{ github.workspace }} && source ./scripts/fix-open-files.sh git submodule update --init software/firemarshal && cd software/firemarshal && ./init-submodules.sh cd ${{ github.workspace }}/generators/gemmini/software && ${{ github.workspace }}/software/firemarshal/marshal -v -d build gemmini-smoke.json - name: Running Gemmini FireMarshal smoke test run: | - conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) cd ${{ github.workspace }}/generators/gemmini/software && ${{ github.workspace }}/software/firemarshal/marshal -v -d launch -s gemmini-smoke.json chipyard-rocket-run-tests: diff --git a/.github/workflows/update-circt.yml b/.github/workflows/update-circt.yml index f0c3158b2d..f0cb6e1215 100644 --- a/.github/workflows/update-circt.yml +++ b/.github/workflows/update-circt.yml @@ -1,10 +1,10 @@ name: update-circt -# run daily +# TODO: update weekly (to avoid spam) on: schedule: - cron: 0,15,30,45 * * * * - #- cron: 0 8 * * * + #- cron: 0 0 * * 1 defaults: run: @@ -23,3 +23,4 @@ jobs: pr-reviewers: '' pr-labels: 'changelog:changed' circt-config: './conda-reqs/circt.json' + github-token: ${{ secrets.BARTENDER_PERSONAL_ACCESS_TOKEN }} diff --git a/scripts/build-setup.sh b/scripts/build-setup.sh index d738c19f14..2115602f12 100755 --- a/scripts/build-setup.sh +++ b/scripts/build-setup.sh @@ -37,8 +37,10 @@ usage() { echo " --help -h : Display this message" echo " --verbose -v : Verbose printout" echo " --use-unpinned-deps -ud : Use unpinned conda environment" - echo " --use-lean-conda : Install a leaner version of the repository (Smaller conda env, no FireSim, no FireMarshal)" + echo " --use-lean-conda : Install a leaner version of the repository (smaller conda env, no FireSim, no FireMarshal)" echo " --build-circt : Builds CIRCT from source, instead of downloading the precompiled binary" + echo " --conda-env-name NAME : Optionally use a global conda env name instead of storing env in Chipyard directory" + echo " --github-token TOKEN : Optionally use a Github token to download CIRCT" echo " --skip -s N : Skip step N in the list above. Use multiple times to skip multiple steps ('-s N -s M ...')." echo " --skip-conda : Skip Conda initialization (step 1)" @@ -61,6 +63,8 @@ USE_UNPINNED_DEPS=false USE_LEAN_CONDA=false SKIP_LIST=() BUILD_CIRCT=false +GLOBAL_ENV_NAME="" +GITHUB_TOKEN="null" # getopts does not support long options, and is inflexible while [ "$1" != "" ]; @@ -78,6 +82,12 @@ do SKIP_LIST+=(4 6 7 8 9) ;; --build-circt) BUILD_CIRCT=true ;; + --conda-env-name) + shift + GLOBAL_ENV_NAME=${1} ;; + --github-token) + shift + GITHUB_TOKEN=${1} ;; -ud | --use-unpinned-deps ) USE_UNPINNED_DEPS=true ;; --skip | -s) @@ -154,11 +164,12 @@ if run_step "1"; then # create conda-lock only environment to be used in this section. # done with cloning base then installing conda lock to speed up dependency solving. - rm -rf $CYDIR/.conda-lock-env && - conda create -y -p $CYDIR/.conda-lock-env --clone base && + CONDA_LOCK_ENV_PATH=$CYDIR/.conda-lock-env + rm -rf $CONDA_LOCK_ENV_PATH && + conda create -y -p $CONDA_LOCK_ENV_PATH --clone base && source $(conda info --base)/etc/profile.d/conda.sh && - conda activate $CYDIR/.conda-lock-env && - conda install -y -c conda-forge -p $CYDIR/.conda-lock-env $(grep "conda-lock" $CONDA_REQS/chipyard-base.yaml | sed 's/^ \+-//') + conda activate $CONDA_LOCK_ENV_PATH && + conda install -y -c conda-forge -p $CONDA_LOCK_ENV_PATH $(grep "conda-lock" $CONDA_REQS/chipyard-base.yaml | sed 's/^ \+-//') exit_if_last_command_failed if [ "$USE_UNPINNED_DEPS" = true ]; then @@ -166,12 +177,22 @@ if run_step "1"; then $CYDIR/scripts/generate-conda-lockfiles.sh exit_if_last_command_failed fi - echo "Using lockfile: $LOCKFILE" + echo "Using lockfile for conda: $LOCKFILE" # use conda-lock to create env - conda-lock install --conda $(which conda) -p $CYDIR/.conda-env $LOCKFILE && + if [ -z "$GLOBAL_ENV_NAME" ] ; then + CONDA_ENV_PATH=$CYDIR/.conda-env + CONDA_ENV_ARG="-p $CONDA_ENV_PATH" + CONDA_ENV_NAME=$CONDA_ENV_PATH + else + CONDA_ENV_ARG="-n $GLOBAL_ENV_NAME" + CONDA_ENV_NAME=$GLOBAL_ENV_NAME + fi + echo "Storing main conda environment in $CONDA_ENV_NAME" + + conda-lock install --conda $(which conda) $CONDA_ENV_ARG $LOCKFILE && source $(conda info --base)/etc/profile.d/conda.sh && - conda activate $CYDIR/.conda-env + conda activate $CONDA_ENV_NAME exit_if_last_command_failed # Conda Setup @@ -191,7 +212,7 @@ END_CONDA_ACTIVATE replace_content env.sh build-setup-conda "# line auto-generated by $0 $CONDA_ACTIVATE_PREAMBLE -conda activate $CYDIR/.conda-env +conda activate $CONDA_ENV_NAME source $CYDIR/scripts/fix-open-files.sh" fi @@ -308,7 +329,7 @@ if run_step "10"; then -i $PREFIX \ -v version-file \ -x $CYDIR/conda-reqs/circt.json \ - -g null + -g $GITHUB_TOKEN fi exit_if_last_command_failed fi