Gadi Prerelease Config Input Copy to /scratch/tm70/remote-copy-test/dest #187
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Config Inputs Remote Copy | |
| run-name: "${{ inputs.remote-environment }} Config Input Copy to ${{ inputs.target }}" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| remote-environment: | |
| type: choice | |
| required: true | |
| description: The Github Environment for the given remote | |
| options: | |
| - Gadi | |
| - Gadi Prerelease | |
| source: | |
| type: string | |
| required: true | |
| description: Remote absolute path to configuration input file/folder | |
| target: | |
| type: string | |
| required: true | |
| description: Remote absolute path for the copied configuration input file/folder | |
| overwrite-target: | |
| type: boolean | |
| required: true | |
| description: Overwrite the remote target if it already exists | |
| target-acl-spec: | |
| type: string | |
| required: true | |
| # Default to no write for everyone except tm70_ci | |
| # TODO: This default will probably not work for other `remote-environment`s | |
| default: >- | |
| u::rwx, | |
| u:tm70_ci:rwx, | |
| g::r-x, | |
| m::rwx, | |
| o::---, | |
| d:u::rwx, | |
| d:u:tm70_ci:rwx, | |
| d:g::r-x, | |
| d:m::rwx, | |
| d:o::--- | |
| description: ACL spec to be passed to `setfacl -m` for the given target | |
| store-on-tape: | |
| type: boolean | |
| required: true | |
| default: true | |
| description: Also store target on the remotes cold storage service | |
| jobs: | |
| setup: | |
| name: Setup | |
| runs-on: ubuntu-latest | |
| outputs: | |
| # The `inputs.target-acl-spec` with spaces removed | |
| formatted-acl: ${{ steps.fmt.outputs.acl }} | |
| steps: | |
| - name: Log inputs | |
| run: | | |
| echo "::notice::Copy on ${{ inputs.remote-environment }} from '${{ inputs.source }}' to '${{ inputs.target }}' with ACLs '${{ inputs.target-acl-spec }}'" | |
| echo "::${{ inputs.overwrite-target && 'warning' || 'notice' }}::This operation ${{ inputs.overwrite-target && 'WILL' || 'will not' }} overwrite ${{ inputs.target }}" | |
| - name: Verify inputs | |
| run: | | |
| errors=false | |
| if [ -z "${{ inputs.source }}"]; then | |
| echo "::error::No 'source' input given, can't copy anything." | |
| errors=true | |
| fi | |
| if [ -z "${{ inputs.target }}"]; then | |
| echo "::error::No 'target' input given, can't copy to anywhere." | |
| errors=true | |
| fi | |
| if [ -z "${{ inputs.target-acl-spec }}" ]; then | |
| echo "::notice::No 'ACL' input given, not setting the ACLs explicitly." | |
| fi | |
| if [[ "$errors" == "true" ]]; then | |
| echo "::error::Errors above, exiting..." | |
| exit 1 | |
| fi | |
| - name: Format ACL | |
| id: fmt | |
| # Remove spaces from ACL string as we have later logic that relies on | |
| # the IFS=, | |
| run: | | |
| acl=$(echo '${{ inputs.target-acl-spec }}' | tr -d ' ') | |
| echo "Formatted ACL: $acl" | |
| echo "acl=$acl" >> $GITHUB_OUTPUT | |
| test-acl: | |
| name: Test ACL | |
| runs-on: ubuntu-latest | |
| if: inputs.target-acl-spec != '' | |
| needs: | |
| - setup | |
| container: rockylinux/rockylinux:8.10 | |
| env: | |
| TEST_DIR: /opt/test | |
| steps: | |
| - name: Create Users in ACL String | |
| # We don't error out here as it could have been because we are adding the same user twice | |
| run: | | |
| set +e | |
| acl="${{ needs.setup.outputs.formatted-acl }}" | |
| IFS=, | |
| for entry in $acl; do | |
| echo "Testing ACL for u(ser): $entry" | |
| if [[ $entry =~ ^u(ser)?:([^:]+): ]]; then | |
| user="${BASH_REMATCH[2]}" | |
| echo "Adding user $user" | |
| useradd $user | |
| fi | |
| done | |
| - name: Create Groups in ACL String | |
| # We don't error out here as it could have been because we are adding the same group twice | |
| run: | | |
| set +e | |
| acl="${{ needs.setup.outputs.formatted-acl }}" | |
| IFS=, | |
| for entry in $acl; do | |
| if [[ $entry =~ ^g(roup)?:([^:]+): ]]; then | |
| echo "Testing ACL for g(roup): $entry" | |
| group="${BASH_REMATCH[2]}" | |
| echo "Adding group $group" | |
| groupadd $group | |
| fi | |
| done | |
| - name: Verify Valid ACL Spec | |
| # Now that we have created the users and groups from the ACL string, check if it is valid! | |
| run: | | |
| mkdir ${{ env.TEST_DIR }} | |
| echo "---- Users ----" | |
| cut -d: -f1 /etc/passwd | |
| echo "---- Groups ----" | |
| groups | |
| if setfacl --test --recursive --modify "${{ needs.setup.outputs.formatted-acl }}" ${{ env.TEST_DIR }}; then | |
| echo "::notice::ACL Verification Successful. This does not test that the users/groups exist on the remote environment" | |
| else | |
| echo "::error::ACL Verification Failed. Check the preceding lines." | |
| exit 1 | |
| fi | |
| copy-to-remote: | |
| name: Copy To ${{ inputs.remote-environment }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| - test-acl | |
| environment: ${{ inputs.remote-environment }} | |
| outputs: | |
| # Space-separated list of paths copied to the target | |
| files: ${{ steps.copy.outputs.paths }} | |
| # Space-separated list of manifests created at the target | |
| manifests: ${{ steps.manifest.outputs.paths }} | |
| steps: | |
| - name: Setup SSH | |
| id: ssh | |
| uses: access-nri/actions/.github/actions/setup-ssh@main | |
| with: | |
| private-key: ${{ secrets.SSH_KEY }} | |
| hosts: | | |
| ${{ secrets.SSH_HOST }} | |
| ${{ secrets.SSH_HOST_DATA }} | |
| - name: Verify Remote Target | |
| run: | | |
| if [[ "${{ startsWith(inputs.target, vars.CONFIGS_INPUT_DIR) }}" == "false" ]]; then | |
| echo "::error::Remote target '${{ inputs.target }}' doesn't look like a configurations input directory." | |
| exit 1 | |
| fi | |
| - name: Rsync Source to Target | |
| id: copy | |
| env: | |
| REMOTE_RSYNC_FILE_LIST_PATH: ${{ vars.REMOTE_TMP_DIR }}/remote-copy-files-${{ github.run_id }}.log | |
| LOCAL_RSYNC_FILE_LIST_PATH: ./files-copied.log | |
| # In this step, we rsync the files from the source to the target, | |
| # capturing the list of files copied as output of the step, since | |
| # it's used in the manifest step. | |
| run: | | |
| ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' | |
| rsync --recursive --out-format="${{ inputs.target }}/%n" \ | |
| ${{ ! inputs.overwrite-target && '--ignore-existing' || '--update' }} \ | |
| ${{ inputs.source }} ${{ inputs.target }} \ | |
| | grep --invert-match '/$' \ | |
| | tr '\n' ' ' \ | |
| > ${{ env.REMOTE_RSYNC_FILE_LIST_PATH }} | |
| EOT | |
| rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ | |
| ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ env.REMOTE_RSYNC_FILE_LIST_PATH }} \ | |
| ${{ env.LOCAL_RSYNC_FILE_LIST_PATH }} | |
| cat ${{ env.LOCAL_RSYNC_FILE_LIST_PATH }} | |
| paths=$(cat ${{ env.LOCAL_RSYNC_FILE_LIST_PATH }}) | |
| echo "paths are $paths" | |
| echo "paths=$paths" >> $GITHUB_OUTPUT | |
| - name: Set ACLs on Target | |
| if: inputs.target-acl-spec != '' | |
| run: | | |
| ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' | |
| setfacl --recursive --modify "${{ needs.setup.outputs.formatted-acl }}" ${{ inputs.target }} | |
| getfacl -t ${{ inputs.target }} | |
| EOT | |
| - name: Update Manifests | |
| id: manifest | |
| env: | |
| REMOTE_MANIFEST_FILE_LIST_PATH: ${{ vars.REMOTE_TMP_DIR }}/remote-copy-manifests-${{ github.run_id }}.log | |
| LOCAL_MANIFEST_FILE_LIST_PATH: ./manifests-copied.log | |
| MANIFEST_FILE_NAME: manifest.yaml | |
| # Generate manifests for files copied over in the earlier rsync job | |
| run: | | |
| ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' | |
| module use ${{ vars.YAMF_MODULE_PATH }} | |
| module load ${{ vars.YAMF_MODULE_NAME }} | |
| manifest_paths=() | |
| for path in ${{ steps.copy.outputs.paths }}; do | |
| echo "Path is $path" | |
| manifest_dir=$(dirname $path) | |
| cd $manifest_dir || exit | |
| manifest_file=$(basename $path) | |
| echo "Generating a manifest in $manifest_dir for $manifest_file" | |
| yamf add -n ${{ env.MANIFEST_FILE_NAME }} --force $manifest_file | |
| manifest_paths+=("$manifest_dir/${{ env.MANIFEST_FILE_NAME }}") | |
| done | |
| echo "${manifest_paths[@]}" > ${{ env.REMOTE_MANIFEST_FILE_LIST_PATH }} | |
| EOT | |
| rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ | |
| ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ env.REMOTE_MANIFEST_FILE_LIST_PATH }} \ | |
| ${{ env.LOCAL_MANIFEST_FILE_LIST_PATH }} | |
| cat ${{ env.LOCAL_MANIFEST_FILE_LIST_PATH }} | |
| paths=$(cat ${{ env.LOCAL_MANIFEST_FILE_LIST_PATH }}) | |
| echo "paths are $paths" | |
| echo "paths=$paths" >> $GITHUB_OUTPUT | |
| copy-to-tape: | |
| name: Copy To Tape | |
| runs-on: ubuntu-latest | |
| needs: | |
| - copy-to-remote | |
| # TODO: Generalize the copy-to-tape job for future remote-environments | |
| if: inputs.store-on-tape && startsWith(inputs.remote-environment, 'Gadi') | |
| environment: ${{ inputs.remote-environment }} | |
| steps: | |
| - name: Setup SSH | |
| id: ssh | |
| uses: access-nri/actions/.github/actions/setup-ssh@main | |
| with: | |
| private-key: ${{ secrets.SSH_KEY }} | |
| hosts: | | |
| ${{ secrets.SSH_HOST }} | |
| ${{ secrets.SSH_HOST_DATA }} | |
| - name: Send Target to Tape | |
| run: | | |
| ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' | |
| now=$(date +%Y_%m_%d_%H_%M) | |
| mdss -P ${{ vars.PROJECT_CODE }} mkdir -p ${{ vars.TAPE_ROOT_DIR }}/$now | |
| mdss -P ${{ vars.PROJECT_CODE }} put -r ${{ needs.copy-to-remote.outputs.files }} ${{ vars.TAPE_ROOT_DIR }}/$now | |
| mdss -P ${{ vars.PROJECT_CODE }} put -r ${{ needs.copy-to-remote.outputs.manifests }} ${{ vars.TAPE_ROOT_DIR }}/$now | |
| EOT |