Merge branch 'main' into 3803-update-premium-service-pages

Author: Kemi-Elizabeth

.github/actions/redirection-verification/package-lock.json

@@ -7,14 +7,37 @@ on:
     inputs:
       url:
         description: 'Site to check for broken links'
-        required: true
         type: string
-        default: 'https://docs.platform.sh/'
+        default: ''
 
 jobs:
+  determine-urls:
+    name: determine which urls to use
+    runs-on: ubuntu-latest
+    outputs:
+      urls: ${{ steps.determine-url.outputs.urls }}
+    steps:
+      - id: determine-url
+        run: |
+          # default URLs we want to check
+          # todo: replace this with a vars.VAL ?
+          urls="[\"https://docs.platform.sh\",\"https://docs.upsun.com\"]"
+          if [ "workflow_dispatch" = "${{ github.event_name }}" ]; then
+            # make sure there's no extra spaces in what we were given
+            inputURL=$(echo "${{ inputs.url }}" | xargs)
+            if [ -n "${inputURL}" ]; then
+              urls="[\"${inputURL}\"]"
+            fi
+          fi
+          echo "urls=${urls}" >> $GITHUB_OUTPUT
   check-links:
     name: Check links
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        url: ${{ fromJSON(needs.determine-urls.outputs.urls) }}
+    needs:
+      - determine-urls
     steps:
       - uses: actions/setup-python@v4
         with:
@@ -26,38 +49,29 @@ jobs:
           pip3 install linkchecker
 
       - name: checkout xml to markdown
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: platformsh/linkchecker-xml2md
 
+      - name: checkout linkchecker config
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            linkcheckerrc
+          sparse-checkout-cone-mode: false
+          path: linkchecker
       - name: install-parser
         run: |
           pip3 install -r requirements.txt
 
-      - name: Set URL to scan
-        id: set-url
-        shell: bash
-        run: |
-          # If this is a workflow dispatch, then use what was entered.
-          # If not, then use the repository variable SCAN_URL if it is set
-          # if not, fall back to the hard-coded default
-          scanURL="https://docs.platform.sh/"
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            scanURL="${{ inputs.url }}"
-          elif [[ -n "${{ vars.SCAN_URL }}" ]]; then
-            scanURL="${{ vars.SCAN_URL }}"
-          fi
-
-          echo "scanURL=${scanURL}" >> "$GITHUB_ENV"
-
       - name: Run linkchecker
         id: run-link-checker
         continue-on-error: true
         shell: bash
         run: |
-          echo "::notice::Scanning ${{ inputs.url }} for broken links."
+          echo "::notice::Scanning ${{ matrix.url }} for broken links."
           # if linkchecker exits with a non-zero then it means broken links were found.
-          scan=$(linkchecker ${{ env.scanURL }} -F xml/utf_8/brokenlinks.xml --check-extern --user-agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" --ignore-url="^https://github.com/platformsh*" --ignore-url="console.platform.sh/projects/*" --ignore-url="support.blackfire.io/*" --ignore-url="cloud.orange-business.com/*" --ignore-url="developers.cloudflare.com/*" --ignore-url="discord.com/*" --ignore-url="pptr.dev/*" --ignore-url="mvnrepository.com/*" --ignore-url="https://dev.mysql.com/doc/refman/en/" --ignore-url="https://www.microsoft.com/en-us/trust-center/privacy/data-management")
+          scan=$(linkchecker ${{ matrix.url }} -f ./linkchecker/linkcheckerrc -F xml/utf_8/brokenlinks.xml )
           result=$?
           if [[ $result -ne 0 ]]; then
             echo "::notice::Broken links detected."
@@ -74,14 +88,21 @@ jobs:
         run: |
           echo "::notice::No broken links were detected"
 
+      - name: get domain name
+        if: ${{ steps.run-link-checker.outcome == 'failure' }}
+        id: get-domain-name
+        run: |
+          domainName=$(basename ${{ matrix.url }})
+          echo "domain=${domainName}" >> $GITHUB_OUTPUT
       - name: check message size
         id: check-msg-size
         if: ${{ steps.run-link-checker.outcome == 'failure' }}
         run: |
           artifact="false"
           sizeOfMessage=$(cat broken_links.md | wc -c)
+          echo "::notice::Size of report: ${sizeOfMessage}"
           # the size of a comment in an issue is 65536
-          if [ "${sizeOfMessage}" -gt 650000 ]; then
+          if [ "${sizeOfMessage}" -gt 65000 ]; then
             # we'll need to store the broken_links.md file as an aritifact
             artifact="true"
           fi
@@ -93,26 +114,27 @@ jobs:
         if: ${{ steps.run-link-checker.outcome == 'failure' && env.artifact == 'true' }}
         uses: actions/upload-artifact@v4
         with:
-          name: broken-links
+          name: broken-links-${{ steps.get-domain-name.outputs.domain }}
           path: |
-            ${{ github.action_path }}/broken_links.md
+            ./broken_links.md
 
       - name: Create issue
         id: create-issue
         if: ${{ steps.run-link-checker.outcome == 'failure' }}
         env:
           ARTIFACT: ${{ env.artifact }}
+          DOMAIN: ${{ steps.get-domain-name.outputs.domain }}
         uses: actions/github-script@v6
         with:
           script: |
-            const { ARTIFACT } = process.env
+            const { ARTIFACT, DOMAIN } = process.env
             var fs = require('fs');
             //
-            failMsg = ':bug: Broken links detected'
+            failMsg = ':bug: Broken links detected on ' + DOMAIN
 
             if ('true' == ARTIFACT) {
               address = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}#artifacts`
-              var bodyMsg = `Visual Regression Testing failed. Please view the report artifact at ${address}`
+              var bodyMsg = `Broken links found. Please view the report artifact at ${address}`
             } else {
               var bodyMsg = fs.readFileSync('broken_links.md','utf8');
             }

.github/workflows/scheduled-check-links.yaml

@@ -0,0 +1,30 @@
+[checking]
+threads=5
+useragent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36
+maxrequestspersecond=5
+
+[filtering]
+checkextern=1
+ignore=
+    ^https://github.com/platformsh*
+    console.platform.sh/projects/*
+    support.blackfire.io/*
+    cloud.orange-business.com/*
+    developers.cloudflare.com/*
+    discord.com/*
+    pptr.dev/*
+    mvnrepository.com/*
+    https://dev.mysql.com/doc/refman/en/
+    https://www.microsoft.com/en-us/trust-center/privacy/data-management
+    http://localhost:1337/*
+
+[output]
+warnings=0
+ignoreerrors=
+    ^https://(www|new)\.drupal\.org/.*$ ^403 Forbidden
+    ^https://support\.platform\.sh/hc/en-us/.*$ ^403 Forbidden
+    ^https://www\.ipaddressguide\.com/.*$ ^403 Forbidden
+    ^https://www\.abuseipdb\.com/.*$ ^403 Forbidden
+    ^https://www\.cloudflare\.com/.*$ ^403 Forbidden
+    ^https://mkyong.com/java/java-how-to-send-email/$ ^403 Forbidden
+    ^https://www.baeldung.com/maven-wrapper$ ^403 Forbidden

linkcheckerrc

@@ -7,7 +7,7 @@ license = "Attribution-ShareAlike 4.0 International"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-cryptography = "^43.0.1"
+cryptography = "^44.0.1"
 Scrapy = "^2.11.2"
 meilisearch = "^0.24.0"
 platformshconfig = "^2.3.1"

search/poetry.lock

@@ -208,6 +208,7 @@
         "1.9"
       ],
       "supported": [
+        "1.18",
         "1.15",
         "1.14"
       ],