01-parsec-integration.yml

name: parsec integration

on:
  workflow_call:
    outputs:
      build_matrix_main:
        description: deepin main build tasks
        value: ${{ jobs.parsec_integration.outputs.build_matrix_main }}
      build_matrix_dde:
        description: build tasks
        value: ${{ jobs.parsec_integration.outputs.build_matrix_dde }}
      build_matrix_community:
        description: build tasks
        value: ${{ jobs.parsec_integration.outputs.build_matrix_community }}
      build_matrix_commercial:
        description: build tasks
        value: ${{ jobs.parsec_integration.outputs.build_matrix_commercial }}
      all_build_task:
        description: all build tasks used for
        value: ${{ jobs.parsec_integration.outputs.all_build_task }}
      testrepoaddr:
        description: testing repositories addresses
        value: ${{ jobs.parsec_integration.outputs.testrepoaddr }}
      deploy:
        description: deploy way AUTO or MANUALLY
        value: ${{ jobs.parsec_integration.outputs.deploy }}
      integration_message:
        description: integration message at integration.yml
        value: ${{ jobs.parsec_integration.outputs.integration_message }}
      integrationinfo:
        description: integration repos list
        value: ${{ jobs.parsec_integration.outputs.integrationinfo }}
      changelog:
        description: integration pacakge changelog
        value: ${{ jobs.parsec_integration.outputs.changelog }}
      influence:
        description: integration package for test influence information
        value: ${{ jobs.parsec_integration.outputs.influence }}
      milestone:
        description: integration milestone
        value: ${{ jobs.parsec_integration.outputs.milestone }}
      labels:
        description: integration repos with issue labels
        value: ${{ jobs.parsec_integration.outputs.labels }}

jobs:
  parsec_integration:
    name: parsec integration
    runs-on: ubuntu-latest
    outputs:
      build_matrix_main: ${{ steps.parsec_integration.outputs.build_matrix_main }}
      build_matrix_dde: ${{ steps.parsec_integration.outputs.build_matrix_dde }}
      build_matrix_community: ${{ steps.parsec_integration.outputs.build_matrix_community }}
      build_matrix_commercial: ${{ steps.parsec_integration.outputs.build_matrix_commercial }}
      all_build_task: ${{ steps.parsec_integration.outputs.all_build_task }}
      testrepoaddr: ${{ steps.parsec_integration.outputs.testrepoaddr }}
      deploy: ${{ steps.illegal_repo_tags.outputs.DEPLOY }}
      integration_message: ${{ steps.parsec_integration.outputs.integration_message }}
      integrationinfo: ${{ steps.parsec_integration.outputs.integrationinfo }}
      changelog: ${{ steps.parsec_integration.outputs.changelog }}
      influence: ${{ steps.parsec_integration.outputs.influence }}
      milestone: ${{ steps.parsec_integration.outputs.milestone }}
      labels: ${{ steps.illegal_repo_tags.outputs.labels }}

    steps:
      - name: checkout_pull_request
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          persist-credentials: false

      - name: parsec_integration
        id: parsec_integration
        shell: python
        env:
          GITHUB_TOKEN: ${{ github.token }}
          PR_TITLE: ${{ github.event.pull_request.title }}
          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
          COMMERCIALS: ${{ vars.COMMERCIALS }}
        run: |
          import requests
          import yaml
          import os
          import logging
          import base64
          import json
          from requests.utils import requote_uri

          illegalTags = []
          illegalRepos = []
          noChangelogs = []

          permissions = None
          with open("permissions.yml") as permissions_file:
              permissions = yaml.load(permissions_file, Loader=yaml.BaseLoader)
          not_allowed_repo = permissions.get("noauto")

          header = {
              "Accept": "application/vnd.github+json",
              "Authorization":"Bearer " + os.environ.get("GITHUB_TOKEN")
          }

          def read_integration():
              with open("integration.yml", "r+") as integration_file:
                  return yaml.load(integration_file, Loader=yaml.BaseLoader)

          def check_tag_exsist(repo, tag):
              rawurl = requote_uri("https://api.github.com/repos/{repo}/git/ref/tags/{tag}".format(repo=repo, tag=tag))
              res = requests.get(rawurl.replace("+", "%2B"), headers=header)
              print(res.content, tag, rawurl.replace("+", "%2B"))
              if res.status_code != 200:
                  illegalTags.append({"repo": repo, "tag": tag})
              if repo in not_allowed_repo:
                  illegalRepos.append(repo)
          
          def set_output(name, value):
              output_file = os.environ.get("GITHUB_OUTPUT")
              with open(output_file, "a") as output:
                  #output.write(name + "=" + value + "\n")
                  output.write(name + "<<EOF\n")
                  output.write(value)
                  output.write('\nEOF\n')

          def check_changelog_changed(repo, commitortag):
            rawurl = "https://api.github.com/repos/{repo}/commits/{commit}".format(repo=repo, commit=commitortag)
            res = requests.get(rawurl.replace("+", "%2B"), headers=header)
            if res.status_code == 200:
              files = res.json()["files"]
              #print(files)
              for file in files:
                if file["filename"] == "debian/changelog":
                  return True
              return False
            else:
              print("Warn: get changelog files failed!!!")
              return False

          def read_changelog(repo, commitortag):
            rawurl = "https://api.github.com/repos/{repo}/contents/debian/changelog?ref={commit}".format(repo=repo, commit=commitortag)
            res = requests.get(rawurl.replace("+", "%2B"), headers=header)
            repoinfo = repo.split("/")
            name = repo
            version = None
            changelog = []
            influence = []
            if len(repoinfo) == 2:
              name = repoinfo[1]

            #print(res.content)
            if res.status_code == 200:
              content = str(base64.b64decode(res.json()["content"]), encoding='utf-8')
              #print("b64decode: ", str(content))
              first_line_readed = False
              for line in content.split("\n"):
                # read end with newest changelog section
                if line.startswith(" -- "):
                  break

                if not first_line_readed:
                  first_line_readed = True
                  lineinfo = line.split(" ")
                  #print(lineinfo)
                  name = lineinfo[0]
                  version = lineinfo[1].replace("(", "").replace(")", "")
                  line = "#### " + line
                else:
                  print("get influence information")
                  lineinfo = line.split(" ")
                  lineend = lineinfo[len(lineinfo)-1]
                  if lineend.startswith("(Influence: "):
                    influence.append(lineend[10:-1])

                changelog.append(line)

            print(name, version, changelog, influence)
            return name, version, "\n".join(changelog), "\n".join(influence)

          def get_component(repo):
            print("get component")
            commercials = os.environ.get("COMMERCIALS")
            commercial_list = commercials.split(",")
            component = "main"
            package = repo[repo.find('/')+1:]
            if "linuxdeepin/" in repo:
              component = "dde"
            else:
              rawurl = "https://raw.githubusercontent.com/{repo}/master/debian/deepin/workflows.yml".format(repo=repo)
              res = requests.get(rawurl.replace("+", "%2B"))
              if res.status_code == 200:
                if "deepin:Develop:community" in str(res.content):
                  component = "community"
              else:
                print("Warn: get workflows.yml content failed!!!")
                component = "community"

            if package in commercial_list:
              component = "commercial"
            return component

          try:
              components = ["main", "dde", "community", "commercial"]
              allBuildTasks = [{"component": j,"data":{"payload": []}} for j in components]
              data = read_integration()

              message = data.get("message", "Integrated for V23-Release").replace("\n", "")
              if message == "Integrated for V23-Release":
                message = os.environ.get("PR_TITLE")

              message = message + " by " + os.environ.get("PR_AUTHOR")
              set_output("integration_message", message.replace("\n", ""))
              with open("integration-${{ github.event.number }}-detail.md", "w+") as integration_detail:
                  integration_detail.write(message)

              integrationInfos = ["包名 | 版本", "--  | --"]
              changelogs = []
              influences = []
              for repo in data.get("repos"):
                fullreponame = str(repo.get('repo', ""))
                component = get_component(fullreponame)
                component_index = 0
                if component == "dde":
                  component_index = 1
                if component == "community":
                  component_index = 2
                if component == "commercial":
                  component_index = 3
                print("component: %s" % component)
                tagsha = repo.get('tagsha', "")
                tag = repo.get('tag', "")
                if tag and tag != "" and len(tag) > 0:
                  check_tag_exsist(fullreponame, tag)
                  name, version, changelog, influence = read_changelog(fullreponame, tag)
                  #print(name, version, changelog, influence)
                  integrationInfos.append("%s | [%s](https://github.com/%s/tree/%s)" % (name, version, fullreponame, tag))
                  changelogs.append(changelog)
                  influences.append(influence)
                  if not check_changelog_changed(fullreponame, tag):
                    noChangelogs.append({"repo": fullreponame, "tag": tag, "tagsha": tagsha})
                elif tagsha and tagsha != "" and len(tagsha) > 0:
                  name, version, changelog, influence = read_changelog(fullreponame, tagsha)
                  #print("tagsha: ", name, version, changelog, influence)
                  integrationInfos.append("%s | [%s](https://github.com/%s/commit/%s)" % (name, version, fullreponame, tagsha))
                  changelogs.append(changelog)
                  influences.append(influence)
                  if not check_changelog_changed(fullreponame, tagsha):
                    noChangelogs.append({"repo": fullreponame, "tag": tagsha, "tagsha": tagsha})
                if tagsha == "" and len(tagsha) <= 0 and tag == "" and len(tag) <= 0:
                  illegalTags.append({"repo": fullreponame, "tag": tag, "tagsha": tagsha})
                allBuildTasks[component_index].get("data").get("payload").append(repo)
              integrationinfo = str("\n".join(integrationInfos))
              print("integrationinfo: " + integrationinfo)
              changelog = str("\n".join(changelogs))
              influence = str("\n".join(influences))
              set_output("integrationinfo", integrationinfo)
              with open("integration-${{ github.event.number }}-integrationinfo.md", "w+") as f:
                  f.write(integrationinfo)
              set_output("changelog", changelog)
              with open("integration-${{ github.event.number }}-changelog.md", "w+") as f:
                  f.write(changelog)
              set_output("influence", influence)
              with open("integration-${{ github.event.number }}-influence.md", "w+") as f:
                  f.write(influence)
              set_output("milestone", "V23-Beta2")
              if data.get('milestone', "") != "":
                set_output("milestone", data.get('milestone'))

              if len(illegalTags) > 0:
                  set_output("illegal_tags", str(illegalTags))
              else:
                  set_output("illegal_tags", "undefined")

              if len(illegalRepos) > 0:
                  set_output("illegal_repos", str(illegalRepos))
              else:
                  set_output("illegal_repos", "undefined")

              if len(noChangelogs) > 0:
                  set_output("no_changelogs", str(noChangelogs))
              else:
                  set_output("no_changelogs", "undefined")

              allRepos = []
              testrepoaddr = ""
              for c in allBuildTasks:
                  print(c)
                  print(c.get("component"))
                  for repo in c.get("data").get("payload"):
                      allRepos.append(repo)
                  if len(c.get("data").get("payload")) > 0:
                      set_output("build_matrix_" + str(c.get("component")), str(c.get("data")))
                      if str(c.get("component")) == "main":
                        if testrepoaddr != "":
                          testrepoaddr = testrepoaddr + "\n    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}/testing/ ./"
                        else:
                          testrepoaddr ="    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}/testing/ ./"
                      else:
                        if testrepoaddr != "":
                          testrepoaddr = testrepoaddr + "\n    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}:/" + str(c.get("component")) + "/testing/ ./"
                        else:
                          testrepoaddr ="    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}:/" + str(c.get("component")) + "/testing/ ./"
              set_output("all_build_task", str(allRepos))
              print(testrepoaddr)
              set_output("testrepoaddr", str(testrepoaddr))
              with open("integration-${{ github.event.number }}-testrepoaddr.md", "w+") as f:
                f.write(testrepoaddr)

          except BaseException as e:
              logging.error(e)
              exit(-10)

      - name: illegal_repo_tags
        id: illegal_repo_tags
        uses: actions/github-script@v6
        with:
          github-token: ${{ github.token }}
          script: |
            const BOT_NAME = "RepoTagCheck Bot"
            const COMMENT_HEAD = "**" + BOT_NAME + "**\n\n"
            illegal_tags = ${{ steps.parsec_integration.outputs.illegal_tags }}
            illegal_repos = ${{ steps.parsec_integration.outputs.illegal_repos }}
            no_changelogs = ${{ steps.parsec_integration.outputs.no_changelogs }}
            core.setOutput('labels', "Project:integrated")
            let COMMENT_BODY = ""
            if ( illegal_tags !== undefined || illegal_repos !== undefined ) {
              if ( illegal_tags !== undefined ) {
                  COMMENT_BODY += "Check failed!\nListed repo and tag pairs are not exists: \n|repo|tag|\n|-|-|\n"
                  illegal_tags.forEach(tag => {
                    COMMENT_BODY += "|" + tag["repo"] + "|" + tag["tag"] + "|\n"
                  })
                  COMMENT_BODY += "\n"
              }

              if ( illegal_repos !== undefined ) {
                  COMMENT_BODY += "Listed repos are not allowed by automatic integration: \n|repo|\n|-|\n"
                  illegal_repos.forEach(repo => {
                    COMMENT_BODY += "|" + repo + "|\n"
                  })
              }
            } else if ( no_changelogs !== undefined ) {
              core.setOutput('labels', "Project:Pre-integrated")
              COMMENT_BODY += "Listed repo integration are not modified changelog: \n|repo|tagOrsha|\n|-|-|\n"
              no_changelogs.forEach(tag => {
                COMMENT_BODY += "|" + tag["repo"] + "|" + tag["tag"] + "|\n"
              })
              COMMENT_BODY += "\n"
            } else {
              COMMENT_BODY = "All repo and tag pairs check passed!"
            }

            if ( context.issue.number != undefined ) {
              const response = await github.rest.issues.listComments({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number
              })
              const reg = new RegExp("\\*+" + BOT_NAME + "\\*+")
              tagBotComment= response.data.find(comment => comment.body.match(reg))
              if (tagBotComment) {
                await github.rest.issues.updateComment({
                    comment_id: tagBotComment.id,
                    owner: context.repo.owner,
                    repo: context.repo.repo,
                    body: COMMENT_HEAD + COMMENT_BODY
                })
              } else {
                await github.rest.issues.createComment({
                    issue_number: context.issue.number,
                    owner: context.repo.owner,
                    repo: context.repo.repo,
                    body: COMMENT_HEAD + COMMENT_BODY
                })
              }
            }

            core.setOutput('DEPLOY', 'TestingIntegrationTesterReview')
            if ( illegal_tags !== undefined ) {
              core.setFailed("some tags are not invalid!")
            }

      - name: upload integration detail
        uses: actions/upload-artifact@v3
        with:
          name: integration-archive-${{ github.event.number }}
          path: |
            integration-${{ github.event.number }}-detail.md
            integration-${{ github.event.number }}-integrationinfo.md
            integration-${{ github.event.number }}-changelog.md
            integration-${{ github.event.number }}-influence.md
            integration-${{ github.event.number }}-testrepoaddr.md