UBERF-10471: Fix Github miss status updates and allow to re-integrate existing repos (#8842)

Signed-off-by: Andrey Sobolev <[email protected]>

Author: haiodo

services/github/github-resources/src/components/ConnectProject.svelte

@@ -13,12 +13,18 @@
     showPopup
   } from '@hcengineering/ui'
   import DropdownLabelsPopup from '@hcengineering/ui/src/components/DropdownLabelsPopup.svelte'
-  import { GithubIntegration, GithubIntegrationRepository, githubPullRequestStates } from '@hcengineering/github'
+  import {
+    GithubIntegration,
+    GithubIntegrationRepository,
+    githubPullRequestStates,
+    type GithubProject
+  } from '@hcengineering/github'
   import github from '../plugin'
 
   export let integration: WithLookup<GithubIntegration>
   export let repository: GithubIntegrationRepository
   export let projects: Project[] = []
+  export let orphanProjects: GithubProject[] = []
 
   /**
    * @public
@@ -112,16 +118,23 @@
 
     const githubProject = client.getHierarchy().as(projectInst, github.mixin.GithubProject)
 
-    void getClient().update(githubProject, {
+    if (githubProject.integration !== integration._id) {
+      await getClient().update(githubProject, {
+        integration: integration._id
+      })
+    }
+    await getClient().update(githubProject, {
       $push: { repositories: repository._id }
     })
-    void getClient().update(repository, { githubProject: githubProject._id, enabled: true })
+    await getClient().update(repository, { githubProject: githubProject._id, enabled: true })
   }
 
-  $: allowedProjects = projects.filter(
-    (it) =>
-      (client.getHierarchy().asIf(it, github.mixin.GithubProject)?.integration ?? integration._id) === integration._id
-  )
+  $: allowedProjects = projects
+    .filter(
+      (it) =>
+        (client.getHierarchy().asIf(it, github.mixin.GithubProject)?.integration ?? integration._id) === integration._id
+    )
+    .concat(orphanProjects)
   async function selectProject (event: MouseEvent): Promise<void> {
     showPopup(
       DropdownLabelsPopup,

services/github/github-resources/src/components/GithubIntegerations.svelte

@@ -1,11 +1,11 @@
 <script lang="ts">
   import GithubRepositories from './GithubRepositories.svelte'
 
-  import { WithLookup } from '@hcengineering/core'
+  import { toIdMap, WithLookup } from '@hcengineering/core'
+  import { GithubIntegration } from '@hcengineering/github'
   import { getClient } from '@hcengineering/presentation'
   import { Project } from '@hcengineering/tracker'
   import { Scroller } from '@hcengineering/ui'
-  import { GithubIntegration } from '@hcengineering/github'
   import github from '../plugin'
 
   export let integrations: WithLookup<GithubIntegration>[] = []
@@ -14,6 +14,10 @@
   const client = getClient()
 
   $: githubProjects = client.getHierarchy().asIfArray(projects, github.mixin.GithubProject)
+
+  $: integerationsMap = toIdMap(integrations)
+
+  $: orphanProjects = githubProjects.filter((it) => !integerationsMap.has(it.integration))
 </script>
 
 {#if integrations.length > 0}
@@ -23,7 +27,7 @@
         {@const giprj = githubProjects.filter((it) => it.integration === gi._id)}
         <div class="flex flex-col mb-4">
           <!-- svelte-ignore a11y-missing-attribute -->
-          <GithubRepositories integration={gi} giProjects={giprj} {projects} />
+          <GithubRepositories integration={gi} giProjects={giprj} {projects} {orphanProjects} />
         </div>
       {/each}
     </div>

services/github/github-resources/src/components/GithubRepositories.svelte

@@ -26,11 +26,11 @@
   import ConnectProject from './ConnectProject.svelte'
   import { githubLanguageColors } from './languageColors'
   import { sendGHServiceRequest } from './utils'
-  import { BackgroundColor } from '@hcengineering/text'
 
   export let integration: WithLookup<GithubIntegration>
   export let projects: Project[] = []
   export let giProjects: GithubProject[] = []
+  export let orphanProjects: GithubProject[] = []
 
   const client = getClient()
 
@@ -235,7 +235,7 @@
                 />
               </div>
             {:else}
-              <ConnectProject {integration} {repository} {projects} />
+              <ConnectProject {integration} {repository} {projects} {orphanProjects} />
             {/if}
           </div>
         </div>

services/github/pod-github/src/sync/comments.ts

@@ -528,7 +528,7 @@ export class CommentSyncManager implements DocSyncManager {
     }
     const syncInfo = await this.client.findAll<DocSyncInfo>(github.class.DocSyncInfo, {
       space: repo.githubProject,
-      repository: repo._id,
+      // repository: repo._id, // If we skip repository, we will find orphaned comments, so we could connect them on.
       objectClass: chunter.class.ChatMessage,
       url: { $in: comments.map((it) => (it.url ?? '').toLowerCase()) }
     })
@@ -550,14 +550,19 @@ export class CommentSyncManager implements DocSyncManager {
             lastModified
           })
         } else {
-          if (!deepEqual(existing.external, comment) || existing.externalVersion !== githubExternalSyncVersion) {
+          if (
+            !deepEqual(existing.external, comment) ||
+            existing.externalVersion !== githubExternalSyncVersion ||
+            existing.repository !== repo._id
+          ) {
             await derivedClient.diffUpdate(
               existing,
               {
                 needSync: '',
                 external: comment,
                 externalVersion: githubExternalSyncVersion,
-                lastModified
+                lastModified,
+                repository: repo._id
               },
               lastModified
             )

services/github/pod-github/src/sync/issueBase.ts

@@ -69,6 +69,7 @@ import {
   errorToObj,
   getCreateStatus,
   getType,
+  guessStatus,
   isGHWriteAllowed
 } from './utils'
 
@@ -1075,6 +1076,51 @@ export abstract class IssueSyncManagerBase {
     return issueUpdate
   }
 
+  async performDocumentExternalSync (
+    ctx: MeasureContext,
+    info: DocSyncInfo,
+    previousExternal: IssueExternalData,
+    issueExternal: IssueExternalData,
+    derivedClient: TxOperations
+  ): Promise<void> {
+    // TODO: Since Github integeration need to be re-written to use cards, so this is quick fix to not loose data in case of external sync while service was offline.
+
+    const update: IssueUpdate = {}
+    const du: DocumentUpdate<DocSyncInfo> = {}
+    const account = (await this.provider.getAccount(issueExternal.author)) ?? core.account.System
+
+    const container = await this.provider.getContainer(info.space)
+    if (container == null) {
+      return
+    }
+    const type = await this.provider.getTaskTypeOf(container.project.type, tracker.class.Issue)
+    const statuses = await this.provider.getStatuses(type?._id)
+
+    if (previousExternal.state !== issueExternal.state) {
+      update.status = (
+        await guessStatus({ state: issueExternal.state, stateReason: issueExternal.stateReason }, statuses)
+      )._id
+    }
+    if (previousExternal.title !== issueExternal.title) {
+      update.title = issueExternal.title
+    }
+    if (previousExternal.body !== issueExternal.body) {
+      update.description = await this.provider.getMarkupSafe(
+        container.container,
+        issueExternal.body,
+        this.stripGuestLink
+      )
+      du.markdown = await this.provider.getMarkdown(update.description)
+    }
+    if (!deepEqual(previousExternal.assignees, issueExternal.assignees)) {
+      const assignees = await this.getAssignees(issueExternal)
+      update.assignee = assignees?.[0] ?? null
+    }
+    if (Object.keys(update).length > 0) {
+      await this.handleUpdate(issueExternal, derivedClient, update, account, container.project, false)
+    }
+  }
+
   async syncIssues (
     _class: Ref<Class<Doc>>,
     repo: GithubIntegrationRepository,
@@ -1089,7 +1135,7 @@ export abstract class IssueSyncManagerBase {
       syncDocs ??
       (await this.client.findAll<DocSyncInfo>(github.class.DocSyncInfo, {
         space: repo.githubProject,
-        repository: repo._id,
+        // repository: repo._id, // If we skip repository, we will find orphaned issues, so we could connect them on.
         objectClass: _class,
         url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) }
       }))
@@ -1123,9 +1169,19 @@ export abstract class IssueSyncManagerBase {
           if (syncDocs !== undefined) {
             syncDocs = syncDocs.filter((it) => it._id !== existing._id)
           }
-          const externalEqual = deepEqual(existing.external, issue)
+          const externalEqual = deepEqual(existing.external, issue) && existing.repository === repo._id
           if (!externalEqual || existing.externalVersion !== githubExternalSyncVersion) {
-            this.ctx.info('Update sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId() })
+            this.ctx.info('Update sync doc(extarnal changes)', {
+              url: issue.url,
+              workspace: this.provider.getWorkspaceId()
+            })
+
+            if (existing.needSync === githubSyncVersion || existing.repository !== repo._id) {
+              // Sync external if and only if no changes from platform or we do resync from github.
+              // We need to apply changes from Github, while service was offline.
+              await this.performDocumentExternalSync(this.ctx, existing, existing.external, issue, derivedClient)
+            }
+
             await ops.diffUpdate(
               existing,
               {
@@ -1134,6 +1190,7 @@ export abstract class IssueSyncManagerBase {
                 externalVersion: githubExternalSyncVersion,
                 derivedVersion: '', // Clear derived state to recalculate it.
                 externalVersionSince: '',
+                repository: repo._id,
                 lastModified: new Date(issue.updatedAt).getTime()
               },
               Date.now()

services/github/pod-github/src/sync/utils.ts

@@ -370,12 +370,13 @@ export async function syncDerivedDocuments<T extends { url: string }> (
       })
     } else {
       processed.add(existing._id)
-      if (!deepEqual(existing.external, r)) {
+      if (!deepEqual(existing.external, r) || existing.repository !== repo._id) {
         // Only update if had changes.
         await derivedClient.update(existing, {
           external: r,
           needSync: '', // We need to check if we had any changes.
           derivedVersion: '',
+          repository: repo._id,
           externalVersion: githubExternalSyncVersion,
           lastModified: new Date(r.updatedAt ?? r.createdAt).getTime(),
           ...extra