feat: use pagination and map component-progress to include all tasks

packages/component-progress/package.json

@@ -21,8 +21,8 @@
     "prebuild": "npm run clean"
   },
   "devDependencies": {
-    "@octokit/graphql": "8.1.1",
-    "lodash.isempty": "4.4.0",
-    "octokit": "4.0.2"
+    "@octokit/core": "6.1.2",
+    "@octokit/plugin-paginate-graphql": "5.2.2",
+    "lodash.isempty": "4.4.0"
   }
 }

packages/component-progress/src/getComponentProgress.mjs

@@ -0,0 +1,56 @@
+import { graphqlWithAuth } from './graphqlWithAuth.mjs';
+import { mapToComponentProgress } from './utils.mjs';
+
+export const getComponentProgress = async (projectNumbers, projects) => {
+  const octokit = graphqlWithAuth();
+
+  const {
+    repository: { issues },
+  } = await octokit.graphql.paginate(
+    `query componentProgress($cursor: String) {
+  repository(name: "backlog", owner: "nl-design-system") {
+    issues(labels: ["component"], first: 50, after: $cursor) {
+      totalCount
+      nodes {
+        title
+        backlog: bodyUrl
+        projectItems(includeArchived: false, first: 50) {
+          nodes {
+            project {
+              title
+              number
+            }
+            fieldValues(first: 50) {
+              nodes {
+                ... on ProjectV2ItemFieldValueCommon {
+                  field {
+                    ... on ProjectV2FieldCommon {
+                      dataType
+                      name
+                      id
+                    }
+                  }
+                }
+                ... on ProjectV2ItemFieldSingleSelectValue {
+                  value: name
+                  color
+                }
+                ... on ProjectV2ItemFieldTextValue {
+                  value: text
+                }
+              }
+            }
+          }
+        }
+      },
+      pageInfo {
+        hasNextPage
+        endCursor
+      }
+    }
+  }
+}`,
+  );
+
+  return mapToComponentProgress(issues.nodes, projectNumbers, projects);
+};

packages/component-progress/src/getProjects.mjs

@@ -6,50 +6,43 @@ const CANDIDATE_PROJECT = 32;
 const HALL_OF_FAME_PROJECT = 30;
 
 export const PROJECT_NUMBERS = [HELP_WANTED_PROJECT, COMMUNITY_PROJECT, CANDIDATE_PROJECT, HALL_OF_FAME_PROJECT];
+const octokit = graphqlWithAuth();
 
-export const getProjectDetails = async () => {
-  const { organization } = await graphqlWithAuth({
-    query: `query projectDetails($helpWanted: Int!, $community: Int!, $candidate: Int!, $hallOfFame: Int!) {
-    organization(login: "nl-design-system") {
-      HELP_WANTED: projectV2(number:$helpWanted) {
-        ...ProjectFragment
-      }
-      COMMUNITY: projectV2(number:$community) {
-        ...ProjectFragment
-      }
-      CANDIDATE: projectV2(number:$candidate) {
-        ...ProjectFragment
-      }
-      HALL_OF_FAME: projectV2(number:$hallOfFame) {
-        ...ProjectFragment
-      }
-    }
-  }
-  
-  fragment ProjectFragment on ProjectV2 {
-  number
-  title
-  shortDescription
-  url
-  updatedAt
-  view(number: 1) {
-    fields(first: 50) {
-      totalCount
-      checks: nodes {
-        ... on ProjectV2FieldCommon {
-          dataType
-          name
-          id
+export const getProjectDetails = async (projectNumber) => {
+  const {
+    organization: {
+      projectV2: { view, ...project },
+    },
+  } = await octokit.graphql(
+    `query projectDetails($projectNumber: Int!) {
+  organization(login: "nl-design-system") {
+    projectV2(number: $projectNumber) {
+      number
+      title
+      shortDescription
+      url
+      updatedAt
+      view(number: 1) {
+        fields(first: 100) {
+          nodes {
+            ... on ProjectV2FieldCommon {
+              dataType
+              name
+              id
+            }
+          }
         }
       }
     }
   }
 }`,
-    helpWanted: HELP_WANTED_PROJECT,
-    community: COMMUNITY_PROJECT,
-    candidate: CANDIDATE_PROJECT,
-    hallOfFame: HALL_OF_FAME_PROJECT,
-  });
+    {
+      projectNumber,
+    },
+  );
 
-  return organization;
+  return {
+    ...project,
+    columns: view.fields.nodes.filter((check) => check.dataType !== 'TITLE' && check.name !== 'Status'),
+  };
 };

packages/component-progress/src/graphqlWithAuth.mjs

@@ -1,8 +1,8 @@
 /* global process */
-import { graphql } from '@octokit/graphql';
+import { Octokit } from '@octokit/core';
+import { paginateGraphQL } from '@octokit/plugin-paginate-graphql';
 
-export const graphqlWithAuth = graphql.defaults({
-  headers: {
-    authorization: `token ${process.env.GH_ISSUES_TOKEN}`,
-  },
-});
+export const graphqlWithAuth = () => {
+  const MyOctokit = Octokit.plugin(paginateGraphQL);
+  return new MyOctokit({ auth: `token ${process.env.GH_ISSUES_TOKEN}` });
+};

packages/component-progress/src/index.mjs

@@ -1,27 +1,51 @@
 import * as fs from 'fs/promises';
-import { getComponentProgess } from './getComponents.mjs';
+import { getComponentProgress } from './getComponentProgress.mjs';
 import { getProjectDetails } from './getProjects.mjs';
 
+const HELP_WANTED = 27;
+const COMMUNITY = 29;
+const CANDIDATE = 32;
+const HALL_OF_FAME = 30;
+
+export const PROJECT_NUMBERS = {
+  HELP_WANTED: 27,
+  COMMUNITY: 29,
+  CANDIDATE: 32,
+  HALL_OF_FAME: 30,
+};
+
 const init = async () => {
   try {
     await fs.mkdir('./dist', { recursive: true });
   } catch (error) {
-    console.error('Could not create dist directory', error);
+    console.error('Could not create dist directory');
   }
 
-  try {
-    const componentProgress = await getComponentProgess();
-    await fs.writeFile('./dist/component-progress.json', JSON.stringify(componentProgress));
-  } catch (error) {
-    console.error('Could not create component-progress.json', error);
-  }
+  const helpWanted = await getProjectDetails(HELP_WANTED);
+  await fs
+    .writeFile('./dist/help-wanted.json', JSON.stringify(helpWanted))
+    .catch(() => console.error('Could not create help-wanted.json'));
 
-  try {
-    const projectDetails = await getProjectDetails();
-    await fs.writeFile('./dist/project-details.json', JSON.stringify(projectDetails));
-  } catch (error) {
-    console.error('Could not create project-details.json', error);
-  }
+  const community = await getProjectDetails(COMMUNITY);
+  await fs
+    .writeFile('./dist/community.json', JSON.stringify(community))
+    .catch(() => console.error('Could not create help-wanted.json'));
+
+  const candidate = await getProjectDetails(CANDIDATE);
+  await fs
+    .writeFile('./dist/candidate.json', JSON.stringify(candidate))
+    .catch(() => console.error('Could not create candidate.json'));
+
+  const hallOfFame = await getProjectDetails(HALL_OF_FAME);
+  await fs
+    .writeFile('./dist/hall-of-fame.json', JSON.stringify(hallOfFame))
+    .catch(() => console.error('Could not create hall-of-fame.json'));
+
+  const components = await getComponentProgress(PROJECT_NUMBERS, { helpWanted, community, candidate, hallOfFame });
+
+  await fs
+    .writeFile('./dist/index.json', JSON.stringify(components))
+    .catch(() => console.error('Could not create index.json'));
 };
 
 init();

packages/component-progress/src/utils.mjs

@@ -0,0 +1,91 @@
+import isEmpty from 'lodash.isempty';
+
+const cleanupValue = ({ field, value, color }) => {
+  // Only allow https values
+  if (field.dataType === 'TEXT' && URL.canParse(value)) {
+    const url = new URL(value);
+    return url.protocol === 'https:';
+  }
+
+  // Assume green means done
+  if (field.dataType === 'SINGLE_SELECT') {
+    return color === 'GREEN';
+  }
+
+  return !!value;
+};
+
+const toProjectString = (number, projectNumbers) => {
+  switch (number) {
+    case projectNumbers.HELP_WANTED:
+      return 'HELP_WANTED';
+    case projectNumbers.COMMUNITY:
+      return 'COMMUNITY';
+    case projectNumbers.CANDIDATE:
+      return 'CANDIDATE';
+    case projectNumbers.HALL_OF_FAME:
+      return 'HALL_OF_FAME';
+    default:
+      return '';
+  }
+};
+
+const cleanupFields = ({ project, fieldValues }, projectNumbers) => ({
+  title: project.title,
+  number: project.number,
+  project: toProjectString(project.number, projectNumbers),
+  checks: fieldValues.nodes
+    .filter((node) => !isEmpty(node) && node.field.dataType !== 'TITLE' && node.field.name !== 'Status')
+    .map((node) => ({ ...node.field, value: node.value, done: cleanupValue(node) })),
+});
+
+const getAllTasks = (issue, projectNumbers, projects) => {
+  const getProject = (number) => {
+    if (number === projectNumbers.HELP_WANTED) {
+      return projects.helpWanted;
+    }
+    if (number === projectNumbers.COMMUNITY) {
+      return projects.community;
+    }
+    if (number === projectNumbers.CANDIDATE) {
+      return projects.candidate;
+    }
+    if (number === projectNumbers.HALL_OF_FAME) {
+      return projects.hallOfFame;
+    }
+    return [];
+  };
+  const project = getProject(issue.number);
+  const checked = issue.checks.filter((issue) => issue.done);
+  const tasks = project.columns.map((column) => {
+    const check = issue.checks.find((check) => check.id === column.id);
+
+    if (!check) {
+      return {
+        ...column,
+        value: '',
+        done: false,
+      };
+    }
+
+    return check;
+  });
+
+  return {
+    ...issue,
+    progress: { value: checked.length, max: project.columns.length },
+    tasks,
+  };
+};
+
+export const mapToComponentProgress = (issues, projectNumbers, projects) =>
+  issues.map(({ title, projectItems, ...issue }) => {
+    return {
+      title,
+      ...issue,
+      projects: projectItems.nodes
+        .filter(({ project }) => Object.values(projectNumbers).includes(project.number))
+        .map((component) => cleanupFields(component, projectNumbers))
+        .map((component) => getAllTasks(component, projectNumbers, projects)),
+    };
+  });