update deps, add a dev mode, collect sboms

.github/workflows/main.yml

@@ -27,6 +27,9 @@ jobs:
       - run: node index.js
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: du -hs public/*
+      - run: gzip -9v public/sboms.json
+      - run: du -hs public/*
       - run: tar -cf public.tar ./public
       - uses: actions/[email protected]
         with:

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 public/repos.json
+public/sboms.json

index.js

@@ -2,9 +2,14 @@
 
 import yaml from "js-yaml";
 import Octokat from "octokat";
+import { Octokit } from "@octokit/core";
 import Promise from "bluebird";
 import { writeFileSync } from "fs";
 
+const octokit = new Octokit({
+  auth: process.env.GITHUB_TOKEN,
+});
+
 const octo = new Octokat({
   token: process.env.GITHUB_TOKEN,
 });
@@ -25,13 +30,15 @@ const formatResult = (result) => {
 };
 
 const fetchAll = async (org, args) => {
-  let response = await octo.orgs(org).repos.fetch({ per_page: 100 });
+  let response = await octo
+    .orgs(org)
+    .repos.fetch({ per_page: process.env.NODE_ENV === "dev" ? 10 : 100 });
   let aggregate = [response];
 
   console.log(`fetched page 1 for ${org}`);
   let i = 1;
   await Promise.delay(50); //slow down to appease github rate limiting
-  while (response.nextPage) {
+  while (response.nextPage && process.env.NODE_ENV !== "dev") {
     i++;
     response = await response.nextPage();
     console.log(`fetched page ${i} for ${org}`);
@@ -49,18 +56,60 @@ const allDepartments = yaml.safeLoad(
   ).text()
 );
 
-const UKDepartments = [].concat(
-  allDepartments["U.K. Councils"],
-  allDepartments["U.K. Central"]
-);
+const UKDepartments =
+  process.env.NODE_ENV === "dev"
+    ? ["ukhomeoffice"]
+    : [].concat(
+        allDepartments["U.K. Councils"],
+        allDepartments["U.K. Central"]
+      );
 
 const allReposForAllUKDepartments = await Promise.mapSeries(
   UKDepartments,
   fetchAll
 );
 
+const fetchAllSboms = async (repo) => {
+  await Promise.delay(5000); //slow down to appease github rate limiting
+  console.log(`Collecting SBOM for ${repo.owner.login}/${repo.name}`);
+  try {
+    return (
+      await octokit.request("GET /repos/{owner}/{repo}/dependency-graph/sbom", {
+        owner: repo.owner.login,
+        repo: repo.name,
+        headers: {
+          "X-GitHub-Api-Version": "2022-11-28",
+        },
+      })
+    ).data.sbom;
+  } catch (e) {
+    if (e.status != 404) {
+      throw e;
+    }
+  }
+};
+
+const formatSboms = (sbom) => {
+  return {
+    name: sbom.name,
+    packages: sbom?.packages?.map((pkg) => {
+      return `${pkg.name}@${pkg.versionInfo}`;
+    }),
+  };
+};
+
 const formattedResults = allReposForAllUKDepartments.flat(2).map(formatResult);
 
+const allSbomsForAllUKDepartments = await Promise.mapSeries(
+  allReposForAllUKDepartments.flat(2),
+  fetchAllSboms
+);
+const formattedSboms = allSbomsForAllUKDepartments
+  .filter((sbom) => sbom?.name)
+  .map(formatSboms);
+
 console.log("writing results to file");
+
 writeFileSync("./public/repos.json", JSON.stringify(formattedResults));
+writeFileSync("./public/sboms.json", JSON.stringify(formattedSboms));
 console.log("done");