From 0f06c26294eefc002865b797203cd60d1b02fe26 Mon Sep 17 00:00:00 2001 From: Chris Nesbitt-Smith Date: Thu, 27 Apr 2023 16:44:49 +0100 Subject: [PATCH 1/8] update deps, add a dev mode, collect sboms --- .gitignore | 1 + index.js | 61 +++++- package-lock.json | 491 ++++++++++++++++++++++++++++++++++++++++++---- package.json | 2 +- 4 files changed, 506 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index 04443f2..062a505 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules public/repos.json +public/sboms.json \ No newline at end of file diff --git a/index.js b/index.js index 1f6ef60..b359286 100644 --- a/index.js +++ b/index.js @@ -2,9 +2,14 @@ import yaml from "js-yaml"; import Octokat from "octokat"; +import { Octokit } from "@octokit/core"; import Promise from "bluebird"; import { writeFileSync } from "fs"; +const octokit = new Octokit({ + auth: process.env.GITHUB_TOKEN, +}); + const octo = new Octokat({ token: process.env.GITHUB_TOKEN, }); @@ -25,13 +30,15 @@ const formatResult = (result) => { }; const fetchAll = async (org, args) => { - let response = await octo.orgs(org).repos.fetch({ per_page: 100 }); + let response = await octo + .orgs(org) + .repos.fetch({ per_page: process.env.NODE_ENV === "dev" ? 10 : 100 }); let aggregate = [response]; console.log(`fetched page 1 for ${org}`); let i = 1; await Promise.delay(50); //slow down to appease github rate limiting - while (response.nextPage) { + while (response.nextPage && process.env.NODE_ENV !== "dev") { i++; response = await response.nextPage(); console.log(`fetched page ${i} for ${org}`); @@ -49,18 +56,60 @@ const allDepartments = yaml.safeLoad( ).text() ); -const UKDepartments = [].concat( - allDepartments["U.K. Councils"], - allDepartments["U.K. Central"] -); +const UKDepartments = + process.env.NODE_ENV === "dev" + ? ["ukhomeoffice"] + : [].concat( + allDepartments["U.K. Councils"], + allDepartments["U.K. Central"] + ); const allReposForAllUKDepartments = await Promise.mapSeries( UKDepartments, fetchAll ); +const fetchAllSboms = async (repo) => { + await Promise.delay(20); //slow down to appease github rate limiting + console.log(`Collecting SBOM for ${repo.owner.login}/${repo.name}`); + try { + return ( + await octokit.request("GET /repos/{owner}/{repo}/dependency-graph/sbom", { + owner: repo.owner.login, + repo: repo.name, + headers: { + "X-GitHub-Api-Version": "2022-11-28", + }, + }) + ).data.sbom; + } catch (e) { + if (e.status != 404) { + throw e; + } + } +}; + +const formatSboms = (sbom) => { + return { + name: sbom.name, + packages: sbom?.packages?.map((pkg) => { + return `${pkg.name}@${pkg.versionInfo}`; + }), + }; +}; + const formattedResults = allReposForAllUKDepartments.flat(2).map(formatResult); +const allSbomsForAllUKDepartments = await Promise.mapSeries( + allReposForAllUKDepartments.flat(2), + fetchAllSboms +); +const formattedSboms = allSbomsForAllUKDepartments + .filter((sbom) => sbom?.name) + .map(formatSboms); + console.log("writing results to file"); + writeFileSync("./public/repos.json", JSON.stringify(formattedResults)); +writeFileSync("./public/sboms.json", JSON.stringify(formattedSboms)); console.log("done"); diff --git a/package-lock.json b/package-lock.json index 4f53a76..518c99d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,12 +9,131 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@octokit/core": "^4.2.0", + "@octokit/request": "^6.2.3", + "@octokit/rest": "^14.0.9", "bluebird": "^3.7.2", - "github": "^14.0.0", "js-yaml": "^3.13.1", "octokat": "^0.4.18" } }, + "node_modules/@octokit/auth-token": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.3.tgz", + "integrity": "sha512-/aFM2M4HVDBT/jjDBa84sJniv1t9Gm/rLkalaz9htOm+L+8JMj1k9w0CkUdcxNyNxZPlTxKPVko+m1VlM58ZVA==", + "dependencies": { + "@octokit/types": "^9.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/core": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.0.tgz", + "integrity": "sha512-AgvDRUg3COpR82P7PBdGZF/NNqGmtMq2NiPqeSsDIeCfYFOZ9gddqWNQHnFdEUf+YwOj4aZYmJnlPp7OXmDIDg==", + "dependencies": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/core/node_modules/before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" + }, + "node_modules/@octokit/endpoint": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-7.0.5.tgz", + "integrity": "sha512-LG4o4HMY1Xoaec87IqQ41TQ+glvIeTKqfjkCEmt5AIwDZJwQeVZFIEYXrYY6yLwK+pAScb9Gj4q+Nz2qSw1roA==", + "dependencies": { + "@octokit/types": "^9.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/graphql": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.5.tgz", + "integrity": "sha512-Qwfvh3xdqKtIznjX9lz2D458r7dJPP8l6r4GQkIdWQouZwHQK0mVT88uwiU2bdTU2OtT1uOlKpRciUWldpG0yQ==", + "dependencies": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-17.0.0.tgz", + "integrity": "sha512-V8BVJGN0ZmMlURF55VFHFd/L92XQQ43KvFjNmY1IYbCN3V/h/uUFV6iQi19WEHM395Nn+1qhUbViCAD/1czzog==" + }, + "node_modules/@octokit/request": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-6.2.3.tgz", + "integrity": "sha512-TNAodj5yNzrrZ/VxP+H5HiYaZep0H3GU0O7PaF+fhDrt8FPrnkei9Aal/txsN/1P7V3CPiThG0tIvpPDYUsyAA==", + "dependencies": { + "@octokit/endpoint": "^7.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.7", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/request-error": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-3.0.3.tgz", + "integrity": "sha512-crqw3V5Iy2uOU5Np+8M/YexTlT8zxCfI+qu+LxUB7SZpje4Qmx3mub5DfEKSO8Ylyk0aogi6TYdf6kxzh2BguQ==", + "dependencies": { + "@octokit/types": "^9.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/rest": { + "version": "14.0.9", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-14.0.9.tgz", + "integrity": "sha512-irP9phKfTXEZIcW2R+VNCtGHZJrXMWmSYp6RRfFn4BtAqtDRXF5z9JxCEQlAhNBf6X1koNi5k49tIAAAEJNlVQ==", + "dependencies": { + "before-after-hook": "^1.1.0", + "debug": "^3.1.0", + "is-array-buffer": "^1.0.0", + "is-stream": "^1.1.0", + "lodash": "^4.17.4", + "url-template": "^2.0.8" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@octokit/types": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-9.1.2.tgz", + "integrity": "sha512-LPbJIuu1WNoRHbN4UMysEdlissRFpTCWyoKT7kHPufI8T+XX33/qilfMWJo3mCOjNIKu0+43oSQPf+HJa0+TTQ==", + "dependencies": { + "@octokit/openapi-types": "^17.0.0" + } + }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -23,11 +142,34 @@ "sprintf-js": "~1.0.2" } }, + "node_modules/before-after-hook": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-1.4.0.tgz", + "integrity": "sha512-l5r9ir56nda3qu14nAXIlyq1MmUSs0meCIaFAh8HwkFwP1F8eToOuS3ah2VAHHcY04jaYD7FpJC5JTXHYRbkzg==" + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, + "node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" + }, + "node_modules/es6-promise": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz", + "integrity": "sha512-CUD62/uqeE0L+EJeypOKuFfM56CFaH4vo+++J76bff0NkeQ2bBmWVCTNxL2hj9HeCYPkof6Gqea0BSeK17gBzA==" + }, "node_modules/esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", @@ -40,16 +182,31 @@ "node": ">=4" } }, - "node_modules/github": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/github/-/github-14.0.0.tgz", - "integrity": "sha512-34/VqwhYGeYN0VHBSH49TmRWMF7emy32qjK6POiW47T/QI2u/cpuKsmrWt7a218ew/73dF4dQSJE68/HXdNfPw==", - "deprecated": "'github' has been renamed to '@octokit/rest' (https://git.io/vNB11)" + "node_modules/is-array-buffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-1.0.1.tgz", + "integrity": "sha512-lj035IqdAwsodoRGs9/8+Kn3HPoz9CTuZbcw63afugWonxigvUVeHY5d6Ve1O+s1N3RCk7txo2TIWQLbU0SuNA==" + }, + "node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "engines": { + "node": ">=0.10.0" + } }, "node_modules/js-yaml": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", - "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -58,35 +215,199 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/node-fetch": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/octokat": { "version": "0.4.18", "resolved": "https://registry.npmjs.org/octokat/-/octokat-0.4.18.tgz", - "integrity": "sha1-r6flJlS1Rkkj+AGRezoxz/emhI0=", + "integrity": "sha512-n8xyWCKZRXv9TkdxZpCp6K3meyk/6T8amS3bHWU4xNjJxhS4kdY27uYzpaL4NiJvx/eJLDLcqL/uLvgI80hYSQ==", "dependencies": { "es6-promise": "3.0.2", "xmlhttprequest": "~1.8.0" } }, - "node_modules/octokat/node_modules/es6-promise": { - "version": "3.0.2", - "resolved": "http://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz", - "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=" + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" }, - "node_modules/octokat/node_modules/xmlhttprequest": { + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/universal-user-agent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", + "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" + }, + "node_modules/url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==" + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/xmlhttprequest": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz", - "integrity": "sha1-Z/4HXFwk/vOfnWX197f+dRcZaPw=", + "integrity": "sha512-58Im/U0mlVBLM38NdZjHyhuMtCqa61469k2YP/AaPbvCoV9aQGUpbJBj1QRm2ytRiVQBD/fsw7L2bJGDVQswBA==", "engines": { "node": ">=0.4.0" } - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" } }, "dependencies": { + "@octokit/auth-token": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.3.tgz", + "integrity": "sha512-/aFM2M4HVDBT/jjDBa84sJniv1t9Gm/rLkalaz9htOm+L+8JMj1k9w0CkUdcxNyNxZPlTxKPVko+m1VlM58ZVA==", + "requires": { + "@octokit/types": "^9.0.0" + } + }, + "@octokit/core": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.0.tgz", + "integrity": "sha512-AgvDRUg3COpR82P7PBdGZF/NNqGmtMq2NiPqeSsDIeCfYFOZ9gddqWNQHnFdEUf+YwOj4aZYmJnlPp7OXmDIDg==", + "requires": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" + } + } + }, + "@octokit/endpoint": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-7.0.5.tgz", + "integrity": "sha512-LG4o4HMY1Xoaec87IqQ41TQ+glvIeTKqfjkCEmt5AIwDZJwQeVZFIEYXrYY6yLwK+pAScb9Gj4q+Nz2qSw1roA==", + "requires": { + "@octokit/types": "^9.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.5.tgz", + "integrity": "sha512-Qwfvh3xdqKtIznjX9lz2D458r7dJPP8l6r4GQkIdWQouZwHQK0mVT88uwiU2bdTU2OtT1uOlKpRciUWldpG0yQ==", + "requires": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-17.0.0.tgz", + "integrity": "sha512-V8BVJGN0ZmMlURF55VFHFd/L92XQQ43KvFjNmY1IYbCN3V/h/uUFV6iQi19WEHM395Nn+1qhUbViCAD/1czzog==" + }, + "@octokit/request": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-6.2.3.tgz", + "integrity": "sha512-TNAodj5yNzrrZ/VxP+H5HiYaZep0H3GU0O7PaF+fhDrt8FPrnkei9Aal/txsN/1P7V3CPiThG0tIvpPDYUsyAA==", + "requires": { + "@octokit/endpoint": "^7.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.7", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-3.0.3.tgz", + "integrity": "sha512-crqw3V5Iy2uOU5Np+8M/YexTlT8zxCfI+qu+LxUB7SZpje4Qmx3mub5DfEKSO8Ylyk0aogi6TYdf6kxzh2BguQ==", + "requires": { + "@octokit/types": "^9.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/rest": { + "version": "14.0.9", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-14.0.9.tgz", + "integrity": "sha512-irP9phKfTXEZIcW2R+VNCtGHZJrXMWmSYp6RRfFn4BtAqtDRXF5z9JxCEQlAhNBf6X1koNi5k49tIAAAEJNlVQ==", + "requires": { + "before-after-hook": "^1.1.0", + "debug": "^3.1.0", + "is-array-buffer": "^1.0.0", + "is-stream": "^1.1.0", + "lodash": "^4.17.4", + "url-template": "^2.0.8" + } + }, + "@octokit/types": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-9.1.2.tgz", + "integrity": "sha512-LPbJIuu1WNoRHbN4UMysEdlissRFpTCWyoKT7kHPufI8T+XX33/qilfMWJo3mCOjNIKu0+43oSQPf+HJa0+TTQ==", + "requires": { + "@octokit/openapi-types": "^17.0.0" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -95,55 +416,141 @@ "sprintf-js": "~1.0.2" } }, + "before-after-hook": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-1.4.0.tgz", + "integrity": "sha512-l5r9ir56nda3qu14nAXIlyq1MmUSs0meCIaFAh8HwkFwP1F8eToOuS3ah2VAHHcY04jaYD7FpJC5JTXHYRbkzg==" + }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" + }, + "es6-promise": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz", + "integrity": "sha512-CUD62/uqeE0L+EJeypOKuFfM56CFaH4vo+++J76bff0NkeQ2bBmWVCTNxL2hj9HeCYPkof6Gqea0BSeK17gBzA==" + }, "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" }, - "github": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/github/-/github-14.0.0.tgz", - "integrity": "sha512-34/VqwhYGeYN0VHBSH49TmRWMF7emy32qjK6POiW47T/QI2u/cpuKsmrWt7a218ew/73dF4dQSJE68/HXdNfPw==" + "is-array-buffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-1.0.1.tgz", + "integrity": "sha512-lj035IqdAwsodoRGs9/8+Kn3HPoz9CTuZbcw63afugWonxigvUVeHY5d6Ve1O+s1N3RCk7txo2TIWQLbU0SuNA==" + }, + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==" }, "js-yaml": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", - "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", "requires": { "argparse": "^1.0.7", "esprima": "^4.0.0" } }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node-fetch": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, "octokat": { "version": "0.4.18", "resolved": "https://registry.npmjs.org/octokat/-/octokat-0.4.18.tgz", - "integrity": "sha1-r6flJlS1Rkkj+AGRezoxz/emhI0=", + "integrity": "sha512-n8xyWCKZRXv9TkdxZpCp6K3meyk/6T8amS3bHWU4xNjJxhS4kdY27uYzpaL4NiJvx/eJLDLcqL/uLvgI80hYSQ==", "requires": { "es6-promise": "3.0.2", "xmlhttprequest": "~1.8.0" - }, - "dependencies": { - "es6-promise": { - "version": "3.0.2", - "resolved": "http://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz", - "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=" - }, - "xmlhttprequest": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz", - "integrity": "sha1-Z/4HXFwk/vOfnWX197f+dRcZaPw=" - } + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" } }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "universal-user-agent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", + "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" + }, + "url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "xmlhttprequest": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz", + "integrity": "sha512-58Im/U0mlVBLM38NdZjHyhuMtCqa61469k2YP/AaPbvCoV9aQGUpbJBj1QRm2ytRiVQBD/fsw7L2bJGDVQswBA==" } } } diff --git a/package.json b/package.json index 4cfa793..3a6550d 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ }, "homepage": "https://github.com/UKHomeOffice/xgov-opensource-repo-scraper#readme", "dependencies": { + "@octokit/core": "^4.2.0", "bluebird": "^3.7.2", - "github": "^14.0.0", "js-yaml": "^3.13.1", "octokat": "^0.4.18" } From ca1ae4ea10ce3e5a993d50aa37472568607ac5af Mon Sep 17 00:00:00 2001 From: Chris Nesbitt-Smith Date: Thu, 27 Apr 2023 17:02:26 +0100 Subject: [PATCH 2/8] see file sizes in logs --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4521204..f2070ec 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,6 +27,9 @@ jobs: - run: node index.js env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: du -hs public/* + - run: gzip -9v public/sboms.json + - run: du -hs public/* - run: tar -cf public.tar ./public - uses: actions/upload-artifact@v3.1.0 with: From e1fa7e77d901b3280f97cb09e560e34f6d13de27 Mon Sep 17 00:00:00 2001 From: Chris Nesbitt-Smith Date: Thu, 27 Apr 2023 17:02:35 +0100 Subject: [PATCH 3/8] slow down --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index b359286..408fac4 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,7 @@ const allReposForAllUKDepartments = await Promise.mapSeries( ); const fetchAllSboms = async (repo) => { - await Promise.delay(20); //slow down to appease github rate limiting + await Promise.delay(50); //slow down to appease github rate limiting console.log(`Collecting SBOM for ${repo.owner.login}/${repo.name}`); try { return ( From c9ff67b36fe4789e9f12647a4ce1072def57c00e Mon Sep 17 00:00:00 2001 From: Chris Nesbitt-Smith Date: Fri, 28 Apr 2023 08:09:43 +0100 Subject: [PATCH 4/8] more slow --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 408fac4..3a6191b 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,7 @@ const allReposForAllUKDepartments = await Promise.mapSeries( ); const fetchAllSboms = async (repo) => { - await Promise.delay(50); //slow down to appease github rate limiting + await Promise.delay(250); //slow down to appease github rate limiting console.log(`Collecting SBOM for ${repo.owner.login}/${repo.name}`); try { return ( From c60821aa0b95b2f0c4bd0966918111e5e84ecc02 Mon Sep 17 00:00:00 2001 From: Chris Nesbitt-Smith Date: Fri, 28 Apr 2023 08:18:45 +0100 Subject: [PATCH 5/8] really slow --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 3a6191b..798044a 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,7 @@ const allReposForAllUKDepartments = await Promise.mapSeries( ); const fetchAllSboms = async (repo) => { - await Promise.delay(250); //slow down to appease github rate limiting + await Promise.delay(500); //slow down to appease github rate limiting console.log(`Collecting SBOM for ${repo.owner.login}/${repo.name}`); try { return ( From 8e00e1c6ebb38915b858aa07545d2b36572cdb53 Mon Sep 17 00:00:00 2001 From: Chris Nesbitt-Smith Date: Fri, 28 Apr 2023 08:21:05 +0100 Subject: [PATCH 6/8] super slow --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 798044a..c6ae3c2 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,7 @@ const allReposForAllUKDepartments = await Promise.mapSeries( ); const fetchAllSboms = async (repo) => { - await Promise.delay(500); //slow down to appease github rate limiting + await Promise.delay(1000); //slow down to appease github rate limiting console.log(`Collecting SBOM for ${repo.owner.login}/${repo.name}`); try { return ( From 01c8652269265cf61103aa9e119504d1012fc523 Mon Sep 17 00:00:00 2001 From: Chris Nesbitt-Smith Date: Fri, 28 Apr 2023 09:33:28 +0100 Subject: [PATCH 7/8] slower --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index c6ae3c2..d2aefd6 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,7 @@ const allReposForAllUKDepartments = await Promise.mapSeries( ); const fetchAllSboms = async (repo) => { - await Promise.delay(1000); //slow down to appease github rate limiting + await Promise.delay(1700); //slow down to appease github rate limiting console.log(`Collecting SBOM for ${repo.owner.login}/${repo.name}`); try { return ( From 45854544de98a1a7b9efe892a9e8189bf70a1cb3 Mon Sep 17 00:00:00 2001 From: Chris Nesbitt-Smith Date: Fri, 28 Apr 2023 19:43:56 +0100 Subject: [PATCH 8/8] really slow --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index d2aefd6..ff4058b 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,7 @@ const allReposForAllUKDepartments = await Promise.mapSeries( ); const fetchAllSboms = async (repo) => { - await Promise.delay(1700); //slow down to appease github rate limiting + await Promise.delay(5000); //slow down to appease github rate limiting console.log(`Collecting SBOM for ${repo.owner.login}/${repo.name}`); try { return (