Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cli: rate limit requests in load-test-extensions #963

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

cli: rate limit requests in load-test-extensions #963

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cli/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
"follow-redirects": "^1.14.6",
"is-ci": "^2.0.0",
"leven": "^3.1.0",
"limiter": "^2.1.0",
"semver": "^7.6.0",
"tmp": "^0.2.1"
},
Expand Down
36 changes: 22 additions & 14 deletions cli/scripts/load-test-extensions.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const { Registry, DEFAULT_URL } = require('../lib/registry');
const { makeDirs } = require('../lib/util');
const path = require('path');
const fs = require('fs');
const { RateLimiter } = require('limiter');

let searchSize = 100;
if (process.argv.length >= 3) {
Expand All @@ -24,42 +25,48 @@ if (process.argv.length >= 4) {
accessToken = process.argv[3];
}

const rateLimiter = new RateLimiter({ tokensPerInterval: 15, interval: 'second' });

async function loadTestExtensions() {
const publicReg = new Registry();
const localReg = new Registry({registryUrl: process.env.OVSX_REGISTRY_URL ?? 'http://localhost:8080'});
const localReg = new Registry({ registryUrl: process.env.OVSX_REGISTRY_URL ?? 'http://localhost:8080' });
/** @type {{ extensions: import('../lib/registry').Extension[] } & import('../lib/registry').Response} */
await rateLimiter.removeTokens(1);
const search = await publicReg.getJson(new URL(`${DEFAULT_URL}/api/-/search?size=${searchSize}`));
if (search.error) {
console.error(search.error);
process.exit(1);
}
console.log(`Found ${search.extensions.length} extensions in ${DEFAULT_URL}`);
for (const ext of search.extensions) {
await rateLimiter.removeTokens(1);
const meta = await publicReg.getMetadata(ext.namespace, ext.name);
if (meta.error) {
console.error(`\u274c ${meta.error}`);
continue;
}
const fileName = await download(publicReg, meta);
try {
const nsResult = await localReg.createNamespace(meta.namespace, accessToken);
console.log(nsResult.success);
await rateLimiter.removeTokens(1);
const nsResult = await localReg.createNamespace(meta.namespace, accessToken);
console.log(nsResult.success);
} catch (error) {
if (!error.message.startsWith('Namespace already exists')) {
console.error(error);
process.exit(1);
}
if (!error.message.startsWith('Namespace already exists')) {
console.error(error);
process.exit(1);
}
}

try {
const published = await localReg.publish(fileName, accessToken);
if (published.namespace && published.name) {
console.log(`\u2713 Published ${published.namespace}.${published.name}@${published.version}`);
}
await rateLimiter.removeTokens(1);
const published = await localReg.publish(fileName, accessToken);
if (published.namespace && published.name) {
console.log(`\u2713 Published ${published.namespace}.${published.name}@${published.version}`);
}
} catch (error) {
if (!error.message.endsWith('is already published.')) {
console.error(`\u274c ${error}`);
}
if (!error.message.endsWith('is already published.')) {
console.error(`\u274c ${error}`);
}
}
}
}
Expand All @@ -78,6 +85,7 @@ async function download(registry, extension) {
}
await makeDirs(path.dirname(filePath));
console.log(`Downloading ${extension.namespace}.${extension.name}@${extension.version} to ${filePath}`);
await rateLimiter.removeTokens(1);
await registry.download(filePath, new URL(downloadUrl));
return filePath;
}
Expand Down
17 changes: 17 additions & 0 deletions cli/yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1725,6 +1725,13 @@ __metadata:
languageName: node
linkType: hard

"just-performance@npm:4.3.0":
version: 4.3.0
resolution: "just-performance@npm:4.3.0"
checksum: 10/5e5f0664f357d291aac9ccca6357b997254600f2d2dd429e9b63e058035c44b6b1c73dbae51f58af7918c0701db4b20d4f365ed61e77d87c92a0183efb77a72a
languageName: node
linkType: hard

"keytar@npm:^7.7.0":
version: 7.9.0
resolution: "keytar@npm:7.9.0"
Expand Down Expand Up @@ -1762,6 +1769,15 @@ __metadata:
languageName: node
linkType: hard

"limiter@npm:^2.1.0":
version: 2.1.0
resolution: "limiter@npm:2.1.0"
dependencies:
just-performance: "npm:4.3.0"
checksum: 10/d6292c2a576fe7c4664652a64d286cb3701dd18e1878e01207840d809c62ce6db46ec6ee5e2ab53d548cb56152d0aeeb8555e32fe9177fcb4b04983ef04f805e
languageName: node
linkType: hard

"linkify-it@npm:^3.0.1":
version: 3.0.3
resolution: "linkify-it@npm:3.0.3"
Expand Down Expand Up @@ -2165,6 +2181,7 @@ __metadata:
follow-redirects: "npm:^1.14.6"
is-ci: "npm:^2.0.0"
leven: "npm:^3.1.0"
limiter: "npm:^2.1.0"
rimraf: "npm:^3.0.2"
semver: "npm:^7.6.0"
tmp: "npm:^0.2.1"
Expand Down
Loading