From 2da825585a091cb87b47ba54920263e22dedde4b Mon Sep 17 00:00:00 2001 From: Devin Jean Date: Tue, 2 Jan 2024 16:31:32 -0600 Subject: [PATCH 01/15] movebank auto-accept cc* licenses --- package.json | 2 + src/procedures/movebank/movebank.js | 159 ++++++++++++++++++++++++++++ src/procedures/utils/api-key.js | 4 + 3 files changed, 165 insertions(+) create mode 100644 src/procedures/movebank/movebank.js diff --git a/package.json b/package.json index b47efceb..94634b34 100644 --- a/package.json +++ b/package.json @@ -23,11 +23,13 @@ "cookie-parser": "^1.4.4", "doctrine": "^2.0.0", "express": "^4.18.1", + "fast-csv": "^4.3.6", "fs-extra": "^11.1.0", "geolib": "^2.0.18", "gnuplot": "^0.3.1", "jimp": "^0.16.1", "json-query": "^2.2.2", + "md5": "^2.3.0", "moment": "^2.29.4", "mongodb": "^3.7.3", "mongoose": "^5.10.15", diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js new file mode 100644 index 00000000..e174fbf0 --- /dev/null +++ b/src/procedures/movebank/movebank.js @@ -0,0 +1,159 @@ +/** + * Access to Movebank, a free, online database of animal tracking data hosted by the Max Planck Institute of Animal Behavior. + * + * @alpha + * @service + * @category GeoData + */ + +const logger = require("../utils/logger")("movebank"); +const ApiConsumer = require("../utils/api-consumer"); +const types = require("../../input-types"); +const csv = require("fast-csv"); +const md5 = require('md5'); +const axios = require("axios"); +const { MovebankKey } = require("../utils/api-key"); +const Movebank = new ApiConsumer( + "Movebank", + "https://www.movebank.org/movebank/service/direct-read/", + { cache: { ttl: 1 * 60 * 60 } }, +); +ApiConsumer.setRequiredApiKey(Movebank, MovebankKey); + +// some studies require us to accept a license before we can view the data, +// and there doesn't seem to be a way to filter these out before attempting to download the data. +// so we'll just auto-accept any licenses that are (specifically) of the following open license types. +const ALLOWED_LICENSE_TYPES = { + "CC_0": "CC0", + "CC_BY": "CC BY", + "CC_BY_NC": "CC BY-NC", + "CC_BY_SA": "CC BY-SA", + "CC_BY_NC_SA": "CC BY-NC-SA", +}; + +async function parseCSV(content) { + return new Promise((resolve, reject) => { + const res = []; + const stream = csv.parse({ headers: true, objectMode: true }) + .on("data", (x) => res.push(x)) + .on("error", (e) => reject(e)) + .on("end", () => resolve(res)); + + stream.write(content); + stream.end(); + }); +} + +async function fetchLicensed(settings, licenseHash = null) { + const licenseSuffix = licenseHash ? `&license-md5=${licenseHash}` : ''; + const url = `${Movebank._baseUrl}?${settings.queryString}${licenseSuffix}`; + logger.info(`fetching possibly licensed content: ${url}`); + + const res = await axios({ url, method: "GET" }); + if (!licenseHash && res.headers['accept-license'] === 'true') { + logger.info('> failed with license request'); + for (const ty in ALLOWED_LICENSE_TYPES) { + if (res.data.includes(`License Type: ${ALLOWED_LICENSE_TYPES[ty]}`)) { + logger.info(`> accepting license of type ${ty} and retrying...`); + return await fetchLicensed(settings, md5(res.data)); + } + } + logger.info(`> unknown license type`, res.data); + throw Error('failed to download licensed material'); + } + logger.info('> success!'); + return res.data; +} + +let SENSOR_TYPES_META = []; +types.defineType({ + name: "MovebankSensor", + description: "A sensor type used by :doc:`/services/Movebank/index`.", + baseType: "Enum", + baseParams: (async () => { + SENSOR_TYPES_META = await parseCSV(await Movebank._requestData({ + queryString: `entity_type=tag_type&api-token=${Movebank.apiKey.value}`, + })); + + const res = {}; + for (const ty of SENSOR_TYPES_META) { + res[ty.name] = parseInt(ty.id); + } + return res; + })(), +}); + +/** + * Get a list of all the sensor types supported by Movebank. + * + * @returns {Array} A list of supported sensor types + */ +Movebank.getSensorTypes = function () { + return SENSOR_TYPES_META.map((x) => x.name); +}; + +/** + * Get a list of all the studies available for (public) viewing. + * + * @returns {Array} A list of available studies + */ +Movebank.getStudies = async function () { + const data = await parseCSV(await Movebank._requestData({ + queryString: `entity_type=study&i_have_download_access=true&api-token=${Movebank.apiKey.value}`, + })); + + const res = []; + for (const raw of data) { + if (raw.id && raw.main_location_lat && raw.main_location_long && raw.citation && ALLOWED_LICENSE_TYPES[raw.license_type]) { + res.push({ + id: parseInt(raw.id), + latitude: parseFloat(raw.main_location_lat), + longitude: parseFloat(raw.main_location_long), + species: raw.taxon_ids.split(',').map((x) => x.trim()), + sensors: raw.sensor_type_ids.split(',').map((x) => x.trim()), + citation: raw.citation, + }); + } + } + return res; +}; + +/** + * Get a list of all the animals that participated in a specific study. + * + * @param {Union} study A study object or study id returned by :func:`Movebank.getStudies` + * @returns {Array} A list of animals + */ +Movebank.getAnimalsInStudy = async function (study) { + if (typeof(study) === "object") study = parseInt(study.id); + if (isNaN(study)) throw Error("invalid study"); + + const data = await parseCSV(await fetchLicensed({ + queryString: `entity_type=individual&study_id=${study}&api-token=${Movebank.apiKey.value}`, + })); + + console.log(SENSOR_TYPES_META); + + const res = [] + for (const raw of data) { + if (raw.id && raw.sex && raw.taxon_canonical_name) { + res.push({ + id: parseInt(raw.id), + sex: raw.sex === 'm' ? 'male' : raw.sex === 'f' ? 'female' : raw.sex, + species: raw.taxon_canonical_name, + sensors: raw.sensor_type_ids.split(',').map((x) => { + x = x.trim(); + for (const meta of SENSOR_TYPES_META) { + if (meta.external_id === x) { + return meta.name; + } + } + return x; + }), + }); + } + } + return res; +}; + +module.exports = Movebank; diff --git a/src/procedures/utils/api-key.js b/src/procedures/utils/api-key.js index 043b99a0..f9a8b11c 100644 --- a/src/procedures/utils/api-key.js +++ b/src/procedures/utils/api-key.js @@ -19,6 +19,10 @@ class ApiKey { } } +module.exports.MovebankKey = new ApiKey( + "Movebank", + "https://github.com/movebank/movebank-api-doc/blob/master/movebank-api.md#authenticate-by-token" +); module.exports.TimezoneDBKey = new ApiKey( "TimezoneDB", "https://timezonedb.com/register", From 6392b5d9f5868b3b2369782665d19d1d572ee726 Mon Sep 17 00:00:00 2001 From: Devin Jean Date: Wed, 3 Jan 2024 08:22:59 -0600 Subject: [PATCH 02/15] cache licensed requests --- src/procedures/movebank/movebank.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js index e174fbf0..0263b33f 100644 --- a/src/procedures/movebank/movebank.js +++ b/src/procedures/movebank/movebank.js @@ -49,20 +49,22 @@ async function fetchLicensed(settings, licenseHash = null) { const url = `${Movebank._baseUrl}?${settings.queryString}${licenseSuffix}`; logger.info(`fetching possibly licensed content: ${url}`); - const res = await axios({ url, method: "GET" }); - if (!licenseHash && res.headers['accept-license'] === 'true') { - logger.info('> failed with license request'); - for (const ty in ALLOWED_LICENSE_TYPES) { - if (res.data.includes(`License Type: ${ALLOWED_LICENSE_TYPES[ty]}`)) { - logger.info(`> accepting license of type ${ty} and retrying...`); - return await fetchLicensed(settings, md5(res.data)); + return await Movebank._cache.wrap(`::<${licenseHash}>::<${url}>`, async () => { + logger.info('> request is not cached - calling external endpoint'); + const res = await axios({ url, method: "GET" }); + if (!licenseHash && res.headers['accept-license'] === 'true') { + logger.info('> failed with license request'); + for (const ty in ALLOWED_LICENSE_TYPES) { + if (res.data.includes(`License Type: ${ALLOWED_LICENSE_TYPES[ty]}`)) { + logger.info(`> accepting license of type ${ty} and retrying...`); + return await fetchLicensed(settings, md5(res.data)); + } } + logger.info(`> unknown license type`, res.data); + throw Error('failed to download licensed material'); } - logger.info(`> unknown license type`, res.data); - throw Error('failed to download licensed material'); - } - logger.info('> success!'); - return res.data; + return res.data; + }); } let SENSOR_TYPES_META = []; @@ -132,8 +134,6 @@ Movebank.getAnimalsInStudy = async function (study) { queryString: `entity_type=individual&study_id=${study}&api-token=${Movebank.apiKey.value}`, })); - console.log(SENSOR_TYPES_META); - const res = [] for (const raw of data) { if (raw.id && raw.sex && raw.taxon_canonical_name) { From c4f5ecb4142734e4cea1456515b11ccba8982a96 Mon Sep 17 00:00:00 2001 From: Devin Jean Date: Wed, 3 Jan 2024 08:59:29 -0600 Subject: [PATCH 03/15] get sensor events --- src/procedures/movebank/movebank.js | 43 +++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js index 0263b33f..970c56cc 100644 --- a/src/procedures/movebank/movebank.js +++ b/src/procedures/movebank/movebank.js @@ -123,11 +123,11 @@ Movebank.getStudies = async function () { /** * Get a list of all the animals that participated in a specific study. * - * @param {Union} study A study object or study id returned by :func:`Movebank.getStudies` + * @param {Union} study A study object returned by :func:`Movebank.getStudies` * @returns {Array} A list of animals */ -Movebank.getAnimalsInStudy = async function (study) { - if (typeof(study) === "object") study = parseInt(study.id); +Movebank.getAnimals = async function (study) { + study = parseInt(study.id); if (isNaN(study)) throw Error("invalid study"); const data = await parseCSV(await fetchLicensed({ @@ -136,10 +136,10 @@ Movebank.getAnimalsInStudy = async function (study) { const res = [] for (const raw of data) { - if (raw.id && raw.sex && raw.taxon_canonical_name) { + if (raw.local_identifier && raw.taxon_canonical_name) { res.push({ - id: parseInt(raw.id), - sex: raw.sex === 'm' ? 'male' : raw.sex === 'f' ? 'female' : raw.sex, + id: raw.local_identifier, + sex: raw.sex === 'm' ? 'male' : raw.sex === 'f' ? 'female' : 'unknown', species: raw.taxon_canonical_name, sensors: raw.sensor_type_ids.split(',').map((x) => { x = x.trim(); @@ -156,4 +156,35 @@ Movebank.getAnimalsInStudy = async function (study) { return res; }; +/** + * Get a list of all the events for an animal in a specific study. + * + * @param {Union} study A study object returned by :func:`Movebank.getStudies` + * @param {Union} animal An animal object returned by :func:`Movebank.getAnimals`. The animal should be part of the same study. + * @returns {Array} A list of events for the animal + */ +Movebank.getEvents = async function (study, animal) { + study = parseInt(study.id); + if (isNaN(study)) throw Error("invalid study"); + + animal = animal.id.toString(); + if (!animal) throw Error("invalid animal"); + + const data = await parseCSV(await fetchLicensed({ + queryString: `entity_type=event&study_id=${study}&individual_local_identifier=${animal}&attributes=visible,timestamp,location_lat,location_long&api-token=${Movebank.apiKey.value}`, + })); + + const res = []; + for (const raw of data) { + if (raw.visible === 'true' && raw.timestamp && raw.location_lat && raw.location_long) { + res.push({ + timestamp: new Date(raw.timestamp), + latitude: parseFloat(raw.location_lat), + longitude: parseFloat(raw.location_long), + }); + } + } + return res; +}; + module.exports = Movebank; From d2e0cabe8b95808e848c4403ee93c14edc431b27 Mon Sep 17 00:00:00 2001 From: Devin Jean Date: Wed, 3 Jan 2024 10:44:01 -0600 Subject: [PATCH 04/15] filter studies by proximity --- src/procedures/movebank/movebank.js | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js index 970c56cc..62a5ff5c 100644 --- a/src/procedures/movebank/movebank.js +++ b/src/procedures/movebank/movebank.js @@ -8,11 +8,13 @@ const logger = require("../utils/logger")("movebank"); const ApiConsumer = require("../utils/api-consumer"); +const { MovebankKey } = require("../utils/api-key"); const types = require("../../input-types"); +const geolib = require("geolib"); const csv = require("fast-csv"); -const md5 = require('md5'); const axios = require("axios"); -const { MovebankKey } = require("../utils/api-key"); +const md5 = require('md5'); + const Movebank = new ApiConsumer( "Movebank", "https://www.movebank.org/movebank/service/direct-read/", @@ -120,6 +122,20 @@ Movebank.getStudies = async function () { return res; }; +/** + * Get a list of all the studies available for (public) viewing within a certain max distance of a point of interest. + * Note that some of the animals involved in these studies may travel outside of this distance. + * + * @param {Latitude} latitude Latitude of the point of interest + * @param {Longitude} longitude Longitude of the point of interest + * @param {BoundedNumber<0>} distance Max distance from the point of interest (in meters) + * @returns {Array} A list of available studies near the point of interest + */ +Movebank.getStudiesNear = async function (latitude, longitude, distance) { + const p = { latitude, longitude }; + return (await Movebank.getStudies()).filter((x) => geolib.getDistance(p, { latitude: x.latitude, longitude: x.longitude }) <= distance); +}; + /** * Get a list of all the animals that participated in a specific study. * From dec673d9b6d7bc4b7b789a6baad87ceb54b77114 Mon Sep 17 00:00:00 2001 From: Devin Jean Date: Wed, 3 Jan 2024 13:30:43 -0600 Subject: [PATCH 05/15] event distance delta filtering --- src/procedures/movebank/movebank.js | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js index 62a5ff5c..5c70920d 100644 --- a/src/procedures/movebank/movebank.js +++ b/src/procedures/movebank/movebank.js @@ -173,13 +173,14 @@ Movebank.getAnimals = async function (study) { }; /** - * Get a list of all the events for an animal in a specific study. + * Get a chronological list of all the migration events for an animal in a specific study. * * @param {Union} study A study object returned by :func:`Movebank.getStudies` * @param {Union} animal An animal object returned by :func:`Movebank.getAnimals`. The animal should be part of the same study. - * @returns {Array} A list of events for the animal + * @param {BoundedNumber<0>=} minDistance The minimum distance (in meters) between consecutive returned events (default 0, which gives all available data). + * @returns {Array} A list of chronological migration events for the animal */ -Movebank.getEvents = async function (study, animal) { +Movebank.getEvents = async function (study, animal, minDistance = 0) { study = parseInt(study.id); if (isNaN(study)) throw Error("invalid study"); @@ -191,13 +192,24 @@ Movebank.getEvents = async function (study, animal) { })); const res = []; + let prevPos = null; for (const raw of data) { if (raw.visible === 'true' && raw.timestamp && raw.location_lat && raw.location_long) { - res.push({ + const entry = { timestamp: new Date(raw.timestamp), latitude: parseFloat(raw.location_lat), longitude: parseFloat(raw.location_long), - }); + }; + + if (minDistance > 0) { + const pos = { latitude: entry.latitude, longitude: entry.longitude }; + if (!prevPos || geolib.getDistance(prevPos, pos) >= minDistance) { + prevPos = pos; + res.push(entry); + } + } else { + res.push(entry); + } } } return res; From 1490d8e3f2c08723d0c357dc193acb9b4835a479 Mon Sep 17 00:00:00 2001 From: Devin Jean Date: Wed, 3 Jan 2024 17:52:54 -0600 Subject: [PATCH 06/15] graceful errors --- src/procedures/movebank/movebank.js | 40 ++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js index 5c70920d..b667088c 100644 --- a/src/procedures/movebank/movebank.js +++ b/src/procedures/movebank/movebank.js @@ -69,6 +69,14 @@ async function fetchLicensed(settings, licenseHash = null) { }); } +async function tryOrElse(ok, err) { + try { + return await ok(); + } catch (e) { + return await err(); + } +} + let SENSOR_TYPES_META = []; types.defineType({ name: "MovebankSensor", @@ -139,15 +147,19 @@ Movebank.getStudiesNear = async function (latitude, longitude, distance) { /** * Get a list of all the animals that participated in a specific study. * - * @param {Union} study A study object returned by :func:`Movebank.getStudies` + * @param {Object} study A study object returned by :func:`Movebank.getStudies` * @returns {Array} A list of animals */ Movebank.getAnimals = async function (study) { study = parseInt(study.id); - if (isNaN(study)) throw Error("invalid study"); - - const data = await parseCSV(await fetchLicensed({ - queryString: `entity_type=individual&study_id=${study}&api-token=${Movebank.apiKey.value}`, + if (isNaN(study)) throw Error("unknown study"); + + const data = await parseCSV(await tryOrElse(async () => { + return await fetchLicensed({ + queryString: `entity_type=individual&study_id=${study}&api-token=${Movebank.apiKey.value}`, + }); + }, () => { + return ""; })); const res = [] @@ -175,20 +187,24 @@ Movebank.getAnimals = async function (study) { /** * Get a chronological list of all the migration events for an animal in a specific study. * - * @param {Union} study A study object returned by :func:`Movebank.getStudies` - * @param {Union} animal An animal object returned by :func:`Movebank.getAnimals`. The animal should be part of the same study. + * @param {Object} study A study object returned by :func:`Movebank.getStudies` + * @param {Object} animal An animal object returned by :func:`Movebank.getAnimals`. The animal should be part of the same study. * @param {BoundedNumber<0>=} minDistance The minimum distance (in meters) between consecutive returned events (default 0, which gives all available data). * @returns {Array} A list of chronological migration events for the animal */ Movebank.getEvents = async function (study, animal, minDistance = 0) { study = parseInt(study.id); - if (isNaN(study)) throw Error("invalid study"); + if (isNaN(study)) throw Error("unknown study"); animal = animal.id.toString(); - if (!animal) throw Error("invalid animal"); - - const data = await parseCSV(await fetchLicensed({ - queryString: `entity_type=event&study_id=${study}&individual_local_identifier=${animal}&attributes=visible,timestamp,location_lat,location_long&api-token=${Movebank.apiKey.value}`, + if (!animal) throw Error("unknown animal"); + + const data = await parseCSV(await tryOrElse(async () => { + return await fetchLicensed({ + queryString: `entity_type=event&study_id=${study}&individual_local_identifier=${animal}&attributes=visible,timestamp,location_lat,location_long&api-token=${Movebank.apiKey.value}`, + }); + }, () => { + return ""; })); const res = []; From 794e31cf9b21fbb792c68efc034b78b315dc181a Mon Sep 17 00:00:00 2001 From: Devin Jean Date: Wed, 3 Jan 2024 17:59:30 -0600 Subject: [PATCH 07/15] interface tests --- test/procedures/movebank.spec.js | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 test/procedures/movebank.spec.js diff --git a/test/procedures/movebank.spec.js b/test/procedures/movebank.spec.js new file mode 100644 index 00000000..a18aa47e --- /dev/null +++ b/test/procedures/movebank.spec.js @@ -0,0 +1,11 @@ +const utils = require("../assets/utils"); + +describe(utils.suiteName(__filename), function () { + utils.verifyRPCInterfaces("Movebank", [ + ["getSensorTypes", []], + ["getStudies", []], + ["getStudiesNear", ["latitude", "longitude", "distance"]], + ["getAnimals", ["study"]], + ["getEvents", ["study", "animal", "minDistance"]], + ]); +}); From e95c99eaef03bf6ff2ed59dbba5c7e25a5fc65ef Mon Sep 17 00:00:00 2001 From: Format Bot Date: Thu, 4 Jan 2024 00:11:26 +0000 Subject: [PATCH 08/15] Fix code formatting --- src/procedures/movebank/movebank.js | 307 +++++++++++++++------------- src/procedures/utils/api-key.js | 2 +- 2 files changed, 169 insertions(+), 140 deletions(-) diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js index b667088c..8477e96a 100644 --- a/src/procedures/movebank/movebank.js +++ b/src/procedures/movebank/movebank.js @@ -13,12 +13,12 @@ const types = require("../../input-types"); const geolib = require("geolib"); const csv = require("fast-csv"); const axios = require("axios"); -const md5 = require('md5'); +const md5 = require("md5"); const Movebank = new ApiConsumer( - "Movebank", - "https://www.movebank.org/movebank/service/direct-read/", - { cache: { ttl: 1 * 60 * 60 } }, + "Movebank", + "https://www.movebank.org/movebank/service/direct-read/", + { cache: { ttl: 1 * 60 * 60 } }, ); ApiConsumer.setRequiredApiKey(Movebank, MovebankKey); @@ -26,209 +26,238 @@ ApiConsumer.setRequiredApiKey(Movebank, MovebankKey); // and there doesn't seem to be a way to filter these out before attempting to download the data. // so we'll just auto-accept any licenses that are (specifically) of the following open license types. const ALLOWED_LICENSE_TYPES = { - "CC_0": "CC0", - "CC_BY": "CC BY", - "CC_BY_NC": "CC BY-NC", - "CC_BY_SA": "CC BY-SA", - "CC_BY_NC_SA": "CC BY-NC-SA", + "CC_0": "CC0", + "CC_BY": "CC BY", + "CC_BY_NC": "CC BY-NC", + "CC_BY_SA": "CC BY-SA", + "CC_BY_NC_SA": "CC BY-NC-SA", }; async function parseCSV(content) { - return new Promise((resolve, reject) => { - const res = []; - const stream = csv.parse({ headers: true, objectMode: true }) - .on("data", (x) => res.push(x)) - .on("error", (e) => reject(e)) - .on("end", () => resolve(res)); - - stream.write(content); - stream.end(); - }); + return new Promise((resolve, reject) => { + const res = []; + const stream = csv.parse({ headers: true, objectMode: true }) + .on("data", (x) => res.push(x)) + .on("error", (e) => reject(e)) + .on("end", () => resolve(res)); + + stream.write(content); + stream.end(); + }); } async function fetchLicensed(settings, licenseHash = null) { - const licenseSuffix = licenseHash ? `&license-md5=${licenseHash}` : ''; - const url = `${Movebank._baseUrl}?${settings.queryString}${licenseSuffix}`; - logger.info(`fetching possibly licensed content: ${url}`); - - return await Movebank._cache.wrap(`::<${licenseHash}>::<${url}>`, async () => { - logger.info('> request is not cached - calling external endpoint'); - const res = await axios({ url, method: "GET" }); - if (!licenseHash && res.headers['accept-license'] === 'true') { - logger.info('> failed with license request'); - for (const ty in ALLOWED_LICENSE_TYPES) { - if (res.data.includes(`License Type: ${ALLOWED_LICENSE_TYPES[ty]}`)) { - logger.info(`> accepting license of type ${ty} and retrying...`); - return await fetchLicensed(settings, md5(res.data)); - } - } - logger.info(`> unknown license type`, res.data); - throw Error('failed to download licensed material'); + const licenseSuffix = licenseHash ? `&license-md5=${licenseHash}` : ""; + const url = `${Movebank._baseUrl}?${settings.queryString}${licenseSuffix}`; + logger.info(`fetching possibly licensed content: ${url}`); + + return await Movebank._cache.wrap( + `::<${licenseHash}>::<${url}>`, + async () => { + logger.info("> request is not cached - calling external endpoint"); + const res = await axios({ url, method: "GET" }); + if (!licenseHash && res.headers["accept-license"] === "true") { + logger.info("> failed with license request"); + for (const ty in ALLOWED_LICENSE_TYPES) { + if ( + res.data.includes( + `License Type: ${ + ALLOWED_LICENSE_TYPES[ty] + }`, + ) + ) { + logger.info(`> accepting license of type ${ty} and retrying...`); + return await fetchLicensed(settings, md5(res.data)); + } } - return res.data; - }); + logger.info(`> unknown license type`, res.data); + throw Error("failed to download licensed material"); + } + return res.data; + }, + ); } async function tryOrElse(ok, err) { - try { - return await ok(); - } catch (e) { - return await err(); - } + try { + return await ok(); + } catch (e) { + return await err(); + } } let SENSOR_TYPES_META = []; types.defineType({ - name: "MovebankSensor", - description: "A sensor type used by :doc:`/services/Movebank/index`.", - baseType: "Enum", - baseParams: (async () => { - SENSOR_TYPES_META = await parseCSV(await Movebank._requestData({ - queryString: `entity_type=tag_type&api-token=${Movebank.apiKey.value}`, - })); - - const res = {}; - for (const ty of SENSOR_TYPES_META) { - res[ty.name] = parseInt(ty.id); - } - return res; - })(), + name: "MovebankSensor", + description: "A sensor type used by :doc:`/services/Movebank/index`.", + baseType: "Enum", + baseParams: (async () => { + SENSOR_TYPES_META = await parseCSV( + await Movebank._requestData({ + queryString: `entity_type=tag_type&api-token=${Movebank.apiKey.value}`, + }), + ); + + const res = {}; + for (const ty of SENSOR_TYPES_META) { + res[ty.name] = parseInt(ty.id); + } + return res; + })(), }); /** * Get a list of all the sensor types supported by Movebank. - * + * * @returns {Array} A list of supported sensor types */ Movebank.getSensorTypes = function () { - return SENSOR_TYPES_META.map((x) => x.name); + return SENSOR_TYPES_META.map((x) => x.name); }; /** * Get a list of all the studies available for (public) viewing. - * + * * @returns {Array} A list of available studies */ Movebank.getStudies = async function () { - const data = await parseCSV(await Movebank._requestData({ - queryString: `entity_type=study&i_have_download_access=true&api-token=${Movebank.apiKey.value}`, - })); + const data = await parseCSV( + await Movebank._requestData({ + queryString: + `entity_type=study&i_have_download_access=true&api-token=${Movebank.apiKey.value}`, + }), + ); - const res = []; - for (const raw of data) { - if (raw.id && raw.main_location_lat && raw.main_location_long && raw.citation && ALLOWED_LICENSE_TYPES[raw.license_type]) { - res.push({ - id: parseInt(raw.id), - latitude: parseFloat(raw.main_location_lat), - longitude: parseFloat(raw.main_location_long), - species: raw.taxon_ids.split(',').map((x) => x.trim()), - sensors: raw.sensor_type_ids.split(',').map((x) => x.trim()), - citation: raw.citation, - }); - } + const res = []; + for (const raw of data) { + if ( + raw.id && raw.main_location_lat && raw.main_location_long && + raw.citation && ALLOWED_LICENSE_TYPES[raw.license_type] + ) { + res.push({ + id: parseInt(raw.id), + latitude: parseFloat(raw.main_location_lat), + longitude: parseFloat(raw.main_location_long), + species: raw.taxon_ids.split(",").map((x) => x.trim()), + sensors: raw.sensor_type_ids.split(",").map((x) => x.trim()), + citation: raw.citation, + }); } - return res; + } + return res; }; /** * Get a list of all the studies available for (public) viewing within a certain max distance of a point of interest. * Note that some of the animals involved in these studies may travel outside of this distance. - * + * * @param {Latitude} latitude Latitude of the point of interest * @param {Longitude} longitude Longitude of the point of interest * @param {BoundedNumber<0>} distance Max distance from the point of interest (in meters) * @returns {Array} A list of available studies near the point of interest */ Movebank.getStudiesNear = async function (latitude, longitude, distance) { - const p = { latitude, longitude }; - return (await Movebank.getStudies()).filter((x) => geolib.getDistance(p, { latitude: x.latitude, longitude: x.longitude }) <= distance); + const p = { latitude, longitude }; + return (await Movebank.getStudies()).filter((x) => + geolib.getDistance(p, { latitude: x.latitude, longitude: x.longitude }) <= + distance + ); }; /** * Get a list of all the animals that participated in a specific study. - * + * * @param {Object} study A study object returned by :func:`Movebank.getStudies` * @returns {Array} A list of animals */ Movebank.getAnimals = async function (study) { - study = parseInt(study.id); - if (isNaN(study)) throw Error("unknown study"); + study = parseInt(study.id); + if (isNaN(study)) throw Error("unknown study"); - const data = await parseCSV(await tryOrElse(async () => { - return await fetchLicensed({ - queryString: `entity_type=individual&study_id=${study}&api-token=${Movebank.apiKey.value}`, - }); + const data = await parseCSV( + await tryOrElse(async () => { + return await fetchLicensed({ + queryString: + `entity_type=individual&study_id=${study}&api-token=${Movebank.apiKey.value}`, + }); }, () => { - return ""; - })); - - const res = [] - for (const raw of data) { - if (raw.local_identifier && raw.taxon_canonical_name) { - res.push({ - id: raw.local_identifier, - sex: raw.sex === 'm' ? 'male' : raw.sex === 'f' ? 'female' : 'unknown', - species: raw.taxon_canonical_name, - sensors: raw.sensor_type_ids.split(',').map((x) => { - x = x.trim(); - for (const meta of SENSOR_TYPES_META) { - if (meta.external_id === x) { - return meta.name; - } - } - return x; - }), - }); - } + return ""; + }), + ); + + const res = []; + for (const raw of data) { + if (raw.local_identifier && raw.taxon_canonical_name) { + res.push({ + id: raw.local_identifier, + sex: raw.sex === "m" ? "male" : raw.sex === "f" ? "female" : "unknown", + species: raw.taxon_canonical_name, + sensors: raw.sensor_type_ids.split(",").map((x) => { + x = x.trim(); + for (const meta of SENSOR_TYPES_META) { + if (meta.external_id === x) { + return meta.name; + } + } + return x; + }), + }); } - return res; + } + return res; }; /** * Get a chronological list of all the migration events for an animal in a specific study. - * + * * @param {Object} study A study object returned by :func:`Movebank.getStudies` * @param {Object} animal An animal object returned by :func:`Movebank.getAnimals`. The animal should be part of the same study. * @param {BoundedNumber<0>=} minDistance The minimum distance (in meters) between consecutive returned events (default 0, which gives all available data). * @returns {Array} A list of chronological migration events for the animal */ Movebank.getEvents = async function (study, animal, minDistance = 0) { - study = parseInt(study.id); - if (isNaN(study)) throw Error("unknown study"); + study = parseInt(study.id); + if (isNaN(study)) throw Error("unknown study"); - animal = animal.id.toString(); - if (!animal) throw Error("unknown animal"); + animal = animal.id.toString(); + if (!animal) throw Error("unknown animal"); - const data = await parseCSV(await tryOrElse(async () => { - return await fetchLicensed({ - queryString: `entity_type=event&study_id=${study}&individual_local_identifier=${animal}&attributes=visible,timestamp,location_lat,location_long&api-token=${Movebank.apiKey.value}`, - }); + const data = await parseCSV( + await tryOrElse(async () => { + return await fetchLicensed({ + queryString: + `entity_type=event&study_id=${study}&individual_local_identifier=${animal}&attributes=visible,timestamp,location_lat,location_long&api-token=${Movebank.apiKey.value}`, + }); }, () => { - return ""; - })); + return ""; + }), + ); - const res = []; - let prevPos = null; - for (const raw of data) { - if (raw.visible === 'true' && raw.timestamp && raw.location_lat && raw.location_long) { - const entry = { - timestamp: new Date(raw.timestamp), - latitude: parseFloat(raw.location_lat), - longitude: parseFloat(raw.location_long), - }; - - if (minDistance > 0) { - const pos = { latitude: entry.latitude, longitude: entry.longitude }; - if (!prevPos || geolib.getDistance(prevPos, pos) >= minDistance) { - prevPos = pos; - res.push(entry); - } - } else { - res.push(entry); - } + const res = []; + let prevPos = null; + for (const raw of data) { + if ( + raw.visible === "true" && raw.timestamp && raw.location_lat && + raw.location_long + ) { + const entry = { + timestamp: new Date(raw.timestamp), + latitude: parseFloat(raw.location_lat), + longitude: parseFloat(raw.location_long), + }; + + if (minDistance > 0) { + const pos = { latitude: entry.latitude, longitude: entry.longitude }; + if (!prevPos || geolib.getDistance(prevPos, pos) >= minDistance) { + prevPos = pos; + res.push(entry); } + } else { + res.push(entry); + } } - return res; + } + return res; }; module.exports = Movebank; diff --git a/src/procedures/utils/api-key.js b/src/procedures/utils/api-key.js index f9a8b11c..10049caa 100644 --- a/src/procedures/utils/api-key.js +++ b/src/procedures/utils/api-key.js @@ -21,7 +21,7 @@ class ApiKey { module.exports.MovebankKey = new ApiKey( "Movebank", - "https://github.com/movebank/movebank-api-doc/blob/master/movebank-api.md#authenticate-by-token" + "https://github.com/movebank/movebank-api-doc/blob/master/movebank-api.md#authenticate-by-token", ); module.exports.TimezoneDBKey = new ApiKey( "TimezoneDB", From 6b46f929e7bc35e3c8bd891f18389711a26b7cc1 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Fri, 26 Jan 2024 13:52:38 -0600 Subject: [PATCH 09/15] Add docs for cloud, extensions url params --- docs/adv/url.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/adv/url.rst b/docs/adv/url.rst index 3a453bb8..7dacfc2d 100644 --- a/docs/adv/url.rst +++ b/docs/adv/url.rst @@ -61,3 +61,5 @@ The following settings are generally available, regardless of the action: - **noExitWarning**: Don't confirm that the user wants to leave the page on tab close. - **lang**: Set the language immediately. For example, "lang=hu" will set NetsBlox to Hungarian on start. - **setVariable**: Set a variable to the given value on start. The value is expected to be a URL-encoded pair so setting "hello" to "world" would be "hello%3Dworld". +- **cloud**: Override the default cloud URL. This can be used to point the public deployment to your own local cloud (e.g, `cloud=http://localhost:7777`). +- **extensions**: A list of extensions (by URL) to load on start. From 1e6394beb541fa4a1e8e58905487567c06012f74 Mon Sep 17 00:00:00 2001 From: Devin Jean Date: Mon, 5 Feb 2024 14:20:03 -0600 Subject: [PATCH 10/15] update lock file --- package-lock.json | 112 ++++++++++++++++++++++++++++ src/procedures/movebank/movebank.js | 14 ++-- 2 files changed, 121 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5e9480ab..133b2ff2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,11 +22,13 @@ "cookie-parser": "^1.4.4", "doctrine": "^2.0.0", "express": "^4.18.1", + "fast-csv": "^4.3.6", "fs-extra": "^11.1.0", "geolib": "^2.0.18", "gnuplot": "^0.3.1", "jimp": "^0.16.1", "json-query": "^2.2.2", + "md5": "^2.3.0", "moment": "^2.29.4", "mongodb": "^3.7.3", "mongoose": "^5.10.15", @@ -69,6 +71,43 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==" }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + }, "node_modules/@jimp/bmp": { "version": "0.16.1", "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.16.1.tgz", @@ -1653,6 +1692,14 @@ "node": ">=4" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "engines": { + "node": "*" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -2059,6 +2106,14 @@ "node": ">=0.8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "engines": { + "node": "*" + } + }, "node_modules/crypto-browserify": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", @@ -2712,6 +2767,18 @@ "node": "> 0.1.90" } }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4241,6 +4308,41 @@ "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==" }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==" + }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==" + }, "node_modules/lodash.memoize": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz", @@ -4398,6 +4500,16 @@ "semver": "bin/semver.js" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js index 8477e96a..fe991a0c 100644 --- a/src/procedures/movebank/movebank.js +++ b/src/procedures/movebank/movebank.js @@ -92,11 +92,15 @@ types.defineType({ description: "A sensor type used by :doc:`/services/Movebank/index`.", baseType: "Enum", baseParams: (async () => { - SENSOR_TYPES_META = await parseCSV( - await Movebank._requestData({ - queryString: `entity_type=tag_type&api-token=${Movebank.apiKey.value}`, - }), - ); + try { + SENSOR_TYPES_META = await parseCSV( + await Movebank._requestData({ + queryString: `entity_type=tag_type&api-token=${Movebank.apiKey.value}`, + }), + ); + } catch (e) { + console.error('failed to load MoveBank sensor types', e); + } const res = {}; for (const ty of SENSOR_TYPES_META) { From 74236bd4faae9ea81db3cd45997e7de7fab162ee Mon Sep 17 00:00:00 2001 From: Format Bot Date: Mon, 5 Feb 2024 20:20:33 +0000 Subject: [PATCH 11/15] Fix code formatting --- src/procedures/movebank/movebank.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js index fe991a0c..0689c97b 100644 --- a/src/procedures/movebank/movebank.js +++ b/src/procedures/movebank/movebank.js @@ -95,11 +95,12 @@ types.defineType({ try { SENSOR_TYPES_META = await parseCSV( await Movebank._requestData({ - queryString: `entity_type=tag_type&api-token=${Movebank.apiKey.value}`, + queryString: + `entity_type=tag_type&api-token=${Movebank.apiKey.value}`, }), ); } catch (e) { - console.error('failed to load MoveBank sensor types', e); + console.error("failed to load MoveBank sensor types", e); } const res = {}; From 6b7a54dc5ed74fcf026ab4cda662c5ab4df5a19c Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Mon, 5 Feb 2024 14:29:05 -0600 Subject: [PATCH 12/15] update copyright year --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 70aba35f..2380db79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = 'NetsBlox' -copyright = '2021, Vanderbilt University' +copyright = '2024, Vanderbilt University' author = 'NetsBlox Team' # -- General configuration --------------------------------------------------- From 8173cde966d3c67886822280c686539d3ed24a2b Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 6 Feb 2024 16:40:44 -0600 Subject: [PATCH 13/15] Update alexa routes to use new cloud --- src/cloud-client.js | 4 + src/error.js | 9 +- src/procedures/alexa/routes.js | 20 ++-- src/procedures/music-app/SoundLibrary | 1 - src/procedures/music-app/music-app.js | 143 -------------------------- src/procedures/music-app/types.js | 133 ------------------------ 6 files changed, 20 insertions(+), 290 deletions(-) delete mode 160000 src/procedures/music-app/SoundLibrary delete mode 100644 src/procedures/music-app/music-app.js delete mode 100644 src/procedures/music-app/types.js diff --git a/src/cloud-client.js b/src/cloud-client.js index 63bd6144..2e6a3767 100644 --- a/src/cloud-client.js +++ b/src/cloud-client.js @@ -118,6 +118,10 @@ class NetsBloxCloud { return clients; } + async getOAuthToken(tokenId) { + return await this.get(`/oauth/token/${tokenId}`); + } + isConfigured() { return this.cloudUrl && this.id && this.secret; } diff --git a/src/error.js b/src/error.js index 13214e8e..4967daa8 100644 --- a/src/error.js +++ b/src/error.js @@ -27,7 +27,7 @@ class MissingFieldError extends RequestError { } function handleUserErrors(fn) { - return async function (req, res) { + return async function (_req, res) { try { await fn.call(this, ...arguments); } catch (err) { @@ -40,11 +40,18 @@ function handleUserErrors(fn) { }; } +class LoginRequired extends RequestError { + constructor() { + super(401, "Login Required."); + } +} + module.exports = { UserError, NotAllowedError, InvalidKeyProviderError, MissingFieldError, + LoginRequired, handleUserErrors, }; diff --git a/src/procedures/alexa/routes.js b/src/procedures/alexa/routes.js index d405554e..2c504ea1 100644 --- a/src/procedures/alexa/routes.js +++ b/src/procedures/alexa/routes.js @@ -1,14 +1,10 @@ const AlexaSkill = require("./skill"); const express = require("express"); -module.exports = express(); // FIXME: add support for OAuth on NetsBlox cloud -return; -// TODO: refactor the next imports -const OAuth = require("../../../api/core/oauth"); -// FIXME: TODO: Update this -const { handleErrors } = require("../../../api/rest/utils"); +const NetsBloxCloud = require("../../cloud-client"); +const { handleUserErrors } = require("../../error"); const { setUsernameFromCookie } = require("../utils/router-utils"); -const { LoginRequired, RequestError } = require("../../../api/core/errors"); +const { LoginRequired, RequestError } = require("../../error"); const bodyParser = require("body-parser"); const axios = require("axios"); @@ -27,13 +23,13 @@ const logger = require("../utils/logger")("alexa:routes"); const router = express(); const parseCookies = cookieParser(); -router.get("/ping", (req, res) => res.send("pong")); +router.get("/ping", (_req, res) => res.send("pong")); router.get( "/login.html", bodyParser.json(), parseCookies, setUsernameFromCookie, - handleErrors((req, res) => { + handleUserErrors((req, res) => { const username = req.session.username; const isLoggedIn = !!username; @@ -63,7 +59,7 @@ router.put( bodyParser.json(), parseCookies, setUsernameFromCookie, - handleErrors(async (req, res) => { + handleUserErrors(async (req, res) => { const { username } = req.session; const isLoggedIn = !!username; @@ -126,7 +122,7 @@ router.post( handleErrorsInAlexa(async (req, res) => { const reqData = req.body; const { accessToken } = reqData.session.user; - const token = await OAuth.getToken(accessToken); + const token = await NetsBloxCloud.getToken(accessToken); const { username } = token; const skillId = reqData.session.application.applicationId; @@ -194,7 +190,7 @@ function speak(text) { } function handleErrorsInAlexa(fn) { - return async function (req, res) { + return async function (_req, res) { try { await fn(...arguments); } catch (err) { diff --git a/src/procedures/music-app/SoundLibrary b/src/procedures/music-app/SoundLibrary deleted file mode 160000 index 437e8650..00000000 --- a/src/procedures/music-app/SoundLibrary +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 437e86509b176faa23139893c2b441aa199dfeb3 diff --git a/src/procedures/music-app/music-app.js b/src/procedures/music-app/music-app.js deleted file mode 100644 index 7fa9e6ed..00000000 --- a/src/procedures/music-app/music-app.js +++ /dev/null @@ -1,143 +0,0 @@ -/** - * This service allows users to play songs. - * @alpha - * @service - * @category Music - */ - -"use strict"; - -const fsp = require("fs/promises"); -const { registerTypes } = require("./types"); -const path = require("path"); -const utils = require("../utils/index"); -const MusicApp = {}; -const soundLibrary = require("./SoundLibrary/soundLibrary.json"); -const drumLibrary = require("./SoundLibrary/drumSoundLibrary.json"); -const masterSoundLibrary = [ - ...soundLibrary.netsbloxSoundLibrary, - ...drumLibrary.drumSoundLibrary, -]; - -registerTypes(); - -MusicApp._filetoBuffer = async function (audio_path) { - const data = await fsp.readFile(audio_path); - utils.sendAudioBuffer(this.response, data); -}; - -/** - * Get Sounds based on query. - * @param {String=} soundType - * @returns {Array} - */ -MusicApp._getNamesBySoundType = async function (soundType = "") { - var names = []; - - //Filter SoundCategories JSON by soundType - const queriedJSON = soundLibrary.netsbloxSoundLibrary.filter((obj) => - obj.SoundType === soundType.toUpperCase() - ); - - //Convert JSON to array of String names - for (let i = 0; i < queriedJSON.length; i++) { - names.push(queriedJSON[i].Name); - } - - return names; -}; - -/** - * Get sounds based on query. - * @param {DrumPackName=} packName - * @param {DrumOneShotTypes=} drumType - * @returns {String} - */ -MusicApp.getDrumOneShotNames = async function ( - packName = "", - drumType = "", -) { - var names = []; - let queriedJSON = ""; - - //Ensure at least one field is selected - if (packName !== "" || drumType !== "") { - queriedJSON = drumLibrary.drumSoundLibrary.filter(function (obj) { // Check if field value is empty before finding obj with value. - return (packName === "" || obj.packName === packName) && - (drumType === "" || obj.Instrument === drumType); - }); - } else { - throw Error("At least one field must be selected"); - } - - //Convert JSON to array of String names - for (let i = 0; i < queriedJSON.length; i++) { - names.push(queriedJSON[i].soundName); - } - return names; -}; - -/** - * Get sounds based on query. - * @param {Chords=} chords - * @param {Keys=} key - * @param {BPM=} bpm - * @param {InstrumentNames=} instrumentName - * @returns {Array} - */ -MusicApp.getSoundNames = async function ( - chords = "", - key = "", - bpm = "", - instrumentName = "", -) { - var names = []; - let queriedJSON = ""; - - //Ensure at least one field is selected - if (chords !== "" || key !== "" || bpm !== "" || instrumentName !== "") { - queriedJSON = soundLibrary.netsbloxSoundLibrary.filter(function (obj) { // Check if field value is empty before finding obj with value. - return (instrumentName === "" || obj.InstrumentName === instrumentName) && - (bpm === "" || obj.BPM === bpm) && - (key === "" || obj.Key === key) && - (chords === "" || obj.ChordProgression === chords); - }); - } else { - throw Error("At least one field must be selected"); - } - - //Convert JSON to array of String names - for (let i = 0; i < queriedJSON.length; i++) { - names.push(queriedJSON[i].soundName); - } - - return names; -}; - -/** - * Get sound by name. - * @param {String=} nameOfSound - */ -MusicApp.nameToSound = async function (nameOfSound = "") { - const metadata = masterSoundLibrary - .find((obj) => obj.soundName === nameOfSound); - - if (metadata) { - const audioPath = path.join(__dirname, metadata.Path); - const data = await fsp.readFile(audioPath); - return utils.sendAudioBuffer(this.response, data); - } -}; - -/** - * Get sound metadata by name. - * @param {String=} nameOfSound - * @returns {Array} - */ -MusicApp._getMetaDataByName = async function (nameOfSound = "") { - const metadata = soundLibrary.netsbloxSoundLibrary - .find((obj) => obj.soundName === nameOfSound); - return metadata; -}; - -module.exports = MusicApp; diff --git a/src/procedures/music-app/types.js b/src/procedures/music-app/types.js deleted file mode 100644 index 3577d8a1..00000000 --- a/src/procedures/music-app/types.js +++ /dev/null @@ -1,133 +0,0 @@ -const types = require("../../input-types"); - -const INSTRUMENTS = { - Piano: "PIANO", - Synth: "SYNTH", - Bass: "BASS", -}; - -const DRUMONESHOTTYPES = { - Kick: "KICK", - Snare: "SNARE", - Toms: "TOMS", - HiHat: "HI-HAT", - Clap: "CLAP", - Cymbal: "CYMBAL", - Percussion: "PERCUSSION", - FX: "FX", -}; -const DRUMPACKS = { - Peace: "PeaceTreaty", - UK: "UK", - US: "US", - DSS: "DSSxDP", -}; - -const NAMES = [ - "AcousticGuitar", - "BoardingArea", - "BrightDigitalChords", - "BrightSynthBrass", - "ChilledClav", - "ClassicElectricPiano", - "ClassicSuitcase", - "ClassicSuitcasePiano", - "CloudyPluckedSynth", - "DreamSinesPad", - "FadedKeys", - "FingerstyleBass", - "FutureFeelsBrass", - "GhostlyReversedOrgan", - "JazzOrgan", - "Liverpool", - "PulsatingWaves", - "Saxophone", - "SolidSoulElectricBass", - "SunriseChords", - "Steinway", - "ToyPiano", - "80sWaveBells", - "90sSolidSynthBass", -]; - -const KEYS = [ - "C", - "C#", - "D", - "Eb", - "E", - "F", - "F#", - "G", - "Ab", - "A", - "Bb", - "B", -]; - -const BPM = [ - "70BPM", - "80BPM", - "90BPM", - "100BPM", - "110BPM", -]; - -const CHORDS = [ - "1564", - "1251", - "3625", -]; - -function registerTypes() { - types.defineType({ - name: "Instruments", - description: "List of available Instruments", - baseType: "Enum", - baseParams: INSTRUMENTS, - }); - - types.defineType({ - name: "Keys", - description: "List of available keys", - baseType: "Enum", - baseParams: KEYS, - }); - - types.defineType({ - name: "BPM", - description: "List of available BPMs", - baseType: "Enum", - baseParams: BPM, - }); - - types.defineType({ - name: "Chords", - description: "List of available ChordProgressions", - baseType: "Enum", - baseParams: CHORDS, - }); - - types.defineType({ - name: "InstrumentNames", - description: "List of available Instruments", - baseType: "Enum", - baseParams: NAMES, - }); - - types.defineType({ - name: "DrumOneShotTypes", - description: "List of available One-Shot Types", - baseType: "Enum", - baseParams: DRUMONESHOTTYPES, - }); - - types.defineType({ - name: "DrumPackName", - description: "List of available Drum Packs", - baseType: "Enum", - baseParams: DRUMPACKS, - }); -} - -module.exports = { registerTypes }; From 90a8508d65b5ef7f7098bf3cfd0d151a5ef53331 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 6 Feb 2024 16:55:37 -0600 Subject: [PATCH 14/15] Implement whoami to get the authenticated user's name using the cookies --- src/cloud-client.js | 14 ++++++++++++-- src/error.js | 3 ++- src/procedures/alexa/routes.js | 4 ++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/cloud-client.js b/src/cloud-client.js index 2e6a3767..e9d3ee54 100644 --- a/src/cloud-client.js +++ b/src/cloud-client.js @@ -15,8 +15,18 @@ class NetsBloxCloud { this.secret = secret; } - async whoami(cookie) { - // TODO: look up the username using the cookie + async whoami(cookieJar) { + const cookieStr = Object.entries(cookieJar) + .map(([name, value]) => `${name}=${value}`) + .join("; "); + + const opts = { + headers: { + cookie: cookieStr, + }, + }; + console.log("whoami using cookies:", cookieStr); + return await this.get("/users/whoami", opts); } async getRoomState(projectId) { diff --git a/src/error.js b/src/error.js index 4967daa8..89ac1e7f 100644 --- a/src/error.js +++ b/src/error.js @@ -34,7 +34,8 @@ function handleUserErrors(fn) { if (err instanceof RequestError) { res.status(err.status).send(err.message); } else { - res.status(500).send("Internal Error Occurred. Try again later!"); + console.warn(err.stack); + res.status(500).send("Internal error occurred. Try again later!"); } } }; diff --git a/src/procedures/alexa/routes.js b/src/procedures/alexa/routes.js index 2c504ea1..4afad0db 100644 --- a/src/procedures/alexa/routes.js +++ b/src/procedures/alexa/routes.js @@ -30,7 +30,7 @@ router.get( parseCookies, setUsernameFromCookie, handleUserErrors((req, res) => { - const username = req.session.username; + const username = req.username; const isLoggedIn = !!username; if (!isLoggedIn) { @@ -60,7 +60,7 @@ router.put( parseCookies, setUsernameFromCookie, handleUserErrors(async (req, res) => { - const { username } = req.session; + const { username } = req; const isLoggedIn = !!username; if (!isLoggedIn) { From 433d7e0868a813d6d0eb32186f11a659a10af36a Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 6 Feb 2024 17:00:50 -0600 Subject: [PATCH 15/15] Restore music-app --- src/procedures/music-app/SoundLibrary | 1 + src/procedures/music-app/music-app.js | 143 ++++++++++++++++++++++++++ src/procedures/music-app/types.js | 133 ++++++++++++++++++++++++ 3 files changed, 277 insertions(+) create mode 160000 src/procedures/music-app/SoundLibrary create mode 100644 src/procedures/music-app/music-app.js create mode 100644 src/procedures/music-app/types.js diff --git a/src/procedures/music-app/SoundLibrary b/src/procedures/music-app/SoundLibrary new file mode 160000 index 00000000..437e8650 --- /dev/null +++ b/src/procedures/music-app/SoundLibrary @@ -0,0 +1 @@ +Subproject commit 437e86509b176faa23139893c2b441aa199dfeb3 diff --git a/src/procedures/music-app/music-app.js b/src/procedures/music-app/music-app.js new file mode 100644 index 00000000..7fa9e6ed --- /dev/null +++ b/src/procedures/music-app/music-app.js @@ -0,0 +1,143 @@ +/** + * This service allows users to play songs. + * @alpha + * @service + * @category Music + */ + +"use strict"; + +const fsp = require("fs/promises"); +const { registerTypes } = require("./types"); +const path = require("path"); +const utils = require("../utils/index"); +const MusicApp = {}; +const soundLibrary = require("./SoundLibrary/soundLibrary.json"); +const drumLibrary = require("./SoundLibrary/drumSoundLibrary.json"); +const masterSoundLibrary = [ + ...soundLibrary.netsbloxSoundLibrary, + ...drumLibrary.drumSoundLibrary, +]; + +registerTypes(); + +MusicApp._filetoBuffer = async function (audio_path) { + const data = await fsp.readFile(audio_path); + utils.sendAudioBuffer(this.response, data); +}; + +/** + * Get Sounds based on query. + * @param {String=} soundType + * @returns {Array} + */ +MusicApp._getNamesBySoundType = async function (soundType = "") { + var names = []; + + //Filter SoundCategories JSON by soundType + const queriedJSON = soundLibrary.netsbloxSoundLibrary.filter((obj) => + obj.SoundType === soundType.toUpperCase() + ); + + //Convert JSON to array of String names + for (let i = 0; i < queriedJSON.length; i++) { + names.push(queriedJSON[i].Name); + } + + return names; +}; + +/** + * Get sounds based on query. + * @param {DrumPackName=} packName + * @param {DrumOneShotTypes=} drumType + * @returns {String} + */ +MusicApp.getDrumOneShotNames = async function ( + packName = "", + drumType = "", +) { + var names = []; + let queriedJSON = ""; + + //Ensure at least one field is selected + if (packName !== "" || drumType !== "") { + queriedJSON = drumLibrary.drumSoundLibrary.filter(function (obj) { // Check if field value is empty before finding obj with value. + return (packName === "" || obj.packName === packName) && + (drumType === "" || obj.Instrument === drumType); + }); + } else { + throw Error("At least one field must be selected"); + } + + //Convert JSON to array of String names + for (let i = 0; i < queriedJSON.length; i++) { + names.push(queriedJSON[i].soundName); + } + return names; +}; + +/** + * Get sounds based on query. + * @param {Chords=} chords + * @param {Keys=} key + * @param {BPM=} bpm + * @param {InstrumentNames=} instrumentName + * @returns {Array} + */ +MusicApp.getSoundNames = async function ( + chords = "", + key = "", + bpm = "", + instrumentName = "", +) { + var names = []; + let queriedJSON = ""; + + //Ensure at least one field is selected + if (chords !== "" || key !== "" || bpm !== "" || instrumentName !== "") { + queriedJSON = soundLibrary.netsbloxSoundLibrary.filter(function (obj) { // Check if field value is empty before finding obj with value. + return (instrumentName === "" || obj.InstrumentName === instrumentName) && + (bpm === "" || obj.BPM === bpm) && + (key === "" || obj.Key === key) && + (chords === "" || obj.ChordProgression === chords); + }); + } else { + throw Error("At least one field must be selected"); + } + + //Convert JSON to array of String names + for (let i = 0; i < queriedJSON.length; i++) { + names.push(queriedJSON[i].soundName); + } + + return names; +}; + +/** + * Get sound by name. + * @param {String=} nameOfSound + */ +MusicApp.nameToSound = async function (nameOfSound = "") { + const metadata = masterSoundLibrary + .find((obj) => obj.soundName === nameOfSound); + + if (metadata) { + const audioPath = path.join(__dirname, metadata.Path); + const data = await fsp.readFile(audioPath); + return utils.sendAudioBuffer(this.response, data); + } +}; + +/** + * Get sound metadata by name. + * @param {String=} nameOfSound + * @returns {Array} + */ +MusicApp._getMetaDataByName = async function (nameOfSound = "") { + const metadata = soundLibrary.netsbloxSoundLibrary + .find((obj) => obj.soundName === nameOfSound); + return metadata; +}; + +module.exports = MusicApp; diff --git a/src/procedures/music-app/types.js b/src/procedures/music-app/types.js new file mode 100644 index 00000000..3577d8a1 --- /dev/null +++ b/src/procedures/music-app/types.js @@ -0,0 +1,133 @@ +const types = require("../../input-types"); + +const INSTRUMENTS = { + Piano: "PIANO", + Synth: "SYNTH", + Bass: "BASS", +}; + +const DRUMONESHOTTYPES = { + Kick: "KICK", + Snare: "SNARE", + Toms: "TOMS", + HiHat: "HI-HAT", + Clap: "CLAP", + Cymbal: "CYMBAL", + Percussion: "PERCUSSION", + FX: "FX", +}; +const DRUMPACKS = { + Peace: "PeaceTreaty", + UK: "UK", + US: "US", + DSS: "DSSxDP", +}; + +const NAMES = [ + "AcousticGuitar", + "BoardingArea", + "BrightDigitalChords", + "BrightSynthBrass", + "ChilledClav", + "ClassicElectricPiano", + "ClassicSuitcase", + "ClassicSuitcasePiano", + "CloudyPluckedSynth", + "DreamSinesPad", + "FadedKeys", + "FingerstyleBass", + "FutureFeelsBrass", + "GhostlyReversedOrgan", + "JazzOrgan", + "Liverpool", + "PulsatingWaves", + "Saxophone", + "SolidSoulElectricBass", + "SunriseChords", + "Steinway", + "ToyPiano", + "80sWaveBells", + "90sSolidSynthBass", +]; + +const KEYS = [ + "C", + "C#", + "D", + "Eb", + "E", + "F", + "F#", + "G", + "Ab", + "A", + "Bb", + "B", +]; + +const BPM = [ + "70BPM", + "80BPM", + "90BPM", + "100BPM", + "110BPM", +]; + +const CHORDS = [ + "1564", + "1251", + "3625", +]; + +function registerTypes() { + types.defineType({ + name: "Instruments", + description: "List of available Instruments", + baseType: "Enum", + baseParams: INSTRUMENTS, + }); + + types.defineType({ + name: "Keys", + description: "List of available keys", + baseType: "Enum", + baseParams: KEYS, + }); + + types.defineType({ + name: "BPM", + description: "List of available BPMs", + baseType: "Enum", + baseParams: BPM, + }); + + types.defineType({ + name: "Chords", + description: "List of available ChordProgressions", + baseType: "Enum", + baseParams: CHORDS, + }); + + types.defineType({ + name: "InstrumentNames", + description: "List of available Instruments", + baseType: "Enum", + baseParams: NAMES, + }); + + types.defineType({ + name: "DrumOneShotTypes", + description: "List of available One-Shot Types", + baseType: "Enum", + baseParams: DRUMONESHOTTYPES, + }); + + types.defineType({ + name: "DrumPackName", + description: "List of available Drum Packs", + baseType: "Enum", + baseParams: DRUMPACKS, + }); +} + +module.exports = { registerTypes };