diff --git a/docs/adv/url.rst b/docs/adv/url.rst index 3a453bb8..7dacfc2d 100644 --- a/docs/adv/url.rst +++ b/docs/adv/url.rst @@ -61,3 +61,5 @@ The following settings are generally available, regardless of the action: - **noExitWarning**: Don't confirm that the user wants to leave the page on tab close. - **lang**: Set the language immediately. For example, "lang=hu" will set NetsBlox to Hungarian on start. - **setVariable**: Set a variable to the given value on start. The value is expected to be a URL-encoded pair so setting "hello" to "world" would be "hello%3Dworld". +- **cloud**: Override the default cloud URL. This can be used to point the public deployment to your own local cloud (e.g, `cloud=http://localhost:7777`). +- **extensions**: A list of extensions (by URL) to load on start. diff --git a/docs/conf.py b/docs/conf.py index 70aba35f..2380db79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = 'NetsBlox' -copyright = '2021, Vanderbilt University' +copyright = '2024, Vanderbilt University' author = 'NetsBlox Team' # -- General configuration --------------------------------------------------- diff --git a/package-lock.json b/package-lock.json index 0d38f69c..8677aceb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,11 +22,13 @@ "cookie-parser": "^1.4.4", "doctrine": "^2.0.0", "express": "^4.18.1", + "fast-csv": "^4.3.6", "fs-extra": "^11.1.0", "geolib": "^2.0.18", "gnuplot": "^0.3.1", "jimp": "^0.16.1", "json-query": "^2.2.2", + "md5": "^2.3.0", "moment": "^2.29.4", "mongodb": "^3.7.3", "mongoose": "^5.10.15", @@ -69,6 +71,43 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==" }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + }, "node_modules/@jimp/bmp": { "version": "0.16.1", "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.16.1.tgz", @@ -1653,6 +1692,14 @@ "node": ">=4" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "engines": { + "node": "*" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -2059,6 +2106,14 @@ "node": ">=0.8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "engines": { + "node": "*" + } + }, "node_modules/crypto-browserify": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", @@ -2712,6 +2767,18 @@ "node": "> 0.1.90" } }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4241,6 +4308,41 @@ "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==" }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==" + }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==" + }, "node_modules/lodash.memoize": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz", @@ -4398,6 +4500,16 @@ "semver": "bin/semver.js" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", diff --git a/package.json b/package.json index 1f735b03..b3bb60c5 100644 --- a/package.json +++ b/package.json @@ -23,11 +23,13 @@ "cookie-parser": "^1.4.4", "doctrine": "^2.0.0", "express": "^4.18.1", + "fast-csv": "^4.3.6", "fs-extra": "^11.1.0", "geolib": "^2.0.18", "gnuplot": "^0.3.1", "jimp": "^0.16.1", "json-query": "^2.2.2", + "md5": "^2.3.0", "moment": "^2.29.4", "mongodb": "^3.7.3", "mongoose": "^5.10.15", diff --git a/src/cloud-client.js b/src/cloud-client.js index 63bd6144..e9d3ee54 100644 --- a/src/cloud-client.js +++ b/src/cloud-client.js @@ -15,8 +15,18 @@ class NetsBloxCloud { this.secret = secret; } - async whoami(cookie) { - // TODO: look up the username using the cookie + async whoami(cookieJar) { + const cookieStr = Object.entries(cookieJar) + .map(([name, value]) => `${name}=${value}`) + .join("; "); + + const opts = { + headers: { + cookie: cookieStr, + }, + }; + console.log("whoami using cookies:", cookieStr); + return await this.get("/users/whoami", opts); } async getRoomState(projectId) { @@ -118,6 +128,10 @@ class NetsBloxCloud { return clients; } + async getOAuthToken(tokenId) { + return await this.get(`/oauth/token/${tokenId}`); + } + isConfigured() { return this.cloudUrl && this.id && this.secret; } diff --git a/src/error.js b/src/error.js index 13214e8e..89ac1e7f 100644 --- a/src/error.js +++ b/src/error.js @@ -27,24 +27,32 @@ class MissingFieldError extends RequestError { } function handleUserErrors(fn) { - return async function (req, res) { + return async function (_req, res) { try { await fn.call(this, ...arguments); } catch (err) { if (err instanceof RequestError) { res.status(err.status).send(err.message); } else { - res.status(500).send("Internal Error Occurred. Try again later!"); + console.warn(err.stack); + res.status(500).send("Internal error occurred. Try again later!"); } } }; } +class LoginRequired extends RequestError { + constructor() { + super(401, "Login Required."); + } +} + module.exports = { UserError, NotAllowedError, InvalidKeyProviderError, MissingFieldError, + LoginRequired, handleUserErrors, }; diff --git a/src/procedures/alexa/routes.js b/src/procedures/alexa/routes.js index d405554e..4afad0db 100644 --- a/src/procedures/alexa/routes.js +++ b/src/procedures/alexa/routes.js @@ -1,14 +1,10 @@ const AlexaSkill = require("./skill"); const express = require("express"); -module.exports = express(); // FIXME: add support for OAuth on NetsBlox cloud -return; -// TODO: refactor the next imports -const OAuth = require("../../../api/core/oauth"); -// FIXME: TODO: Update this -const { handleErrors } = require("../../../api/rest/utils"); +const NetsBloxCloud = require("../../cloud-client"); +const { handleUserErrors } = require("../../error"); const { setUsernameFromCookie } = require("../utils/router-utils"); -const { LoginRequired, RequestError } = require("../../../api/core/errors"); +const { LoginRequired, RequestError } = require("../../error"); const bodyParser = require("body-parser"); const axios = require("axios"); @@ -27,14 +23,14 @@ const logger = require("../utils/logger")("alexa:routes"); const router = express(); const parseCookies = cookieParser(); -router.get("/ping", (req, res) => res.send("pong")); +router.get("/ping", (_req, res) => res.send("pong")); router.get( "/login.html", bodyParser.json(), parseCookies, setUsernameFromCookie, - handleErrors((req, res) => { - const username = req.session.username; + handleUserErrors((req, res) => { + const username = req.username; const isLoggedIn = !!username; if (!isLoggedIn) { @@ -63,8 +59,8 @@ router.put( bodyParser.json(), parseCookies, setUsernameFromCookie, - handleErrors(async (req, res) => { - const { username } = req.session; + handleUserErrors(async (req, res) => { + const { username } = req; const isLoggedIn = !!username; if (!isLoggedIn) { @@ -126,7 +122,7 @@ router.post( handleErrorsInAlexa(async (req, res) => { const reqData = req.body; const { accessToken } = reqData.session.user; - const token = await OAuth.getToken(accessToken); + const token = await NetsBloxCloud.getToken(accessToken); const { username } = token; const skillId = reqData.session.application.applicationId; @@ -194,7 +190,7 @@ function speak(text) { } function handleErrorsInAlexa(fn) { - return async function (req, res) { + return async function (_req, res) { try { await fn(...arguments); } catch (err) { diff --git a/src/procedures/movebank/movebank.js b/src/procedures/movebank/movebank.js new file mode 100644 index 00000000..0689c97b --- /dev/null +++ b/src/procedures/movebank/movebank.js @@ -0,0 +1,268 @@ +/** + * Access to Movebank, a free, online database of animal tracking data hosted by the Max Planck Institute of Animal Behavior. + * + * @alpha + * @service + * @category GeoData + */ + +const logger = require("../utils/logger")("movebank"); +const ApiConsumer = require("../utils/api-consumer"); +const { MovebankKey } = require("../utils/api-key"); +const types = require("../../input-types"); +const geolib = require("geolib"); +const csv = require("fast-csv"); +const axios = require("axios"); +const md5 = require("md5"); + +const Movebank = new ApiConsumer( + "Movebank", + "https://www.movebank.org/movebank/service/direct-read/", + { cache: { ttl: 1 * 60 * 60 } }, +); +ApiConsumer.setRequiredApiKey(Movebank, MovebankKey); + +// some studies require us to accept a license before we can view the data, +// and there doesn't seem to be a way to filter these out before attempting to download the data. +// so we'll just auto-accept any licenses that are (specifically) of the following open license types. +const ALLOWED_LICENSE_TYPES = { + "CC_0": "CC0", + "CC_BY": "CC BY", + "CC_BY_NC": "CC BY-NC", + "CC_BY_SA": "CC BY-SA", + "CC_BY_NC_SA": "CC BY-NC-SA", +}; + +async function parseCSV(content) { + return new Promise((resolve, reject) => { + const res = []; + const stream = csv.parse({ headers: true, objectMode: true }) + .on("data", (x) => res.push(x)) + .on("error", (e) => reject(e)) + .on("end", () => resolve(res)); + + stream.write(content); + stream.end(); + }); +} + +async function fetchLicensed(settings, licenseHash = null) { + const licenseSuffix = licenseHash ? `&license-md5=${licenseHash}` : ""; + const url = `${Movebank._baseUrl}?${settings.queryString}${licenseSuffix}`; + logger.info(`fetching possibly licensed content: ${url}`); + + return await Movebank._cache.wrap( + `::<${licenseHash}>::<${url}>`, + async () => { + logger.info("> request is not cached - calling external endpoint"); + const res = await axios({ url, method: "GET" }); + if (!licenseHash && res.headers["accept-license"] === "true") { + logger.info("> failed with license request"); + for (const ty in ALLOWED_LICENSE_TYPES) { + if ( + res.data.includes( + `License Type: ${ + ALLOWED_LICENSE_TYPES[ty] + }`, + ) + ) { + logger.info(`> accepting license of type ${ty} and retrying...`); + return await fetchLicensed(settings, md5(res.data)); + } + } + logger.info(`> unknown license type`, res.data); + throw Error("failed to download licensed material"); + } + return res.data; + }, + ); +} + +async function tryOrElse(ok, err) { + try { + return await ok(); + } catch (e) { + return await err(); + } +} + +let SENSOR_TYPES_META = []; +types.defineType({ + name: "MovebankSensor", + description: "A sensor type used by :doc:`/services/Movebank/index`.", + baseType: "Enum", + baseParams: (async () => { + try { + SENSOR_TYPES_META = await parseCSV( + await Movebank._requestData({ + queryString: + `entity_type=tag_type&api-token=${Movebank.apiKey.value}`, + }), + ); + } catch (e) { + console.error("failed to load MoveBank sensor types", e); + } + + const res = {}; + for (const ty of SENSOR_TYPES_META) { + res[ty.name] = parseInt(ty.id); + } + return res; + })(), +}); + +/** + * Get a list of all the sensor types supported by Movebank. + * + * @returns {Array} A list of supported sensor types + */ +Movebank.getSensorTypes = function () { + return SENSOR_TYPES_META.map((x) => x.name); +}; + +/** + * Get a list of all the studies available for (public) viewing. + * + * @returns {Array} A list of available studies + */ +Movebank.getStudies = async function () { + const data = await parseCSV( + await Movebank._requestData({ + queryString: + `entity_type=study&i_have_download_access=true&api-token=${Movebank.apiKey.value}`, + }), + ); + + const res = []; + for (const raw of data) { + if ( + raw.id && raw.main_location_lat && raw.main_location_long && + raw.citation && ALLOWED_LICENSE_TYPES[raw.license_type] + ) { + res.push({ + id: parseInt(raw.id), + latitude: parseFloat(raw.main_location_lat), + longitude: parseFloat(raw.main_location_long), + species: raw.taxon_ids.split(",").map((x) => x.trim()), + sensors: raw.sensor_type_ids.split(",").map((x) => x.trim()), + citation: raw.citation, + }); + } + } + return res; +}; + +/** + * Get a list of all the studies available for (public) viewing within a certain max distance of a point of interest. + * Note that some of the animals involved in these studies may travel outside of this distance. + * + * @param {Latitude} latitude Latitude of the point of interest + * @param {Longitude} longitude Longitude of the point of interest + * @param {BoundedNumber<0>} distance Max distance from the point of interest (in meters) + * @returns {Array} A list of available studies near the point of interest + */ +Movebank.getStudiesNear = async function (latitude, longitude, distance) { + const p = { latitude, longitude }; + return (await Movebank.getStudies()).filter((x) => + geolib.getDistance(p, { latitude: x.latitude, longitude: x.longitude }) <= + distance + ); +}; + +/** + * Get a list of all the animals that participated in a specific study. + * + * @param {Object} study A study object returned by :func:`Movebank.getStudies` + * @returns {Array} A list of animals + */ +Movebank.getAnimals = async function (study) { + study = parseInt(study.id); + if (isNaN(study)) throw Error("unknown study"); + + const data = await parseCSV( + await tryOrElse(async () => { + return await fetchLicensed({ + queryString: + `entity_type=individual&study_id=${study}&api-token=${Movebank.apiKey.value}`, + }); + }, () => { + return ""; + }), + ); + + const res = []; + for (const raw of data) { + if (raw.local_identifier && raw.taxon_canonical_name) { + res.push({ + id: raw.local_identifier, + sex: raw.sex === "m" ? "male" : raw.sex === "f" ? "female" : "unknown", + species: raw.taxon_canonical_name, + sensors: raw.sensor_type_ids.split(",").map((x) => { + x = x.trim(); + for (const meta of SENSOR_TYPES_META) { + if (meta.external_id === x) { + return meta.name; + } + } + return x; + }), + }); + } + } + return res; +}; + +/** + * Get a chronological list of all the migration events for an animal in a specific study. + * + * @param {Object} study A study object returned by :func:`Movebank.getStudies` + * @param {Object} animal An animal object returned by :func:`Movebank.getAnimals`. The animal should be part of the same study. + * @param {BoundedNumber<0>=} minDistance The minimum distance (in meters) between consecutive returned events (default 0, which gives all available data). + * @returns {Array} A list of chronological migration events for the animal + */ +Movebank.getEvents = async function (study, animal, minDistance = 0) { + study = parseInt(study.id); + if (isNaN(study)) throw Error("unknown study"); + + animal = animal.id.toString(); + if (!animal) throw Error("unknown animal"); + + const data = await parseCSV( + await tryOrElse(async () => { + return await fetchLicensed({ + queryString: + `entity_type=event&study_id=${study}&individual_local_identifier=${animal}&attributes=visible,timestamp,location_lat,location_long&api-token=${Movebank.apiKey.value}`, + }); + }, () => { + return ""; + }), + ); + + const res = []; + let prevPos = null; + for (const raw of data) { + if ( + raw.visible === "true" && raw.timestamp && raw.location_lat && + raw.location_long + ) { + const entry = { + timestamp: new Date(raw.timestamp), + latitude: parseFloat(raw.location_lat), + longitude: parseFloat(raw.location_long), + }; + + if (minDistance > 0) { + const pos = { latitude: entry.latitude, longitude: entry.longitude }; + if (!prevPos || geolib.getDistance(prevPos, pos) >= minDistance) { + prevPos = pos; + res.push(entry); + } + } else { + res.push(entry); + } + } + } + return res; +}; + +module.exports = Movebank; diff --git a/src/procedures/utils/api-key.js b/src/procedures/utils/api-key.js index 043b99a0..10049caa 100644 --- a/src/procedures/utils/api-key.js +++ b/src/procedures/utils/api-key.js @@ -19,6 +19,10 @@ class ApiKey { } } +module.exports.MovebankKey = new ApiKey( + "Movebank", + "https://github.com/movebank/movebank-api-doc/blob/master/movebank-api.md#authenticate-by-token", +); module.exports.TimezoneDBKey = new ApiKey( "TimezoneDB", "https://timezonedb.com/register", diff --git a/test/procedures/movebank.spec.js b/test/procedures/movebank.spec.js new file mode 100644 index 00000000..a18aa47e --- /dev/null +++ b/test/procedures/movebank.spec.js @@ -0,0 +1,11 @@ +const utils = require("../assets/utils"); + +describe(utils.suiteName(__filename), function () { + utils.verifyRPCInterfaces("Movebank", [ + ["getSensorTypes", []], + ["getStudies", []], + ["getStudiesNear", ["latitude", "longitude", "distance"]], + ["getAnimals", ["study"]], + ["getEvents", ["study", "animal", "minDistance"]], + ]); +});