From 05df4bb04e1a54d6a7dbe38ddc201b8db5717f41 Mon Sep 17 00:00:00 2001 From: alinarublea Date: Tue, 10 Oct 2023 15:25:16 +0200 Subject: [PATCH] initial-setup of aws --- package.json | 8 +- src/db.js | 120 +++++++++++++++ src/edge-delivery-service-admin-client.js | 37 +++++ src/github-client.js | 103 +++++++++++++ src/index.js | 22 ++- src/psi-client.js | 175 ++++++++++++++++++++++ src/s3.js | 37 +++++ src/util.js | 29 ++++ 8 files changed, 525 insertions(+), 6 deletions(-) create mode 100644 src/db.js create mode 100644 src/edge-delivery-service-admin-client.js create mode 100644 src/github-client.js create mode 100644 src/psi-client.js create mode 100644 src/s3.js create mode 100644 src/util.js diff --git a/package.json b/package.json index 0268e829..cb49babd 100644 --- a/package.json +++ b/package.json @@ -45,5 +45,11 @@ "lint-staged": { "*.js": "eslint", "*.cjs": "eslint" + }, + "dependencies": { + "aws-sdk": "2.1472.0", + "@adobe/helix-shared-wrap": "2.0.0", + "@adobe/helix-status": "10.0.10", + "@adobe/helix-universal-logger": "3.0.11" } -} \ No newline at end of file +} diff --git a/src/db.js b/src/db.js new file mode 100644 index 00000000..a14eb5f0 --- /dev/null +++ b/src/db.js @@ -0,0 +1,120 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +const AWS = require('aws-sdk'); +const { log } = require('./util.js'); + +const TABLE_SITES = 'sites'; +const TABLE_AUDITS = 'audits'; + +function DB(config) { + const { region, accessKeyId, secretAccessKey } = config; + + AWS.config.update({ region, accessKeyId, secretAccessKey }); + const dynamoDb = new AWS.DynamoDB.DocumentClient(); + + /** + * Save an audit record to the DynamoDB. + * @param {object} newAudit - The new audit record to save. + */ + async function saveAuditIndexRecord(newAudit) { + try { + const params = { + TableName: TABLE_AUDITS, + Item: newAudit, + }; + await dynamoDb.put(params).promise(); + } catch (error) { + log('error', 'Error saving audit: ', error); + } + } + /** + * Saves an audit to the DynamoDB. + * @param {object} site - Site object containing details of the audited site. + * @param {object} audit - Audit object containing the type and result of the audit. + * @returns {Promise} Resolves once audit is saved. + */ + async function saveAuditIndex(site, audit) { + const now = new Date().toISOString(); + const newAudit = { + siteId: site.id, + auditDate: now, + error: '', + isLive: site.isLive, + scores: { + mobile: { + performance: audit.lighthouseResults.categories.performance.score, + seo: audit.lighthouseResults.categories.seo.score, + 'best-practices': audit.lighthouseResults.categories['best-practices'].score, + accessibility: audit.lighthouseResults.categories.accesibility.score, + }, + }, + }; + await saveAuditIndexRecord(newAudit); + log('info', `Audit for domain ${site.domain} saved successfully at ${now}`); + } + /** + * Save an error that occurred during a Lighthouse audit to the DynamoDB. + * @param {object} site - site audited. + * @param {Error} error - The error that occurred during the audit. + */ + async function saveAuditError(site, error) { + const now = new Date().toISOString(); + const newAudit = { + siteId: site.id, + auditDate: now, + isLive: site.isLive, + error: error.message, + scores: {}, + }; + await saveAuditIndexRecord(newAudit); + } + /** + * Fetches a site by its ID and gets its latest audit. + * @param {string} siteId - The ID of the site to fetch. + * @returns {Promise} Site document with its latest audit. + */ + async function findSiteById(siteId) { + try { + const siteParams = { + TableName: TABLE_SITES, + Key: { + id: siteId, + }, + }; + const siteResult = await dynamoDb.get(siteParams).promise(); + if (!siteResult.Item) return null; + const auditParams = { + TableName: TABLE_AUDITS, + KeyConditionExpression: 'siteId = :siteId', + ExpressionAttributeValues: { + ':siteId': siteId, + }, + Limit: 1, + ScanIndexForward: false, // get the latest audit + }; + const auditResult = await dynamoDb.query(auditParams).promise(); + // eslint-disable-next-line prefer-destructuring + siteResult.Item.latestAudit = auditResult.Items[0]; + + return siteResult.Item; + } catch (error) { + console.error('Error getting site by site id:', error.message); + return Error(error.message); + } + } + return { + findSiteById, + saveAuditIndex, + saveAuditError, + }; +} +module.exports = DB; diff --git a/src/edge-delivery-service-admin-client.js b/src/edge-delivery-service-admin-client.js new file mode 100644 index 00000000..19c93fa1 --- /dev/null +++ b/src/edge-delivery-service-admin-client.js @@ -0,0 +1,37 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +const BASE_URL = 'https://www.hlx.live/'; +/** + * Retrieve the status of the admin endpoint. + * @returns {Promise} - The lastModification property. + * @throws {Error} - Throws an error if there's a network issue or some other error while fetching data. + */ +function EdgeDeliveryServiceAdminClient() { + const getLastModification = async () => { + try { + const response = await fetch(`${BASE_URL}docs/status.json`); + if (!response.ok) { + throw new Error(`Failed to fetch status: ${response.statusText}`); + } + const data = await response.json(); + if (!data || !data.lastModification) { + throw new Error('lastModification property not found in response data'); + } + return data.lastModification; + } catch (error) { + console.error('Error fetching lastModification:', error); + throw error; + } + }; +} + +module.exports = EdgeDeliveryServiceAdminClient; diff --git a/src/github-client.js b/src/github-client.js new file mode 100644 index 00000000..0d7c7e3a --- /dev/null +++ b/src/github-client.js @@ -0,0 +1,103 @@ +const axios = require('axios'); +const { log } = require('./util.js'); + +const SECONDS_IN_A_DAY = 86400; // Number of seconds in a day + +function GithubClient(config) { + const {baseUrl, githubId, githubSecret} = config; + + /** + * Creates a URL for the GitHub API. + * + * @param {string} githubOrg - The name of the GitHub organization. + * @param {string} repoName - The name of the repository (optional). + * @param {string} path - Additional path (optional). + * @param {number} page - The page number for pagination (optional). + * @returns {string} The created GitHub API URL. + */ + function createGithubApiUrl(githubOrg, repoName = '', path = '', page = 1) { + const repoPart = repoName ? `/${repoName}` : ''; + const pathPart = path ? `/${path}` : ''; + + return `${baseUrl}/repos/${githubOrg}${repoPart}${pathPart}?page=${page}&per_page=100`; + } + + /** + * Creates a Basic Authentication header value from a given GitHub ID and secret. + * + * @returns {string} - The Basic Authentication header value. + * @throws {Error} - Throws an error if GitHub credentials are not provided. + */ + function createGithubAuthHeaderValue() { + if (!githubId || !githubSecret) { + throw new Error('GitHub credentials not provided'); + } + return `Basic ${Buffer.from(`${githubId}:${githubSecret}`).toString('base64')}`; + } + + /** + * Fetches the SHAs of all commits made in a GitHub repository between two date-times using the GitHub API. + * + * @async + * @function + * @param {object} domain - The domain of the audited site. + * @param {string} latestAuditTime - The end date-time in ISO format (e.g. 'YYYY-MM-DDTHH:mm:ss.sssZ'). + * @param {string} lastAuditedAt - The start date-time in ISO format (e.g. 'YYYY-MM-DDTHH:mm:ss.sssZ'). If not provided, it defaults to 24 hours before the end date-time. + * @param {string} gitHubURL - The URL of the GitHub repository from which the SHAs will be fetched (e.g. 'https://github.com/user/repo'). + * @returns {Promise} A promise that resolves to an array of SHAs of commits between the given date-times. If there's an error fetching the data, the promise resolves to an empty array. + * @throws {Error} Will throw an error if there's a network issue or some other error while fetching data from the GitHub API. + * @example + * fetchGithubCommitsSHA( + * { gitHubURL: 'https://github.com/myOrg/myRepo', lastAudited: '2023-06-15T00:00:00.000Z' }, + * { result: { fetchTime: '2023-06-16T00:00:00.000Z' } }, + * 'yourGithubId', + * 'yourGithubSecret' + * ).then(SHAs => console.log(SHAs)); + */ + async function fetchGithubCommitsSHA(domain, latestAuditTime, lastAuditedAt, gitHubURL) { + if (!gitHubURL) { + log('info', `No github repo defined for site ${domain}. Skipping github SHA retrieval`); + return []; + } + + try { + const until = new Date(latestAuditTime); + const since = lastAuditedAt ? new Date(lastAuditedAt) : new Date(until - SECONDS_IN_A_DAY * 1000); // 24 hours before until + const repoPath = new URL(gitHubURL).pathname.slice(1); // Removes leading '/' + + log('info', `Fetching SHAs for domain ${domain} with repo ${repoPath} between ${since.toISOString()} and ${until.toISOString()}`); + + const [githubOrg, repoName] = repoPath.split('/'); + + const authHeader = createGithubAuthHeaderValue(); + const commitsUrl = createGithubApiUrl(githubOrg, repoName, 'commits'); + + const response = await axios.get(commitsUrl, { + params: { + since: since.toISOString(), + until: until.toISOString() + }, + headers: { + Authorization: authHeader + } + }); + + const commitSHAs = response.data.map(commit => commit.sha); + + log('info', `Found ${commitSHAs.length} commits for site ${domain}.`); + + return commitSHAs; + } catch (error) { + log('error', `Error fetching GitHub SHAs for site ${domain}:`, error.response ? error.response.data : error); + return []; + } + } + + return { + createGithubApiUrl, + createGithubAuthHeaderValue, + fetchGithubCommitsSHA + } +} + +module.exports = GithubClient; diff --git a/src/index.js b/src/index.js index cc2ecf3a..21c10f55 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,5 @@ /* - * Copyright 2021 Adobe. All rights reserved. + * Copyright 2019 Adobe. All rights reserved. * This file is licensed to you under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. You may obtain a copy * of the License at http://www.apache.org/licenses/LICENSE-2.0 @@ -9,12 +9,24 @@ * OF ANY KIND, either express or implied. See the License for the specific language * governing permissions and limitations under the License. */ +import wrap from '@adobe/helix-shared-wrap'; +import { logger } from '@adobe/helix-universal-logger'; +import { helixStatus } from '@adobe/helix-status'; +import { Response } from '@adobe/fetch'; /** * This is the main function - * @param {string} name name of the person to greet - * @returns {string} a greeting + * @param {Request} request the request object (see fetch api) + * @param {UniversalContext} context the context of the universal serverless function + * @returns {Response} a response */ -export function main(name = 'world') { - return `Hello, ${name}.`; +function run(request, context) { + const name = new URL(request.url).searchParams.get('name') || 'world'; + context.log.info(`Saying hello to: ${name}.`); + return new Response(`Hello, ${name}.`); } + +export const main = wrap(run) + .with(helixStatus) + .with(logger.trace) + .with(logger); diff --git a/src/psi-client.js b/src/psi-client.js new file mode 100644 index 00000000..220a7608 --- /dev/null +++ b/src/psi-client.js @@ -0,0 +1,175 @@ +const axios = require('axios'); +const { log } = require('./util.js'); + +function PSIClient(config) { + const AUDIT_TYPE = 'PSI'; + const FORM_FACTOR_MOBILE = 'mobile'; + const FORM_FACTOR_DESKTOP = 'desktop'; + const PSI_STRATEGIES = [FORM_FACTOR_MOBILE, FORM_FACTOR_DESKTOP]; + + const { apiKey, baseUrl } = config; + + /** + * Formats an input URL to be HTTPS. + * + * @param {string} input - The input URL. + * @returns {string} The formatted URL with HTTPS. + */ + const formatURL = (input) => { + const urlPattern = /^https?:\/\//i; + + if (urlPattern.test(input)) { + return input.replace(/^http:/i, 'https:'); + } else { + return `https://${input}`; + } + } + + /** + * Builds a PageSpeed Insights API URL with the necessary parameters. + * + * @param {string} siteUrl - The URL of the site to analyze. + * @param {string} strategy - The strategy to use for the PSI check. + * @returns {string} The full API URL with parameters. + */ + const getPSIApiUrl = (siteUrl, strategy) => { + const validStrategies = [FORM_FACTOR_MOBILE, FORM_FACTOR_DESKTOP]; + if (!validStrategies.includes(strategy)) { + strategy = FORM_FACTOR_MOBILE; + } + + const params = new URLSearchParams({ + url: formatURL(siteUrl), + key: apiKey, + strategy, + }); + + ['performance', 'accessibility', 'best-practices', 'seo'].forEach(category => { + params.append('category', category); + }); + + return `${[baseUrl]}?${params.toString()}`; + }; + + /** + * Processes audit data by replacing keys with dots with underscore. + * + * @param {object} data - The audit data object. + * @returns {object} The processed audit data. + */ + const processAuditData = (data) => { + if (!data) { + return null; + } + + const newData = { ...data }; + + for (let key in newData) { + if (typeof newData[key] === 'object' && newData[key] !== null) { + newData[key] = processAuditData(newData[key]); + } + + if (key.includes('.')) { + const newKey = key.replace(/\./g, '_'); + newData[newKey] = newData[key]; + delete newData[key]; + } + } + + return newData; + }; + + /** + * Processes the Lighthouse audit result. Only certain properties are saved. + * @param {object} result - The Lighthouse audit result. + * @returns {object} The processed Lighthouse audit result. + */ + function processLighthouseResult({ + categories, + requestedUrl, + fetchTime, + finalUrl, + mainDocumentUrl, + finalDisplayedUrl, + lighthouseVersion, + userAgent, + environment, + runWarnings, + configSettings, + timing, + audits = {}, + } = {}) { + return { + categories, + requestedUrl, + fetchTime, + finalUrl, + mainDocumentUrl, + finalDisplayedUrl, + lighthouseVersion, + userAgent, + environment, + runWarnings, + configSettings, + timing, + audits: { + 'third-party-summary': audits['third-party-summary'], + 'total-blocking-time': audits['total-blocking-time'], + } + }; + } + + /** + * Performs a PageSpeed Insights check on the specified domain. + * + * @param {string} domain - The domain to perform the PSI check on. + * @param {string} strategy - The strategy to use for the PSI check. + * @returns {Promise} The processed PageSpeed Insights audit data. + */ + const performPSICheck = async (domain, strategy) => { + try { + const apiURL = getPSIApiUrl(domain, strategy); + const { data: lhs } = await axios.get(apiURL); + + const { lighthouseResult } = processAuditData(lhs); + + return processLighthouseResult(lighthouseResult); + } catch (e) { + log('error', `Error happened during PSI check: ${e}`); + throw e; + } + }; + + const runAudit = async (domain) => { + const auditResults = {}; + + for (const strategy of PSI_STRATEGIES) { + const strategyStartTime = process.hrtime(); + const psiResult = await performPSICheck(domain, strategy); + const strategyEndTime = process.hrtime(strategyStartTime); + const strategyElapsedTime = (strategyEndTime[0] + strategyEndTime[1] / 1e9).toFixed(2); + log('info', `Audited ${domain} for ${strategy} strategy in ${strategyElapsedTime} seconds`); + + auditResults[strategy] = psiResult; + } + + return { + type: AUDIT_TYPE, + finalUrl: auditResults[FORM_FACTOR_MOBILE]?.finalUrl, + time: auditResults[FORM_FACTOR_MOBILE]?.fetchTime, + result: auditResults, + } + }; + + return { + FORM_FACTOR_MOBILE, + FORM_FACTOR_DESKTOP, + formatURL, + getPSIApiUrl, + performPSICheck, + processAuditData, + runAudit, + } +} + +module.exports = PSIClient; diff --git a/src/s3.js b/src/s3.js new file mode 100644 index 00000000..4f91edec --- /dev/null +++ b/src/s3.js @@ -0,0 +1,37 @@ +/* + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +const AWS = require('aws-sdk'); +const { log } = require('util'); + +const s3 = new AWS.S3(); + +function S3(config) { + const { region, accessKeyId, secretAccessKey } = config; + AWS.config.update({ region, accessKeyId, secretAccessKey }); + async function putObjectToS3(key, data) { + const params = { + Bucket: config.BUCKET_NAME, + Key: key, + Body: JSON.stringify(data), + ContentType: 'application/json', + }; + + try { + await s3.putObject(params).promise(); + log('info', `Data saved to S3 with key: ${key}`); + } catch (error) { + log('error', 'Error saving data to S3: ', error); + } + } +} + +module.exports = { S3 }; diff --git a/src/util.js b/src/util.js new file mode 100644 index 00000000..6d2c7ad9 --- /dev/null +++ b/src/util.js @@ -0,0 +1,29 @@ +/** + * A utility log method to streamline logging. + * + * @param {string} level - The log level ('info', 'error', 'warn'). + * @param {string} message - The message to log. + * @param {...any} args - Additional arguments to log. + */ +const log = (level, message, ...args) => { + const timestamp = new Date().toISOString(); + + switch (level) { + case 'info': + console.info(`[${timestamp}] INFO: ${message}`, ...args); + break; + case 'error': + console.error(`[${timestamp}] ERROR: ${message}`, ...args); + break; + case 'warn': + console.warn(`[${timestamp}] WARN: ${message}`, ...args); + break; + default: + console.log(`[${timestamp}] ${message}`, ...args); + break; + } +}; + +module.exports = { + log, +};