From 8d39d76919b35e36d0f9450edb8cbcaa98b421c2 Mon Sep 17 00:00:00 2001 From: Gaurav Munjal Date: Fri, 8 Nov 2019 13:45:14 -0500 Subject: [PATCH 1/3] Fix ability to generate local docs by disabling s3 syncing --- main.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/main.js b/main.js index 37037da..ae74ddf 100644 --- a/main.js +++ b/main.js @@ -14,7 +14,7 @@ import revProjVersionFiles from './lib/rev-docs' import { downloadExistingDocsToLocal, uploadDocsToS3 } from './lib/s3-sync' import fixBorkedYuidocFiles from './lib/fix-borked-yuidoc-files' -export function apiDocsProcessor( +export async function apiDocsProcessor( projects, specificDocsVersion, ignorePreviouslyIndexedDoc, @@ -28,13 +28,14 @@ export function apiDocsProcessor( let docsVersionMsg = specificDocsVersion !== '' ? `. For version ${specificDocsVersion}` : '' console.log(`Downloading docs for ${projects.join(' & ')}${docsVersionMsg}`) - downloadExistingDocsToLocal() - .then(() => fetchYuiDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc || runClean)) - .then(async filesToProcess => { - await fs.mkdirp('tmp/s3-original-docs') - return await RSVP.Promise.all(filesToProcess.map(fixBorkedYuidocFiles)) - }) - .then(() => readDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc, runClean)) + if (!ignorePreviouslyIndexedDoc) { + await downloadExistingDocsToLocal() + let filesToProcess = await fetchYuiDocs(projects, specificDocsVersion, runClean) + await fs.mkdirp('tmp/s3-original-docs') + await RSVP.Promise.all(filesToProcess.map(fixBorkedYuidocFiles)) + } + + await readDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc, runClean) .then(docs => { return RSVP.map(projects, projectName => { return RSVP.map(docs[projectName], doc => { From b1f6a44fc895893e807af6388bf3bb0f7e976de9 Mon Sep 17 00:00:00 2001 From: Gaurav Munjal Date: Fri, 15 Nov 2019 04:42:56 -0500 Subject: [PATCH 2/3] Fix ability to generate new doc --- lib/read-docs.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/read-docs.js b/lib/read-docs.js index ddf12a8..7f74a79 100644 --- a/lib/read-docs.js +++ b/lib/read-docs.js @@ -33,9 +33,11 @@ export default function readDocs( ) if (!runClean) { - availableSourceVersions = availableSourceVersions.filter( - version => !prevIndexedVersions.includes(version) - ) + if (!ignorePreviouslyIndexedDoc) { + availableSourceVersions = availableSourceVersions.filter( + version => !prevIndexedVersions.includes(version) + ) + } folders = folders.filter(f => { if (!prevIndexedVersions.includes(f) || ignorePreviouslyIndexedDoc) { From 01bfeb70df8f5abe55a551202b398e1a70c133fa Mon Sep 17 00:00:00 2001 From: Gaurav Munjal Date: Fri, 15 Nov 2019 05:14:30 -0500 Subject: [PATCH 3/3] use a different command line flag --- generate-local.js | 1 + index.js | 3 ++- main.js | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/generate-local.js b/generate-local.js index 9d3ebee..5967e79 100644 --- a/generate-local.js +++ b/generate-local.js @@ -88,5 +88,6 @@ const runCmd = async (cmd, path) => { '--version', version, '--ignorePreviouslyIndexedDoc', + '--no-sync' ]).stdout.pipe(process.stdout) })() diff --git a/index.js b/index.js index 7e4b44c..0d22b21 100644 --- a/index.js +++ b/index.js @@ -12,6 +12,7 @@ let specificDocsVersion = argv.version ? argv.version : '' let ignorePreviouslyIndexedDoc = projects.length !== 0 && specificDocsVersion !== '' && argv.ignorePreviouslyIndexedDoc let runClean = !!argv.clean +let noSync = !!argv.noSync const { apiDocsProcessor } = require('./main.js') -apiDocsProcessor(projects, specificDocsVersion, ignorePreviouslyIndexedDoc, runClean) +apiDocsProcessor(projects, specificDocsVersion, ignorePreviouslyIndexedDoc, runClean, noSync) diff --git a/main.js b/main.js index ae74ddf..9499691 100644 --- a/main.js +++ b/main.js @@ -18,7 +18,8 @@ export async function apiDocsProcessor( projects, specificDocsVersion, ignorePreviouslyIndexedDoc, - runClean + runClean, + noSync ) { RSVP.on('error', reason => { console.log(reason) @@ -28,7 +29,7 @@ export async function apiDocsProcessor( let docsVersionMsg = specificDocsVersion !== '' ? `. For version ${specificDocsVersion}` : '' console.log(`Downloading docs for ${projects.join(' & ')}${docsVersionMsg}`) - if (!ignorePreviouslyIndexedDoc) { + if (!noSync) { await downloadExistingDocsToLocal() let filesToProcess = await fetchYuiDocs(projects, specificDocsVersion, runClean) await fs.mkdirp('tmp/s3-original-docs')