diff --git a/gulpfile.js b/bin/download.js similarity index 50% rename from gulpfile.js rename to bin/download.js index 42d9db7..2fe8215 100644 --- a/gulpfile.js +++ b/bin/download.js @@ -1,84 +1,54 @@ -const gulp = require('gulp'); -const mocha = require('gulp-mocha'); -const argv = require('yargs').argv; -const rimraf = require('rimraf'); -const promiseUtils = require('./lib/utils/promises'); -const Door43Client = require('./'); -const readline = require('readline'); +'use strict'; -const catalogUrl = 'https://api.unfoldingword.org/ts/txt/2/catalog.json'; -const indexPath = './out/library.sqlite'; -const resourceDir = './out/res'; - -var lastProgressId; +const path = require('path'); +const fs = require('fs'); +const mkdirp = require('mkdirp'); +const readline = require('readline'); +const rimraf = require('rimraf'); +const Door43Client = require('../'); +const promiseUtils = require('../lib/utils/promises'); +const util = require('./util'); -function writeProgress(id, total, completed) { - var percent = Math.round(10 * (100 * completed) / total) / 10; - if(id == lastProgressId) { - readline.cursorTo(process.stdout, 0); - readline.clearLine(process.stdout, 0); +exports.command = 'download'; +exports.describe = 'Downloads the resource container for each indexed resource.'; +exports.builder = { + i: { + alias: 'index', + description: 'The path to the index', + default: path.join(process.cwd(), 'index.sqlite') + }, + d: { + alias: 'dir', + description: 'The directory where resource containers will be downloaded', + default: path.join(process.cwd(), 'resource_containers') + }, + f: { + alias: 'force', + description: 'Overwrites the directory if it already exists', + default: false + }, +}; +exports.handler = function(argv) { + // don't overwrite + if(!argv.force) { + let exists = false; + try { + exists = fs.statSync(argv.dir).isFile(); + } catch (err) { + } + if (exists) throw new Error('The resource container directory already exist. Aborting.'); } else { - lastProgressId = id; - process.stdout.write('\n'); + rimraf.sync(argv.dir); } - var progressTitles = { - projects: 'Indexing Projects', - chunks: 'Indexing Chunks', - resources: 'Indexing Resources', - container: 'Downloading Containers', - catalog: 'Indexing Catalogs', - langnames: 'Indexing Target Languages', - 'temp-langnames': 'Indexing Temporary Target Languages', - 'approved-temp-langnames': 'Indexing Approved Temporary Target Languages', - 'new-language-questions': 'Indexing Questionnaire' - }; - process.stdout.write((progressTitles[id] || id) + ' ' + percent + '%'); -} -gulp.task('clean', function (done) { - rimraf.sync('./out'); - done(); -}); - -gulp.task('index', function (done) { - var client = new Door43Client(indexPath, resourceDir); - - if(argv.catalogs) { - client.index.getCatalogs() - .then(function(catalogs) { - var list = []; - for(var catalog of catalogs) { - list.push({ - slug: catalog.slug, - onProgress: writeProgress - }); - } - return promiseUtils.chain(client.updateCatalogIndex, function (err, data) { - console.log(err); - return false; - })(list); - }) - .then(function() { - // so gulp doesn't choke - console.log(); - return Promise.resolve(); - }) - .then(done, done); - return; - } + mkdirp.sync(argv.dir); - client.updatePrimaryIndex(catalogUrl, function(id, total, completed) { - writeProgress(id, total, completed); - }).then(function() { - console.log(); - return Promise.resolve(); - }).then(done, done); -}); + console.log('Downloading resource containers:', argv.dir); -gulp.task('download', function (done) { + // begin download var compression = 'tar'; - if(argv.zip) compression = 'zip'; - var client = new Door43Client(indexPath, resourceDir, {compression_method:compression}); + // if(argv.zip) compression = 'zip'; + var client = new Door43Client(argv.index, argv.dir, {compression_method:compression}); var getLanguages = function(lang) { if(lang) { return client.index.getSourceLanguage(lang) @@ -118,7 +88,7 @@ gulp.task('download', function (done) { var list = []; for(var group of projectGroups) { if(group.constructor !== Array) group = [group]; - for(project of group) { + for(var project of group) { list.push({ projectSlug: project.slug, languageSlug: project.source_language_slug @@ -155,21 +125,13 @@ gulp.task('download', function (done) { } return false; }, {compact: true, onProgress: function(total, completed) { - writeProgress('container', total, completed); + util.logProgress('container', total, completed); }})(list); }) .then(function() { - // so gulp doesn't choke - console.log(); - return Promise.resolve(); + console.log('\nFinished!'); }) - .then(done, done); -}); -gulp.task('list', function(done) { - var client = new Door43Client(indexPath, resourceDir); - client.listResourceContainers() - .then(function(list) { - console.log(list); + .catch(function(err) { + console.error(err); }); -}); -gulp.task('default', ['test']); \ No newline at end of file +}; \ No newline at end of file diff --git a/bin/index.js b/bin/index.js new file mode 100644 index 0000000..56f8339 --- /dev/null +++ b/bin/index.js @@ -0,0 +1,79 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const mkdirp = require('mkdirp'); +const rimraf = require('rimraf'); +const Door43Client = require('../'); +const promiseUtils = require('../lib/utils/promises'); +const util = require('./util'); + +exports.command = 'index'; +exports.describe = 'Generates a brand new index'; +exports.builder = { + i: { + alias: 'index', + description: 'The path to the generated index', + default: path.join(process.cwd(), 'index.sqlite') + }, + f: { + alias: 'force', + description: 'Overwrites the index if it already exists', + default: false + }, + u: { + alias: 'url', + description: 'The api url', + default: 'https://api.unfoldingword.org/ts/txt/2/catalog.json' + } +}; +exports.handler = function(argv) { + let indexPath = argv.index; + + // don't overwrite + if(!argv.force) { + let exists = false; + try { + exists = fs.statSync(indexPath).isFile(); + } catch (err) { + } + if (exists) throw new Error('The index path already exist. Aborting.'); + } else { + rimraf.sync(indexPath); + } + + mkdirp.sync(path.dirname(indexPath)); + + console.log('Generating primary index:'); + var client = new Door43Client(indexPath, null); + client.updatePrimaryIndex(argv.url, function(id, total, completed) { + util.logProgress(id, total, completed); + }).then(function() { + // index the catalogs + console.log('\n\nGenerating catalog indexes:'); + return indexCatalogs(client); + }).catch(function(err) { + console.error(err); + }); +}; + +function indexCatalogs(client) { + return client.index.getCatalogs() + .then(function(catalogs) { + var list = []; + for(var catalog of catalogs) { + list.push({ + slug: catalog.slug, + onProgress: util.logProgress + }); + } + return promiseUtils.chain(client.updateCatalogIndex, function (err, data) { + console.log(err); + return false; + })(list); + }) + .then(function() { + // so gulp doesn't choke + console.log('\nFinished!'); + }); +} \ No newline at end of file diff --git a/bin/util.js b/bin/util.js new file mode 100644 index 0000000..5d22a8d --- /dev/null +++ b/bin/util.js @@ -0,0 +1,34 @@ +const readline = require('readline'); + +var lastProgressId; + +/** + * Displays a progress indicator in the console. + * @param id + * @param total + * @param completed + */ +function writeProgress(id, total, completed) { + var percent = Math.round(10 * (100 * completed) / total) / 10; + if(id == lastProgressId) { + readline.cursorTo(process.stdout, 0); + readline.clearLine(process.stdout, 0); + } else { + lastProgressId = id; + process.stdout.write('\n'); + } + var progressTitles = { + projects: 'Indexing Projects', + chunks: 'Indexing Chunks', + resources: 'Indexing Resources', + container: 'Downloading Containers', + catalog: 'Indexing Catalogs', + langnames: 'Indexing Target Languages', + 'temp-langnames': 'Indexing Temporary Target Languages', + 'approved-temp-langnames': 'Indexing Approved Temporary Target Languages', + 'new-language-questions': 'Indexing Questionnaire' + }; + process.stdout.write((progressTitles[id] || id) + ' ' + percent + '%'); +} + +module.exports.logProgress = writeProgress; \ No newline at end of file diff --git a/client-cli.js b/client-cli.js new file mode 100755 index 0000000..b2c17bf --- /dev/null +++ b/client-cli.js @@ -0,0 +1,11 @@ +#! /usr/bin/env node + +var commandDir = './bin'; + +var argv = require('yargs') + .commandDir(commandDir) + .help('help') + .alias('h', 'help') + .strict(true) + .demand(1) + .argv; \ No newline at end of file diff --git a/lib/main.js b/lib/main.js index a3fb4ad..61cef0f 100644 --- a/lib/main.js +++ b/lib/main.js @@ -22,7 +22,8 @@ const request = require('./request'), * @constructor */ function Client(dbPath, resourceDir, opts) { - mkdirp.sync(resourceDir); + // in case we need the client but don't want to build a resource dir. + if(resourceDir) mkdirp.sync(resourceDir); opts = opts || {compression_method: 'tar'}; const helper = new SqliteHelper('./lib/schema.sqlite', dbPath); diff --git a/package.json b/package.json index 147b751..03d1a1a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "door43-client", - "version": "0.2.1", + "version": "0.2.2", "description": "A client library for interacting with the Door43 Resource Catalog.", "main": "./lib/main.js", "scripts": { @@ -23,16 +23,20 @@ "mocha": "^2.4.5", "yargs": "^4.8.0" }, + "bin": { + "door43-client": "client-cli.js" + }, "dependencies": { "adm-zip": "^0.4.7", "archiver": "^1.0.1", "compressjs": "^1.0.3", - "resource-container": "^0.6.8", "lodash": "^4.13.1", "mkdirp": "^0.5.1", + "resource-container": "^0.6.8", "rimraf": "^2.5.3", "sql.js": "^0.2.21", "tar-fs": "^1.13.0", - "yamljs": "^0.2.8" + "yamljs": "^0.2.8", + "yargs": "^4.8.1" } }