diff --git a/bin/kudusync.js b/bin/kudusync.js index c59250b..da8be40 100644 --- a/bin/kudusync.js +++ b/bin/kudusync.js @@ -358,10 +358,13 @@ var Manifest = (function () { }; return Manifest; })(); -function kuduSync(fromPath, toPath, targetSubFolder, nextManifestPath, previousManifestPath, ignore, whatIf) { +function kuduSync(fromPath, toPath, targetSubFolder, nextManifestPath, previousManifestPath, ignoreManifest, ignore, whatIf) { Ensure.argNotNull(fromPath, "fromPath"); Ensure.argNotNull(toPath, "toPath"); Ensure.argNotNull(nextManifestPath, "nextManifestPath"); + if(ignoreManifest) { + previousManifestPath = null; + } var from = new DirectoryInfo(fromPath, fromPath); var to = new DirectoryInfo(toPath, toPath); if(!from.exists()) { @@ -374,7 +377,7 @@ function kuduSync(fromPath, toPath, targetSubFolder, nextManifestPath, previousM var ignoreList = parseIgnoreList(ignore); log("Kudu sync from: '" + from.path() + "' to: '" + to.path() + "'"); return Manifest.load(previousManifestPath).then(function (manifest) { - return kuduSyncDirectory(from, to, from.path(), to.path(), targetSubFolder, manifest, nextManifest, ignoreList, whatIf); + return kuduSyncDirectory(from, to, from.path(), to.path(), targetSubFolder, manifest, nextManifest, ignoreManifest, ignoreList, whatIf); }).then(function () { if(!whatIf) { return Manifest.save(nextManifest, nextManifestPath); @@ -434,10 +437,10 @@ function copyFileInternal(fromFile, toFilePath) { } return deffered.promise; } -function deleteFileIfInManifest(file, manifest, rootPath, targetSubFolder, whatIf) { +function deleteFile(file, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf) { Ensure.argNotNull(file, "file"); var path = file.path(); - if(manifest.isPathInManifest(file.path(), rootPath, targetSubFolder)) { + if(ignoreManifest || manifest.isPathInManifest(file.path(), rootPath, targetSubFolder)) { log("Deleting file: '" + file.relativePath() + "'"); if(!whatIf) { return Utils.attempt(function () { @@ -447,22 +450,22 @@ function deleteFileIfInManifest(file, manifest, rootPath, targetSubFolder, whatI } return Q.resolve(); } -function deleteDirectoryRecursive(directory, manifest, rootPath, targetSubFolder, whatIf) { +function deleteDirectoryRecursive(directory, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf) { Ensure.argNotNull(directory, "directory"); var path = directory.path(); var relativePath = directory.relativePath(); - if(!manifest.isPathInManifest(path, rootPath, targetSubFolder)) { + if(!ignoreManifest && !manifest.isPathInManifest(path, rootPath, targetSubFolder)) { return Q.resolve(); } return Utils.serialize(function () { return directory.initializeFilesAndSubDirectoriesLists(); }, function () { return Utils.mapSerialized(directory.filesList(), function (file) { - return deleteFileIfInManifest(file, manifest, rootPath, targetSubFolder, whatIf); + return deleteFile(file, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf); }); }, function () { return Utils.mapSerialized(directory.subDirectoriesList(), function (subDir) { - return deleteDirectoryRecursive(subDir, manifest, rootPath, targetSubFolder, whatIf); + return deleteDirectoryRecursive(subDir, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf); }); }, function () { return directory.updateFilesAndSubDirectoriesLists(); @@ -480,7 +483,7 @@ function deleteDirectoryRecursive(directory, manifest, rootPath, targetSubFolder return Q.resolve(); }); } -function kuduSyncDirectory(from, to, fromRootPath, toRootPath, targetSubFolder, manifest, outManifest, ignoreList, whatIf) { +function kuduSyncDirectory(from, to, fromRootPath, toRootPath, targetSubFolder, manifest, outManifest, ignoreManifest, ignoreList, whatIf) { Ensure.argNotNull(from, "from"); Ensure.argNotNull(to, "to"); Ensure.argNotNull(fromRootPath, "fromRootPath"); @@ -524,23 +527,21 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, targetSubFolder, return Q.resolve(); } if(!from.getFile(toFile.name())) { - return deleteFileIfInManifest(toFile, manifest, toRootPath, targetSubFolder, whatIf); + return deleteFile(toFile, manifest, toRootPath, targetSubFolder, ignoreManifest, whatIf); } return Q.resolve(); }); }, function () { return Utils.mapSerialized(to.subDirectoriesList(), function (toSubDirectory) { if(!from.getSubDirectory(toSubDirectory.name())) { - if(manifest.isPathInManifest(toSubDirectory.path(), toRootPath, targetSubFolder)) { - return deleteDirectoryRecursive(toSubDirectory, manifest, toRootPath, targetSubFolder, whatIf); - } + return deleteDirectoryRecursive(toSubDirectory, manifest, toRootPath, targetSubFolder, ignoreManifest, whatIf); } return Q.resolve(); }); }, function () { return Utils.mapSerialized(from.subDirectoriesList(), function (fromSubDirectory) { var toSubDirectory = new DirectoryInfo(pathUtil.join(to.path(), fromSubDirectory.name()), toRootPath); - return kuduSyncDirectory(fromSubDirectory, toSubDirectory, fromRootPath, toRootPath, targetSubFolder, manifest, outManifest, ignoreList, whatIf); + return kuduSyncDirectory(fromSubDirectory, toSubDirectory, fromRootPath, toRootPath, targetSubFolder, manifest, outManifest, ignoreManifest, ignoreList, whatIf); }); }); } catch (err) { @@ -551,13 +552,14 @@ function main() { var commander = require("commander"); var package = require("../package.json"); var path = require("path"); - commander.version(package.version).usage("[options]").option("-f, --fromDir ", "Source directory to sync").option("-t, --toDir ", "Destination directory to sync").option("-s, --targetSubFolder ", "A relative sub folder in the destination to create and copy files to").option("-n, --nextManifest ", "Next manifest file path").option("-p, --previousManifest [manifest file path]", "Previous manifest file path").option("-i, --ignore [patterns]", "List of files/directories to ignore and not sync, delimited by ;").option("-q, --quiet", "No logging").option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines").option("-w, --whatIf", "Only log without actual copy/remove of files").option("--perf", "Print out the time it took to complete KuduSync operation").parse(process.argv); + commander.version(package.version).usage("[options]").option("-f, --fromDir ", "Source directory to sync").option("-t, --toDir ", "Destination directory to sync").option("-s, --targetSubFolder ", "A relative sub folder in the destination to create and copy files to").option("-n, --nextManifest ", "Next manifest file path").option("-p, --previousManifest [manifest file path]", "Previous manifest file path").option("-x, --ignoreManifest", "Disables the processing of the manifest file").option("-i, --ignore [patterns]", "List of files/directories to ignore and not sync, delimited by ;").option("-q, --quiet", "No logging").option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines").option("-w, --whatIf", "Only log without actual copy/remove of files").option("--perf", "Print out the time it took to complete KuduSync operation").parse(process.argv); var commanderValues = commander; var fromDir = commanderValues.fromDir; var toDir = commanderValues.toDir; var targetSubFolder = commanderValues.targetSubFolder; var previousManifest = commanderValues.previousManifest; var nextManifest = commanderValues.nextManifest; + var ignoreManifest = commanderValues.ignoreManifest; var ignore = commanderValues.ignore; var quiet = commanderValues.quiet; var verbose = commanderValues.verbose; @@ -607,7 +609,7 @@ function main() { }; } var start = new Date(); - kuduSync(fromDir, toDir, targetSubFolder, nextManifest, previousManifest, ignore, whatIf).then(function () { + kuduSync(fromDir, toDir, targetSubFolder, nextManifest, previousManifest, ignoreManifest, ignore, whatIf).then(function () { if(perf) { var stop = new Date(); console.log("Operation took " + ((stop.getTime() - start.getTime()) / 1000) + " seconds"); diff --git a/lib/FileUtils.ts b/lib/FileUtils.ts index 59f83c9..6287968 100644 --- a/lib/FileUtils.ts +++ b/lib/FileUtils.ts @@ -1,11 +1,15 @@ /// /// -function kuduSync(fromPath: string, toPath: string, targetSubFolder: string, nextManifestPath: string, previousManifestPath: string, ignore: string, whatIf: bool) : Promise { +function kuduSync(fromPath: string, toPath: string, targetSubFolder: string, nextManifestPath: string, previousManifestPath: string, ignoreManifest: bool, ignore: string, whatIf: bool) : Promise { Ensure.argNotNull(fromPath, "fromPath"); Ensure.argNotNull(toPath, "toPath"); Ensure.argNotNull(nextManifestPath, "nextManifestPath"); + if (ignoreManifest) { + previousManifestPath = null; + } + var from = new DirectoryInfo(fromPath, fromPath); var to = new DirectoryInfo(toPath, toPath); @@ -24,7 +28,7 @@ function kuduSync(fromPath: string, toPath: string, targetSubFolder: string, nex log("Kudu sync from: '" + from.path() + "' to: '" + to.path() + "'"); return Manifest.load(previousManifestPath) - .then((manifest) => kuduSyncDirectory(from, to, from.path(), to.path(), targetSubFolder, manifest, nextManifest, ignoreList, whatIf)) + .then((manifest) => kuduSyncDirectory(from, to, from.path(), to.path(), targetSubFolder, manifest, nextManifest, ignoreManifest, ignoreList, whatIf)) .then(() => { if (!whatIf) { return Manifest.save(nextManifest, nextManifestPath); @@ -97,13 +101,13 @@ function copyFileInternal(fromFile: FileInfo, toFilePath: string): Promise { return deffered.promise; } -function deleteFileIfInManifest(file: FileInfo, manifest: Manifest, rootPath: string, targetSubFolder: string, whatIf: bool) : Promise { +function deleteFile(file: FileInfo, manifest: Manifest, rootPath: string, targetSubFolder: string, ignoreManifest: bool, whatIf: bool) : Promise { Ensure.argNotNull(file, "file"); var path = file.path(); - // Remove file only if it was in previous manifest - if (manifest.isPathInManifest(file.path(), rootPath, targetSubFolder)) { + // Remove file only if it was in previous manifest or if manifest is to be ignored + if (ignoreManifest || manifest.isPathInManifest(file.path(), rootPath, targetSubFolder)) { log("Deleting file: '" + file.relativePath() + "'"); if (!whatIf) { @@ -114,14 +118,14 @@ function deleteFileIfInManifest(file: FileInfo, manifest: Manifest, rootPath: st return Q.resolve(); } -function deleteDirectoryRecursive(directory: DirectoryInfo, manifest: Manifest, rootPath: string, targetSubFolder: string, whatIf: bool) { +function deleteDirectoryRecursive(directory: DirectoryInfo, manifest: Manifest, rootPath: string, targetSubFolder: string, ignoreManifest: bool, whatIf: bool) { Ensure.argNotNull(directory, "directory"); var path = directory.path(); var relativePath = directory.relativePath(); - // Remove directory only if it was in previous manifest - if (!manifest.isPathInManifest(path, rootPath, targetSubFolder)) { + // Remove directory only if it was in previous manifest or if manifest is to be ignored + if (!ignoreManifest && !manifest.isPathInManifest(path, rootPath, targetSubFolder)) { return Q.resolve(); } @@ -131,11 +135,11 @@ function deleteDirectoryRecursive(directory: DirectoryInfo, manifest: Manifest, }, () => { - return Utils.mapSerialized(directory.filesList(), (file: FileInfo) => deleteFileIfInManifest(file, manifest, rootPath, targetSubFolder, whatIf)); + return Utils.mapSerialized(directory.filesList(), (file: FileInfo) => deleteFile(file, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf)); }, () => { - return Utils.mapSerialized(directory.subDirectoriesList(), (subDir) => deleteDirectoryRecursive(subDir, manifest, rootPath, targetSubFolder, whatIf)); + return Utils.mapSerialized(directory.subDirectoriesList(), (subDir) => deleteDirectoryRecursive(subDir, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf)); }, () => { @@ -157,7 +161,7 @@ function deleteDirectoryRecursive(directory: DirectoryInfo, manifest: Manifest, }); } -function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath: string, toRootPath: string, targetSubFolder: string, manifest: Manifest, outManifest: Manifest, ignoreList: string[], whatIf: bool) { +function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath: string, toRootPath: string, targetSubFolder: string, manifest: Manifest, outManifest: Manifest, ignoreManifest: bool, ignoreList: string[], whatIf: bool) { Ensure.argNotNull(from, "from"); Ensure.argNotNull(to, "to"); Ensure.argNotNull(fromRootPath, "fromRootPath"); @@ -236,7 +240,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath: } if (!from.getFile(toFile.name())) { - return deleteFileIfInManifest(toFile, manifest, toRootPath, targetSubFolder, whatIf); + return deleteFile(toFile, manifest, toRootPath, targetSubFolder, ignoreManifest, whatIf); } return Q.resolve(); } @@ -251,9 +255,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath: // 1. We have no previous directory // 2. We have a previous directory and the file exists there if (!from.getSubDirectory(toSubDirectory.name())) { - if (manifest.isPathInManifest(toSubDirectory.path(), toRootPath, targetSubFolder)) { - return deleteDirectoryRecursive(toSubDirectory, manifest, toRootPath, targetSubFolder, whatIf); - } + return deleteDirectoryRecursive(toSubDirectory, manifest, toRootPath, targetSubFolder, ignoreManifest, whatIf); } return Q.resolve(); } @@ -274,6 +276,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath: targetSubFolder, manifest, outManifest, + ignoreManifest, ignoreList, whatIf); } diff --git a/lib/Main.ts b/lib/Main.ts index 61c5db9..a7137aa 100644 --- a/lib/Main.ts +++ b/lib/Main.ts @@ -14,6 +14,7 @@ function main() { .option("-s, --targetSubFolder ", "A relative sub folder in the destination to create and copy files to") .option("-n, --nextManifest ", "Next manifest file path") .option("-p, --previousManifest [manifest file path]", "Previous manifest file path") + .option("-x, --ignoreManifest", "Disables the processing of the manifest file") .option("-i, --ignore [patterns]", "List of files/directories to ignore and not sync, delimited by ;") .option("-q, --quiet", "No logging") .option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines") @@ -27,6 +28,7 @@ function main() { var targetSubFolder = commanderValues.targetSubFolder; var previousManifest = commanderValues.previousManifest; var nextManifest = commanderValues.nextManifest; + var ignoreManifest = commanderValues.ignoreManifest; var ignore = commanderValues.ignore; var quiet = commanderValues.quiet; var verbose = commanderValues.verbose; @@ -97,6 +99,7 @@ function main() { targetSubFolder, nextManifest, previousManifest, + ignoreManifest, ignore, whatIf).then( () => { diff --git a/test/functionalTests.js b/test/functionalTests.js index 1255f41..d8bd7d6 100644 --- a/test/functionalTests.js +++ b/test/functionalTests.js @@ -82,6 +82,19 @@ suite('Kudu Sync Functional Tests', function () { }); }); + test('Single file created then file created only in destination, new file should be removed with -x', function (done) { + runKuduSyncTestScenario(["file1"], ["file1"], null, function (err) { + if (err) { + return done(err); + } + + // Generating a file only in the destination directory, this should be removed with -x + generateToFile("tofile"); + + runKuduSyncTestScenario([], ["file1", "-tofile"], null, done, false /* whatIf */, true /* ignoreManifest */); + }); + }); + test('Directory should not be removed if not empty', function (done) { runKuduSyncTestScenario(["file1", "dir1/file2"], ["file1", "dir1/file2"], null, function () { // Generating a file only in the destination directory, this shouldn't be removed @@ -110,6 +123,16 @@ suite('Kudu Sync Functional Tests', function () { }); }); + test('Several files created then file created only in destination, new files should be removed with -x', function (done) { + runKuduSyncTestScenario(["file1", "dir1/file2"], ["file1", "dir1/file2"], null, function () { + // Generating files only in the destination directory, those files should be removed with -x + generateToFile("dir1/dir2/tofile1"); + generateToFile("dir1/dir2/tofile2"); + + runKuduSyncTestScenario(["-file1"], ["-file1", "dir1/file2", "-dir1/dir2/tofile1", "-dir1/dir2/tofile2"], null, done, false /* whatIf */, true /* ignoreManifest */); + }); + }); + test('File created then removed (resulting in empty manifest) then added while target has an extra file which should stay', function (done) { runKuduSyncTestScenario(["file1"], ["file1"], null, function () { // Generating files only in the destination directory, those files shouldn't be removed @@ -291,10 +314,10 @@ suite('Kudu Sync Functional Tests', function () { // 1. Create/update or remove files from updatedFiles on the source path // 2. Run the kudu sync function // 3. Verify expectedFiles exist (or not exist) in the destination path -function runKuduSyncTestScenario(updatedFiles, expectedFiles, ignore, callback, whatIf) { +function runKuduSyncTestScenario(updatedFiles, expectedFiles, ignore, callback, whatIf, ignoreManifest) { generateFromFiles(updatedFiles); - runKuduSync("manifest1", "manifest1", ignore, whatIf, function (err) { + runKuduSync("manifest1", "manifest1", ignoreManifest, ignore, whatIf, function (err) { if (err) { callback(err); return; @@ -371,13 +394,18 @@ function testFileShouldBeEqual(file) { filesShouldBeEqual(from, to, file); } -function runKuduSync(prevManifestFile, nextManifestFile, ignore, whatIf, callback) { +function runKuduSync(prevManifestFile, nextManifestFile, ignoreManifest, ignore, whatIf, callback) { var from = pathUtil.join(baseTestTempDir, testDir, fromDir); var to = pathUtil.join(baseTestTempDir, testDir, toDir); var prevManifestPath = pathUtil.join(baseTestTempDir, testDir, prevManifestFile); var nextManifestPath = pathUtil.join(baseTestTempDir, testDir, nextManifestFile); var command = testTarget.cmd + " -f " + from + " -t " + to + " -n " + nextManifestPath + " -p " + prevManifestPath; + + if (ignoreManifest) { + command += " -x"; + } + if (ignore) { command += " -i \"" + ignore + "\""; }