Skip to content

Commit

Permalink
Add parameter -x to ignore the manifest (#34)
Browse files Browse the repository at this point in the history 
- The -x option ignores the manifest and thereby allows a clean sync
(i.e. files in destination not present in the source will be deleted
blindly without needing a manifest)
- The .NET implementation of KuduSync already supports this
- With this change 'wardeploy' can use the -x option to do a kudusync to
any arbitrary directory. Without a clean kudusync (-x option), deploying
to arbitrary directories is error prone as the manifest isn't generated
per target directory, but is instead a global one. Deploying to a new
target directory while referring to manifest from an earlier target
directory is incorrect for obvious reasons.
  • Loading branch information
@shrishrirang
shrishrirang authored Mar 15, 2018
1 parent 9258ad2 commit 3fc5ae3
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 34 deletions.
34 changes: 18 additions & 16 deletions bin/kudusync.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,10 +358,13 @@ var Manifest = (function () {
};
return Manifest;
})();
function kuduSync(fromPath, toPath, targetSubFolder, nextManifestPath, previousManifestPath, ignore, whatIf) {
function kuduSync(fromPath, toPath, targetSubFolder, nextManifestPath, previousManifestPath, ignoreManifest, ignore, whatIf) {
Ensure.argNotNull(fromPath, "fromPath");
Ensure.argNotNull(toPath, "toPath");
Ensure.argNotNull(nextManifestPath, "nextManifestPath");
if(ignoreManifest) {
previousManifestPath = null;
}
var from = new DirectoryInfo(fromPath, fromPath);
var to = new DirectoryInfo(toPath, toPath);
if(!from.exists()) {
Expand All @@ -374,7 +377,7 @@ function kuduSync(fromPath, toPath, targetSubFolder, nextManifestPath, previousM
var ignoreList = parseIgnoreList(ignore);
log("Kudu sync from: '" + from.path() + "' to: '" + to.path() + "'");
return Manifest.load(previousManifestPath).then(function (manifest) {
return kuduSyncDirectory(from, to, from.path(), to.path(), targetSubFolder, manifest, nextManifest, ignoreList, whatIf);
return kuduSyncDirectory(from, to, from.path(), to.path(), targetSubFolder, manifest, nextManifest, ignoreManifest, ignoreList, whatIf);
}).then(function () {
if(!whatIf) {
return Manifest.save(nextManifest, nextManifestPath);
Expand Down Expand Up @@ -434,10 +437,10 @@ function copyFileInternal(fromFile, toFilePath) {
}
return deffered.promise;
}
function deleteFileIfInManifest(file, manifest, rootPath, targetSubFolder, whatIf) {
function deleteFile(file, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf) {
Ensure.argNotNull(file, "file");
var path = file.path();
if(manifest.isPathInManifest(file.path(), rootPath, targetSubFolder)) {
if(ignoreManifest || manifest.isPathInManifest(file.path(), rootPath, targetSubFolder)) {
log("Deleting file: '" + file.relativePath() + "'");
if(!whatIf) {
return Utils.attempt(function () {
Expand All @@ -447,22 +450,22 @@ function deleteFileIfInManifest(file, manifest, rootPath, targetSubFolder, whatI
}
return Q.resolve();
}
function deleteDirectoryRecursive(directory, manifest, rootPath, targetSubFolder, whatIf) {
function deleteDirectoryRecursive(directory, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf) {
Ensure.argNotNull(directory, "directory");
var path = directory.path();
var relativePath = directory.relativePath();
if(!manifest.isPathInManifest(path, rootPath, targetSubFolder)) {
if(!ignoreManifest && !manifest.isPathInManifest(path, rootPath, targetSubFolder)) {
return Q.resolve();
}
return Utils.serialize(function () {
return directory.initializeFilesAndSubDirectoriesLists();
}, function () {
return Utils.mapSerialized(directory.filesList(), function (file) {
return deleteFileIfInManifest(file, manifest, rootPath, targetSubFolder, whatIf);
return deleteFile(file, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf);
});
}, function () {
return Utils.mapSerialized(directory.subDirectoriesList(), function (subDir) {
return deleteDirectoryRecursive(subDir, manifest, rootPath, targetSubFolder, whatIf);
return deleteDirectoryRecursive(subDir, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf);
});
}, function () {
return directory.updateFilesAndSubDirectoriesLists();
Expand All @@ -480,7 +483,7 @@ function deleteDirectoryRecursive(directory, manifest, rootPath, targetSubFolder
return Q.resolve();
});
}
function kuduSyncDirectory(from, to, fromRootPath, toRootPath, targetSubFolder, manifest, outManifest, ignoreList, whatIf) {
function kuduSyncDirectory(from, to, fromRootPath, toRootPath, targetSubFolder, manifest, outManifest, ignoreManifest, ignoreList, whatIf) {
Ensure.argNotNull(from, "from");
Ensure.argNotNull(to, "to");
Ensure.argNotNull(fromRootPath, "fromRootPath");
Expand Down Expand Up @@ -524,23 +527,21 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, targetSubFolder,
return Q.resolve();
}
if(!from.getFile(toFile.name())) {
return deleteFileIfInManifest(toFile, manifest, toRootPath, targetSubFolder, whatIf);
return deleteFile(toFile, manifest, toRootPath, targetSubFolder, ignoreManifest, whatIf);
}
return Q.resolve();
});
}, function () {
return Utils.mapSerialized(to.subDirectoriesList(), function (toSubDirectory) {
if(!from.getSubDirectory(toSubDirectory.name())) {
if(manifest.isPathInManifest(toSubDirectory.path(), toRootPath, targetSubFolder)) {
return deleteDirectoryRecursive(toSubDirectory, manifest, toRootPath, targetSubFolder, whatIf);
}
return deleteDirectoryRecursive(toSubDirectory, manifest, toRootPath, targetSubFolder, ignoreManifest, whatIf);
}
return Q.resolve();
});
}, function () {
return Utils.mapSerialized(from.subDirectoriesList(), function (fromSubDirectory) {
var toSubDirectory = new DirectoryInfo(pathUtil.join(to.path(), fromSubDirectory.name()), toRootPath);
return kuduSyncDirectory(fromSubDirectory, toSubDirectory, fromRootPath, toRootPath, targetSubFolder, manifest, outManifest, ignoreList, whatIf);
return kuduSyncDirectory(fromSubDirectory, toSubDirectory, fromRootPath, toRootPath, targetSubFolder, manifest, outManifest, ignoreManifest, ignoreList, whatIf);
});
});
} catch (err) {
Expand All @@ -551,13 +552,14 @@ function main() {
var commander = require("commander");
var package = require("../package.json");
var path = require("path");
commander.version(package.version).usage("[options]").option("-f, --fromDir <dir path>", "Source directory to sync").option("-t, --toDir <dir path>", "Destination directory to sync").option("-s, --targetSubFolder <dir path>", "A relative sub folder in the destination to create and copy files to").option("-n, --nextManifest <manifest file path>", "Next manifest file path").option("-p, --previousManifest [manifest file path]", "Previous manifest file path").option("-i, --ignore [patterns]", "List of files/directories to ignore and not sync, delimited by ;").option("-q, --quiet", "No logging").option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines").option("-w, --whatIf", "Only log without actual copy/remove of files").option("--perf", "Print out the time it took to complete KuduSync operation").parse(process.argv);
commander.version(package.version).usage("[options]").option("-f, --fromDir <dir path>", "Source directory to sync").option("-t, --toDir <dir path>", "Destination directory to sync").option("-s, --targetSubFolder <dir path>", "A relative sub folder in the destination to create and copy files to").option("-n, --nextManifest <manifest file path>", "Next manifest file path").option("-p, --previousManifest [manifest file path]", "Previous manifest file path").option("-x, --ignoreManifest", "Disables the processing of the manifest file").option("-i, --ignore [patterns]", "List of files/directories to ignore and not sync, delimited by ;").option("-q, --quiet", "No logging").option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines").option("-w, --whatIf", "Only log without actual copy/remove of files").option("--perf", "Print out the time it took to complete KuduSync operation").parse(process.argv);
var commanderValues = commander;
var fromDir = commanderValues.fromDir;
var toDir = commanderValues.toDir;
var targetSubFolder = commanderValues.targetSubFolder;
var previousManifest = commanderValues.previousManifest;
var nextManifest = commanderValues.nextManifest;
var ignoreManifest = commanderValues.ignoreManifest;
var ignore = commanderValues.ignore;
var quiet = commanderValues.quiet;
var verbose = commanderValues.verbose;
Expand Down Expand Up @@ -607,7 +609,7 @@ function main() {
};
}
var start = new Date();
kuduSync(fromDir, toDir, targetSubFolder, nextManifest, previousManifest, ignore, whatIf).then(function () {
kuduSync(fromDir, toDir, targetSubFolder, nextManifest, previousManifest, ignoreManifest, ignore, whatIf).then(function () {
if(perf) {
var stop = new Date();
console.log("Operation took " + ((stop.getTime() - start.getTime()) / 1000) + " seconds");
Expand Down
33 changes: 18 additions & 15 deletions lib/FileUtils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
///<reference path='DirectoryInfo.ts'/>
///<reference path='Manifest.ts'/>

function kuduSync(fromPath: string, toPath: string, targetSubFolder: string, nextManifestPath: string, previousManifestPath: string, ignore: string, whatIf: bool) : Promise {
function kuduSync(fromPath: string, toPath: string, targetSubFolder: string, nextManifestPath: string, previousManifestPath: string, ignoreManifest: bool, ignore: string, whatIf: bool) : Promise {
Ensure.argNotNull(fromPath, "fromPath");
Ensure.argNotNull(toPath, "toPath");
Ensure.argNotNull(nextManifestPath, "nextManifestPath");

if (ignoreManifest) {
previousManifestPath = null;
}

var from = new DirectoryInfo(fromPath, fromPath);
var to = new DirectoryInfo(toPath, toPath);

Expand All @@ -24,7 +28,7 @@ function kuduSync(fromPath: string, toPath: string, targetSubFolder: string, nex
log("Kudu sync from: '" + from.path() + "' to: '" + to.path() + "'");

return Manifest.load(previousManifestPath)
.then((manifest) => kuduSyncDirectory(from, to, from.path(), to.path(), targetSubFolder, manifest, nextManifest, ignoreList, whatIf))
.then((manifest) => kuduSyncDirectory(from, to, from.path(), to.path(), targetSubFolder, manifest, nextManifest, ignoreManifest, ignoreList, whatIf))
.then(() => {
if (!whatIf) {
return Manifest.save(nextManifest, nextManifestPath);
Expand Down Expand Up @@ -97,13 +101,13 @@ function copyFileInternal(fromFile: FileInfo, toFilePath: string): Promise {
return deffered.promise;
}

function deleteFileIfInManifest(file: FileInfo, manifest: Manifest, rootPath: string, targetSubFolder: string, whatIf: bool) : Promise {
function deleteFile(file: FileInfo, manifest: Manifest, rootPath: string, targetSubFolder: string, ignoreManifest: bool, whatIf: bool) : Promise {
Ensure.argNotNull(file, "file");

var path = file.path();

// Remove file only if it was in previous manifest
if (manifest.isPathInManifest(file.path(), rootPath, targetSubFolder)) {
// Remove file only if it was in previous manifest or if manifest is to be ignored
if (ignoreManifest || manifest.isPathInManifest(file.path(), rootPath, targetSubFolder)) {
log("Deleting file: '" + file.relativePath() + "'");

if (!whatIf) {
Expand All @@ -114,14 +118,14 @@ function deleteFileIfInManifest(file: FileInfo, manifest: Manifest, rootPath: st
return Q.resolve();
}

function deleteDirectoryRecursive(directory: DirectoryInfo, manifest: Manifest, rootPath: string, targetSubFolder: string, whatIf: bool) {
function deleteDirectoryRecursive(directory: DirectoryInfo, manifest: Manifest, rootPath: string, targetSubFolder: string, ignoreManifest: bool, whatIf: bool) {
Ensure.argNotNull(directory, "directory");

var path = directory.path();
var relativePath = directory.relativePath();

// Remove directory only if it was in previous manifest
if (!manifest.isPathInManifest(path, rootPath, targetSubFolder)) {
// Remove directory only if it was in previous manifest or if manifest is to be ignored
if (!ignoreManifest && !manifest.isPathInManifest(path, rootPath, targetSubFolder)) {
return Q.resolve();
}

Expand All @@ -131,11 +135,11 @@ function deleteDirectoryRecursive(directory: DirectoryInfo, manifest: Manifest,
},

() => {
return Utils.mapSerialized(directory.filesList(), (file: FileInfo) => deleteFileIfInManifest(file, manifest, rootPath, targetSubFolder, whatIf));
return Utils.mapSerialized(directory.filesList(), (file: FileInfo) => deleteFile(file, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf));
},

() => {
return Utils.mapSerialized(directory.subDirectoriesList(), (subDir) => deleteDirectoryRecursive(subDir, manifest, rootPath, targetSubFolder, whatIf));
return Utils.mapSerialized(directory.subDirectoriesList(), (subDir) => deleteDirectoryRecursive(subDir, manifest, rootPath, targetSubFolder, ignoreManifest, whatIf));
},

() => {
Expand All @@ -157,7 +161,7 @@ function deleteDirectoryRecursive(directory: DirectoryInfo, manifest: Manifest,
});
}

function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath: string, toRootPath: string, targetSubFolder: string, manifest: Manifest, outManifest: Manifest, ignoreList: string[], whatIf: bool) {
function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath: string, toRootPath: string, targetSubFolder: string, manifest: Manifest, outManifest: Manifest, ignoreManifest: bool, ignoreList: string[], whatIf: bool) {
Ensure.argNotNull(from, "from");
Ensure.argNotNull(to, "to");
Ensure.argNotNull(fromRootPath, "fromRootPath");
Expand Down Expand Up @@ -236,7 +240,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
}

if (!from.getFile(toFile.name())) {
return deleteFileIfInManifest(toFile, manifest, toRootPath, targetSubFolder, whatIf);
return deleteFile(toFile, manifest, toRootPath, targetSubFolder, ignoreManifest, whatIf);
}
return Q.resolve();
}
Expand All @@ -251,9 +255,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
// 1. We have no previous directory
// 2. We have a previous directory and the file exists there
if (!from.getSubDirectory(toSubDirectory.name())) {
if (manifest.isPathInManifest(toSubDirectory.path(), toRootPath, targetSubFolder)) {
return deleteDirectoryRecursive(toSubDirectory, manifest, toRootPath, targetSubFolder, whatIf);
}
return deleteDirectoryRecursive(toSubDirectory, manifest, toRootPath, targetSubFolder, ignoreManifest, whatIf);
}
return Q.resolve();
}
Expand All @@ -274,6 +276,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
targetSubFolder,
manifest,
outManifest,
ignoreManifest,
ignoreList,
whatIf);
}
Expand Down
3 changes: 3 additions & 0 deletions lib/Main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ function main() {
.option("-s, --targetSubFolder <dir path>", "A relative sub folder in the destination to create and copy files to")
.option("-n, --nextManifest <manifest file path>", "Next manifest file path")
.option("-p, --previousManifest [manifest file path]", "Previous manifest file path")
.option("-x, --ignoreManifest", "Disables the processing of the manifest file")
.option("-i, --ignore [patterns]", "List of files/directories to ignore and not sync, delimited by ;")
.option("-q, --quiet", "No logging")
.option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines")
Expand All @@ -27,6 +28,7 @@ function main() {
var targetSubFolder = commanderValues.targetSubFolder;
var previousManifest = commanderValues.previousManifest;
var nextManifest = commanderValues.nextManifest;
var ignoreManifest = commanderValues.ignoreManifest;
var ignore = commanderValues.ignore;
var quiet = commanderValues.quiet;
var verbose = commanderValues.verbose;
Expand Down Expand Up @@ -97,6 +99,7 @@ function main() {
targetSubFolder,
nextManifest,
previousManifest,
ignoreManifest,
ignore,
whatIf).then(
() => {
Expand Down
34 changes: 31 additions & 3 deletions test/functionalTests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,19 @@ suite('Kudu Sync Functional Tests', function () {
});
});

test('Single file created then file created only in destination, new file should be removed with -x', function (done) {
runKuduSyncTestScenario(["file1"], ["file1"], null, function (err) {
if (err) {
return done(err);
}

// Generating a file only in the destination directory, this should be removed with -x
generateToFile("tofile");

runKuduSyncTestScenario([], ["file1", "-tofile"], null, done, false /* whatIf */, true /* ignoreManifest */);
});
});

test('Directory should not be removed if not empty', function (done) {
runKuduSyncTestScenario(["file1", "dir1/file2"], ["file1", "dir1/file2"], null, function () {
// Generating a file only in the destination directory, this shouldn't be removed
Expand Down Expand Up @@ -110,6 +123,16 @@ suite('Kudu Sync Functional Tests', function () {
});
});

test('Several files created then file created only in destination, new files should be removed with -x', function (done) {
runKuduSyncTestScenario(["file1", "dir1/file2"], ["file1", "dir1/file2"], null, function () {
// Generating files only in the destination directory, those files should be removed with -x
generateToFile("dir1/dir2/tofile1");
generateToFile("dir1/dir2/tofile2");

runKuduSyncTestScenario(["-file1"], ["-file1", "dir1/file2", "-dir1/dir2/tofile1", "-dir1/dir2/tofile2"], null, done, false /* whatIf */, true /* ignoreManifest */);
});
});

test('File created then removed (resulting in empty manifest) then added while target has an extra file which should stay', function (done) {
runKuduSyncTestScenario(["file1"], ["file1"], null, function () {
// Generating files only in the destination directory, those files shouldn't be removed
Expand Down Expand Up @@ -291,10 +314,10 @@ suite('Kudu Sync Functional Tests', function () {
// 1. Create/update or remove files from updatedFiles on the source path
// 2. Run the kudu sync function
// 3. Verify expectedFiles exist (or not exist) in the destination path
function runKuduSyncTestScenario(updatedFiles, expectedFiles, ignore, callback, whatIf) {
function runKuduSyncTestScenario(updatedFiles, expectedFiles, ignore, callback, whatIf, ignoreManifest) {
generateFromFiles(updatedFiles);

runKuduSync("manifest1", "manifest1", ignore, whatIf, function (err) {
runKuduSync("manifest1", "manifest1", ignoreManifest, ignore, whatIf, function (err) {
if (err) {
callback(err);
return;
Expand Down Expand Up @@ -371,13 +394,18 @@ function testFileShouldBeEqual(file) {
filesShouldBeEqual(from, to, file);
}

function runKuduSync(prevManifestFile, nextManifestFile, ignore, whatIf, callback) {
function runKuduSync(prevManifestFile, nextManifestFile, ignoreManifest, ignore, whatIf, callback) {
var from = pathUtil.join(baseTestTempDir, testDir, fromDir);
var to = pathUtil.join(baseTestTempDir, testDir, toDir);
var prevManifestPath = pathUtil.join(baseTestTempDir, testDir, prevManifestFile);
var nextManifestPath = pathUtil.join(baseTestTempDir, testDir, nextManifestFile);

var command = testTarget.cmd + " -f " + from + " -t " + to + " -n " + nextManifestPath + " -p " + prevManifestPath;

if (ignoreManifest) {
command += " -x";
}

if (ignore) {
command += " -i \"" + ignore + "\"";
}
Expand Down

0 comments on commit 3fc5ae3

Please sign in to comment.