From 71ce5893b444fad5c0feb66c40590ae40e08d8e6 Mon Sep 17 00:00:00 2001 From: Daniel Shimshoni Date: Sat, 13 Feb 2016 21:24:37 -0800 Subject: [PATCH 1/4] Bait-java issue 31 : Hidden files should be ignored. --- .../java/gov/loc/repository/bagit/Bag.java | 4 +- .../gov/loc/repository/bagit/BagFactory.java | 19 ++++-- .../gov/loc/repository/bagit/BagHelper.java | 2 +- .../bagit/driver/CommandLineBagDriver.java | 65 ++++++++++++------- .../bagit/filesystem/FileSystem.java | 3 +- .../bagit/filesystem/FileSystemFactory.java | 9 +-- .../filter/DirNodeFileSystemNodeFilter.java | 11 ++++ .../filter/FileNodeFileSystemNodeFilter.java | 13 +++- .../filter/NotHiddenFileSystemNodeFilter.java | 25 +++++++ .../bagit/filesystem/impl/FileFileSystem.java | 9 ++- .../bagit/filesystem/impl/ZipFileSystem.java | 12 ++-- .../repository/bagit/impl/AbstractBag.java | 13 ++-- .../impl/AddFilesToPayloadOperation.java | 33 ++++++++-- .../verify/impl/CompleteVerifierImpl.java | 2 +- .../bagit/writer/impl/FileSystemWriter.java | 2 +- .../bag/CancelTriggeringBagDecorator.java | 16 ++--- .../filesystem/impl/FileFileSystemTest.java | 3 +- 17 files changed, 177 insertions(+), 64 deletions(-) create mode 100644 src/main/java/gov/loc/repository/bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java diff --git a/src/main/java/gov/loc/repository/bagit/Bag.java b/src/main/java/gov/loc/repository/bagit/Bag.java index 29230c70a..4c075cf56 100644 --- a/src/main/java/gov/loc/repository/bagit/Bag.java +++ b/src/main/java/gov/loc/repository/bagit/Bag.java @@ -238,7 +238,9 @@ enum Format { BagConstants getBagConstants(); BagPartFactory getBagPartFactory(); - + + BagFactory getBagFactory(); + /** *

Contains names for constants associated with a bag. * BagIt defines and reserves several names, and some of those names diff --git a/src/main/java/gov/loc/repository/bagit/BagFactory.java b/src/main/java/gov/loc/repository/bagit/BagFactory.java index 0f69137aa..b25f54669 100644 --- a/src/main/java/gov/loc/repository/bagit/BagFactory.java +++ b/src/main/java/gov/loc/repository/bagit/BagFactory.java @@ -2,6 +2,8 @@ import gov.loc.repository.bagit.Bag.BagConstants; import gov.loc.repository.bagit.Bag.BagPartFactory; +import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter; +import gov.loc.repository.bagit.filesystem.filter.NotHiddenFileSystemNodeFilter; import gov.loc.repository.bagit.impl.PreBagImpl; import java.io.File; @@ -23,7 +25,7 @@ * * @see Bag */ -public class BagFactory { +public class BagFactory { /** *

Specifies the mechanism used to load a bag from disk. @@ -73,7 +75,9 @@ public static Version valueOfString(String versionString) { } } - + + private FileSystemNodeFilter defaultNodeFilter; + /** * The latest version of the BagIt spec. Currently, this * is {@link Version#V0_97 0.97}. @@ -84,7 +88,11 @@ public static Version valueOfString(String versionString) { * Creates an instance of a bag factory. */ public BagFactory() { - + this.defaultNodeFilter = new NotHiddenFileSystemNodeFilter(); + } + + public BagFactory(FileSystemNodeFilter defaultNodeFilter) { + this.defaultNodeFilter = defaultNodeFilter; } /** @@ -300,5 +308,8 @@ public PreBag createPreBag(File dir) { preBag.setFile(dir); return preBag; } - + + public FileSystemNodeFilter getDefaultNodeFilter() { + return this.defaultNodeFilter; + } } diff --git a/src/main/java/gov/loc/repository/bagit/BagHelper.java b/src/main/java/gov/loc/repository/bagit/BagHelper.java index 83d8f310e..0c7f4b679 100644 --- a/src/main/java/gov/loc/repository/bagit/BagHelper.java +++ b/src/main/java/gov/loc/repository/bagit/BagHelper.java @@ -30,7 +30,7 @@ public class BagHelper { public static String getVersion(File bagFile) { DirNode bagFileDirNode = null; try { - bagFileDirNode = FileSystemFactory.getDirNodeForBag(bagFile); + bagFileDirNode = FileSystemFactory.getDirNodeForBag(bagFile, new BagFactory()); log.trace(MessageFormat.format("BagFileDirNode has filepath {0} and is a {1}", bagFileDirNode.getFilepath(), bagFileDirNode.getClass().getSimpleName())); FileNode bagItNode = bagFileDirNode.childFile(BAGIT); diff --git a/src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java b/src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java index d1ba621ab..3893cb79a 100644 --- a/src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java +++ b/src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Map; +import gov.loc.repository.bagit.filesystem.filter.NotHiddenFileSystemNodeFilter; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -144,15 +145,16 @@ public class CommandLineBagDriver { public static final String PARAM_FAIL_MODE = "failmode"; public static final String PARAM_COMPRESSION_LEVEL = "compressionlevel"; public static final String PARAM_MOVE = "move"; - + public static final String PARAM_INCLUDE_HIDDEN = "includehiddenfiles"; + public static final String VALUE_WRITER_FILESYSTEM = Format.FILESYSTEM.name().toLowerCase(); public static final String VALUE_WRITER_ZIP = Format.ZIP.name().toLowerCase(); private static final Log log = LogFactory.getLog(CommandLineBagDriver.class); private Map operationMap = new HashMap(); - private BagFactory bagFactory = new BagFactory(); - + + public static void main(String[] args) throws Exception { CommandLineBagDriver driver = new CommandLineBagDriver(); int ret = driver.execute(args); @@ -372,6 +374,7 @@ private void addOperation(String name, String help, Parameter[] params, String[] jsap.registerParameter(new Switch( PARAM_HELP, JSAP.NO_SHORTFLAG, PARAM_HELP, "Prints help." )); jsap.registerParameter(new Switch(PARAM_VERBOSE, JSAP.NO_SHORTFLAG, PARAM_VERBOSE, "Reports progress of the operation to the console.")); jsap.registerParameter(new Switch(PARAM_LOG_VERBOSE, JSAP.NO_SHORTFLAG, PARAM_LOG_VERBOSE, "Reports progress of the operation to the log.")); + jsap.registerParameter(new Switch( PARAM_INCLUDE_HIDDEN, JSAP.NO_SHORTFLAG, PARAM_INCLUDE_HIDDEN, "Include hidden files." )); this.operationMap.put(name, new Operation(name, jsap, help, examples)); } @@ -571,7 +574,8 @@ private static String argsToString(String[] args) { return sb.toString(); } - private Bag getBag(File sourceFile, Version version, LoadOption loadOption) { + private Bag getBag(BagFactory bagFactory, File sourceFile, Version version, LoadOption loadOption) { + if (version != null) { if (sourceFile != null) { return bagFactory.createBag(sourceFile, version, loadOption); @@ -611,6 +615,17 @@ private int performOperation(Operation operation, JSAPResult config) { destFile = new File(config.getString(PARAM_DESTINATION)); } + BagFactory bagFactory = null; + if (config.getBoolean(PARAM_INCLUDE_HIDDEN)) { + // null filter will include all nodes, including hidden ones + // + bagFactory = new BagFactory(null); + } else { + // The default should filter out hidden files + // + bagFactory = new BagFactory(new NotHiddenFileSystemNodeFilter()); + } + Writer writer = null; if (config.contains(PARAM_WRITER)) { Format format = Format.valueOf(config.getString(PARAM_WRITER).toUpperCase()); @@ -726,7 +741,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){ ValidVerifierImpl verifier = new ValidVerifierImpl(completeVerifier, checksumVerifier); verifier.addProgressListener(listener); verifier.setFailMode(FailMode.valueOf(config.getString(PARAM_FAIL_MODE).toUpperCase())); - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS); try { SimpleResult result = verifier.verify(bag); log.info(result.toString()); @@ -745,7 +760,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){ completeVerifier.setIgnoreSymlinks(config.getBoolean(PARAM_EXCLUDE_SYMLINKS)); completeVerifier.addProgressListener(listener); completeVerifier.setFailMode(FailMode.valueOf(config.getString(PARAM_FAIL_MODE).toUpperCase())); - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS); try { SimpleResult result = completeVerifier.verify(bag); log.info(result.toString()); @@ -758,7 +773,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){ bag.close(); } } else if (OPERATION_VERIFY_TAGMANIFESTS.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS); try { ParallelManifestChecksumVerifier verifier = new ParallelManifestChecksumVerifier(); verifier.addProgressListener(listener); @@ -774,7 +789,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){ bag.close(); } } else if (OPERATION_VERIFY_PAYLOADMANIFESTS.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS); try { SimpleResult result; if (bag.getPayloadManifests().size() == 0) { @@ -795,7 +810,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){ bag.close(); } } else if (OPERATION_MAKE_COMPLETE.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES); try { Bag newBag = completer.complete(bag); try { @@ -807,7 +822,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){ bag.close(); } } else if (OPERATION_UPDATE.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES); try { UpdateCompleter updateCompleter = new UpdateCompleter(bagFactory); updateCompleter.setTagManifestAlgorithm(Algorithm.valueOfBagItAlgorithm(config.getString(PARAM_TAG_MANIFEST_ALGORITHM, Algorithm.MD5.bagItAlgorithm))); @@ -825,7 +840,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){ } } else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES); try { TagManifestCompleter tagManifestCompleter = new TagManifestCompleter(bagFactory); tagManifestCompleter.setTagManifestAlgorithm(Algorithm.valueOfBagItAlgorithm(config.getString(PARAM_TAG_MANIFEST_ALGORITHM, Algorithm.MD5.bagItAlgorithm))); @@ -842,8 +857,9 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { bag.close(); } } else if (OPERATION_UPDATE_PAYLOAD_OXUM.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES); try { + UpdatePayloadOxumCompleter updatePayloadOxumCompleter = new UpdatePayloadOxumCompleter(bagFactory); Bag newBag = updatePayloadOxumCompleter.complete(bag); try { @@ -860,7 +876,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { bag.close(); } } else if (OPERATION_BAG_IN_PLACE.equals(operation.name)) { - PreBag preBag = this.bagFactory.createPreBag(sourceFile); + PreBag preBag = bagFactory.createPreBag(sourceFile); if (config.contains(PARAM_BAGINFOTXT)) { File bagInfoTxtFile = config.getFile(PARAM_BAGINFOTXT); if (! bagInfoTxtFile.getName().equals(bagFactory.getBagConstants().getBagInfoTxt())) { @@ -875,7 +891,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { } preBag.makeBagInPlace(version != null ? version : BagFactory.LATEST, config.getBoolean(PARAM_RETAIN_BASE_DIR, false), config.getBoolean(PARAM_KEEP_EMPTY_DIRS, false), completer); } else if (OPERATION_CREATE.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, null); + Bag bag = this.getBag(bagFactory, sourceFile, version, null); try { for(String filepath : config.getStringArray(PARAM_PAYLOAD)) { if (filepath.endsWith(File.separator + "*")) { @@ -918,7 +934,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { } else if (OPERATION_MAKE_HOLEY.equals(operation.name)) { HolePuncherImpl puncher = new HolePuncherImpl(bagFactory); - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS); try { Bag newBag = puncher.makeHoley(bag, config.getString(PARAM_BASE_URL), ! config.getBoolean(PARAM_EXCLUDE_PAYLOAD_DIR, false), false, config.getBoolean(PARAM_RESUME)); try { @@ -930,7 +946,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { bag.close(); } } else if (OPERATION_GENERATE_PAYLOAD_OXUM.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS); try { String oxum = BagHelper.generatePayloadOxum(bag); log.info("Payload-Oxum is " + oxum); @@ -939,7 +955,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { bag.close(); } } else if (OPERATION_CHECK_PAYLOAD_OXUM.equals(operation.name)) { - Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS); + Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS); try { String genOxum = BagHelper.generatePayloadOxum(bag); BagInfoTxt bagInfo = bag.getBagInfoTxt(); @@ -971,7 +987,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { } } else if (OPERATION_FILL_HOLEY.equals(operation.name)) { FileSystemFileDestination dest = new FileSystemFileDestination(sourceFile); - Bag bag = this.getBag(sourceFile, version, null); + Bag bag = this.getBag(bagFactory, sourceFile, version, null); try { SimpleResult result = fetcher.fetch(bag, dest, config.getBoolean(PARAM_RESUME), config.getBoolean(PARAM_VERIFY)); log.info(result.toString()); @@ -996,7 +1012,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { || OPERATION_SPLIT_BAG_BY_FILE_TYPE.equals(operation.name) || OPERATION_SPLIT_BAG_BY_SIZE_AND_FILE_TYPE.equals(operation.name)) { - Bag srcBag = this.bagFactory.createBag(sourceFile, BagFactory.LoadOption.BY_FILES); + Bag srcBag = bagFactory.createBag(sourceFile, BagFactory.LoadOption.BY_FILES); try { Double sourceBagSize = null; if(srcBag.getBagInfoTxt() != null && srcBag.getBagInfoTxt().getPayloadOxum() != null){ @@ -1030,7 +1046,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { return RETURN_FAILURE; } - Splitter splitter = new SplitBySize(this.bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs); + Splitter splitter = new SplitBySize(bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs); List splitBags = splitter.split(srcBag); try { this.completeAndWriteBagToDisk(splitBags, completer, writer, srcBag, destBagFile, true); @@ -1044,7 +1060,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { return RETURN_FAILURE; } - Splitter splitter = new SplitByFileType(this.bagFactory, fileExtensionsIn, excludeDirs); + Splitter splitter = new SplitByFileType(bagFactory, fileExtensionsIn, excludeDirs); List splitBags = splitter.split(srcBag); try { this.completeAndWriteBagToDisk(splitBags, completer, writer, srcBag, destBagFile, false); @@ -1058,9 +1074,9 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { return RETURN_FAILURE; } - Splitter splitter1 = new SplitByFileType(this.bagFactory, fileExtensionsIn, excludeDirs); + Splitter splitter1 = new SplitByFileType(bagFactory, fileExtensionsIn, excludeDirs); List bags = splitter1.split(srcBag); - Splitter splitter2 = new SplitBySize(this.bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs); + Splitter splitter2 = new SplitBySize(bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs); try { for(Bag bag : bags) { List bagsUnderMaxSize = new ArrayList(); @@ -1104,7 +1120,8 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) { private void completeAndWriteBagToDisk(List bags, Completer completer, Writer writer, Bag srcBag, File destBagFile, boolean appendNumber){ - int i = 0; + BagFactory bagFactory = srcBag.getBagFactory(); + int i = 0; for(Bag bag : bags) { Bag newBag = completer.complete(bag); diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/FileSystem.java b/src/main/java/gov/loc/repository/bagit/filesystem/FileSystem.java index fd8b014c8..2fac1e7d9 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/FileSystem.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/FileSystem.java @@ -3,7 +3,7 @@ import java.io.Closeable; import java.io.File; -public interface FileSystem extends Closeable { +public interface FileSystem extends Closeable { DirNode getRoot(); /* * The file that represents the file system. @@ -11,5 +11,6 @@ public interface FileSystem extends Closeable { */ File getFile(); FileNode resolve(String filepath); + FileSystemNodeFilter getDefaultNodeFilter(); void closeQuietly(); } diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/FileSystemFactory.java b/src/main/java/gov/loc/repository/bagit/filesystem/FileSystemFactory.java index 0fa3a50d5..a58e7be66 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/FileSystemFactory.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/FileSystemFactory.java @@ -1,6 +1,7 @@ package gov.loc.repository.bagit.filesystem; import gov.loc.repository.bagit.Bag.Format; +import gov.loc.repository.bagit.BagFactory; import gov.loc.repository.bagit.filesystem.impl.FileFileSystem; import gov.loc.repository.bagit.filesystem.impl.ZipFileSystem; import gov.loc.repository.bagit.utilities.FormatHelper; @@ -11,7 +12,7 @@ public class FileSystemFactory { - public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatException, UnsupportedFormatException { + public static DirNode getDirNodeForBag(File fileForBag, BagFactory bagFactory) throws UnknownFormatException, UnsupportedFormatException { assert fileForBag != null; if (! fileForBag.exists()) { @@ -21,7 +22,7 @@ public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatExce Format format = FormatHelper.getFormat(fileForBag); FileSystem fs = null; if (Format.FILESYSTEM == format) { - fs = new FileFileSystem(fileForBag); + fs = new FileFileSystem(fileForBag, bagFactory.getDefaultNodeFilter()); } else if (Format.ZIP == format) { fs = new ZipFileSystem(fileForBag); } else { @@ -30,11 +31,11 @@ public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatExce DirNode root = fs.getRoot(); if (format.isSerialized) { - if (root.listChildren().size() != 1) { + if (root.listChildren(fs.getDefaultNodeFilter()).size() != 1) { root.getFileSystem().closeQuietly(); throw new RuntimeException("Unable to find bag_dir in serialized bag"); } - FileSystemNode bagDirNode = root.listChildren().iterator().next(); + FileSystemNode bagDirNode = root.listChildren(fs.getDefaultNodeFilter()).iterator().next(); if (! (bagDirNode instanceof DirNode)) { root.getFileSystem().closeQuietly(); throw new RuntimeException("Unable to find bag_dir in serialized bag"); diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java b/src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java index ef4d51f64..2113d0873 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java @@ -3,11 +3,22 @@ import gov.loc.repository.bagit.filesystem.DirNode; import gov.loc.repository.bagit.filesystem.FileSystemNode; import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter; +import gov.loc.repository.bagit.filesystem.impl.AbstractFileNode; public class DirNodeFileSystemNodeFilter implements FileSystemNodeFilter { @Override public boolean accept(FileSystemNode fileSystemNode) { + if (fileSystemNode.getFileSystem().getDefaultNodeFilter() != null && + fileSystemNode instanceof AbstractFileNode) { + // additional checks are required + // + if (!fileSystemNode.getFileSystem().getDefaultNodeFilter().accept(fileSystemNode)) { + // excluded by the basic filter + // + return false; + } + } return fileSystemNode instanceof DirNode; } diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/filter/FileNodeFileSystemNodeFilter.java b/src/main/java/gov/loc/repository/bagit/filesystem/filter/FileNodeFileSystemNodeFilter.java index 06349965e..c5e073802 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/filter/FileNodeFileSystemNodeFilter.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/filter/FileNodeFileSystemNodeFilter.java @@ -3,12 +3,23 @@ import gov.loc.repository.bagit.filesystem.FileNode; import gov.loc.repository.bagit.filesystem.FileSystemNode; import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter; +import gov.loc.repository.bagit.filesystem.impl.AbstractFileNode; public class FileNodeFileSystemNodeFilter implements FileSystemNodeFilter { @Override public boolean accept(FileSystemNode fileSystemNode) { + if (fileSystemNode.getFileSystem().getDefaultNodeFilter() != null && + fileSystemNode instanceof AbstractFileNode) { + // additional checks are required + // + if (!fileSystemNode.getFileSystem().getDefaultNodeFilter().accept(fileSystemNode)) { + // excluded by the basic filter + // + return false; + } + } + return fileSystemNode instanceof FileNode; } - } diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java b/src/main/java/gov/loc/repository/bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java new file mode 100644 index 000000000..50ebedfb5 --- /dev/null +++ b/src/main/java/gov/loc/repository/bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java @@ -0,0 +1,25 @@ +package gov.loc.repository.bagit.filesystem.filter; + +import gov.loc.repository.bagit.filesystem.FileSystemNode; +import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter; +import gov.loc.repository.bagit.filesystem.impl.AbstractFileNode; + +/** + * Created by daniels on 2/12/16. + */ +public class NotHiddenFileSystemNodeFilter implements FileSystemNodeFilter { + + @Override + public boolean accept(FileSystemNode fileSystemNode) { + if (fileSystemNode instanceof AbstractFileNode) { + // File-based node, check if resource is hidden + // + AbstractFileNode fileNode = (AbstractFileNode) fileSystemNode; + return !fileNode.getFile().isHidden(); + } + + // Not a file-based node, accept without checking + // + return true; + } +} diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/impl/FileFileSystem.java b/src/main/java/gov/loc/repository/bagit/filesystem/impl/FileFileSystem.java index a49a0f495..08891e8ef 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/impl/FileFileSystem.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/impl/FileFileSystem.java @@ -5,17 +5,19 @@ import gov.loc.repository.bagit.filesystem.DirNode; import gov.loc.repository.bagit.filesystem.FileNode; import gov.loc.repository.bagit.filesystem.FileSystem; +import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter; public class FileFileSystem implements FileSystem { private File file; private DirNode dirNode; + private FileSystemNodeFilter defaultNodeFilter; - public FileFileSystem(File file) { + public FileFileSystem(File file, FileSystemNodeFilter defaultNodeFilter) { assert file != null; if (! file.isDirectory()){ throw new RuntimeException("Not a directory");} this.file = file; - + this.defaultNodeFilter = defaultNodeFilter; this.dirNode = new FileDirNode(file, this); } @@ -45,4 +47,7 @@ public FileNode resolve(String filepath) { return new FileFileNode(resolvedFile, this); } + public FileSystemNodeFilter getDefaultNodeFilter() { + return this.defaultNodeFilter; + } } diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/impl/ZipFileSystem.java b/src/main/java/gov/loc/repository/bagit/filesystem/impl/ZipFileSystem.java index f6cb307a4..c3bd84caf 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/impl/ZipFileSystem.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/impl/ZipFileSystem.java @@ -1,9 +1,6 @@ package gov.loc.repository.bagit.filesystem.impl; -import gov.loc.repository.bagit.filesystem.DirNode; -import gov.loc.repository.bagit.filesystem.FileNode; -import gov.loc.repository.bagit.filesystem.FileSystem; -import gov.loc.repository.bagit.filesystem.FileSystemNode; +import gov.loc.repository.bagit.filesystem.*; import java.io.File; import java.io.IOException; @@ -114,7 +111,12 @@ public FileNode resolve(String filepath) { log.trace(MessageFormat.format("Resolving {0}", filepath)); return new ZipFileNode(this.zipFile.getEntry(filepath), filepath, this); } - + + @Override + public FileSystemNodeFilter getDefaultNodeFilter() { + return null; + } + @Override protected void finalize() throws Throwable { this.close(); diff --git a/src/main/java/gov/loc/repository/bagit/impl/AbstractBag.java b/src/main/java/gov/loc/repository/bagit/impl/AbstractBag.java index fae254c6b..50861ee83 100644 --- a/src/main/java/gov/loc/repository/bagit/impl/AbstractBag.java +++ b/src/main/java/gov/loc/repository/bagit/impl/AbstractBag.java @@ -106,12 +106,12 @@ public void loadFromManifests() { DirNode bagFileDirNode = null; try { - bagFileDirNode = FileSystemFactory.getDirNodeForBag(this.fileForBag); + bagFileDirNode = FileSystemFactory.getDirNodeForBag(this.fileForBag, this.bagFactory); log.trace(MessageFormat.format("BagFileDirNode has filepath {0} and is a {1}", bagFileDirNode.getFilepath(), bagFileDirNode.getClass().getSimpleName())); //Load root tag map - for(FileSystemNode node : bagFileDirNode.listChildren()) { + for(FileSystemNode node : bagFileDirNode.listChildren(this.bagFactory.getDefaultNodeFilter())) { if (node instanceof FileNode) { FileNode tagFileNode = (FileNode)node; String filepath = FilenameHelper.removeBasePath(bagFileDirNode.getFilepath(), tagFileNode.getFilepath()); @@ -165,7 +165,7 @@ public void loadFromFiles(List ignoreAdditionalDirectories) { DirNode bagFileDirNode; try { - bagFileDirNode = FileSystemFactory.getDirNodeForBag(this.fileForBag); + bagFileDirNode = FileSystemFactory.getDirNodeForBag(this.fileForBag, this.bagFactory); } catch (UnknownFormatException e) { throw new RuntimeException(e); } catch (UnsupportedFormatException e) { @@ -519,7 +519,12 @@ public BagConstants getBagConstants() { public BagPartFactory getBagPartFactory() { return this.bagPartFactory; } - + + @Override + public BagFactory getBagFactory() { + return this.bagFactory; + } + @Override public Bag write(Writer writer, File file) { checkClosed(); diff --git a/src/main/java/gov/loc/repository/bagit/impl/AddFilesToPayloadOperation.java b/src/main/java/gov/loc/repository/bagit/impl/AddFilesToPayloadOperation.java index 4e6889ddc..ef544467c 100644 --- a/src/main/java/gov/loc/repository/bagit/impl/AddFilesToPayloadOperation.java +++ b/src/main/java/gov/loc/repository/bagit/impl/AddFilesToPayloadOperation.java @@ -4,6 +4,12 @@ import java.text.MessageFormat; import java.util.List; +import gov.loc.repository.bagit.filesystem.FileSystem; +import gov.loc.repository.bagit.filesystem.FileSystemFactory; +import gov.loc.repository.bagit.filesystem.FileSystemNode; +import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter; +import gov.loc.repository.bagit.filesystem.impl.FileFileNode; +import gov.loc.repository.bagit.filesystem.impl.FileFileSystem; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,23 +41,40 @@ public void addFileToPayload(File file) { private int addPayload(File file, File rootDir, int count) { if (this.isCancelled()){ return 0;} + file = FileHelper.normalizeForm(file); if (! file.canRead()) { throw new RuntimeException("Can't read " + file.toString()); } + //If directory, recurse on children File[] files = file.listFiles(); - if (file.isDirectory() && files != null) { + if (file.isDirectory() && files != null) { + FileSystem fileSystem = new FileFileSystem(rootDir != null ? rootDir : file, this.bag.getBagFactory().getDefaultNodeFilter()); + FileSystemNodeFilter nodeFilter = fileSystem.getDefaultNodeFilter(); + for(File child : files) { if (this.isCancelled()){ return 0;} - String filepath = file.getAbsolutePath(); + + String filepath = child.getAbsolutePath(); + + // check filters + // + if (nodeFilter != null) { + FileSystemNode node = fileSystem.resolve(filepath); + if (!nodeFilter.accept(node)) { + this.progress("Excluding payload file from data directory", filepath, count, null); + continue; + } + } + this.progress("Adding payload file to data directory", filepath, count, null); log.trace(MessageFormat.format("Adding payload {0} in data directory", filepath)); count = this.addPayload(child, rootDir, count); } - + } else if (file.isFile()) { - + //If file, add to payloadMap String filepath = this.bag.getBagConstants().getDataDirectory() + "/"; if (rootDir != null) { @@ -62,7 +85,7 @@ private int addPayload(File file, File rootDir, int count) { if (filepath.indexOf('\\') != -1) { throw new UnsupportedOperationException(MessageFormat.format("This Library does not support \\ in filepaths: {0}. See README.txt.", filepath)); } - count++; + count++; log.debug(MessageFormat.format("Adding {0} to payload.", filepath)); this.bag.putBagFile(new FileBagFile(filepath, file)); } diff --git a/src/main/java/gov/loc/repository/bagit/verify/impl/CompleteVerifierImpl.java b/src/main/java/gov/loc/repository/bagit/verify/impl/CompleteVerifierImpl.java index 042a2c1bc..b9f500a4f 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/impl/CompleteVerifierImpl.java +++ b/src/main/java/gov/loc/repository/bagit/verify/impl/CompleteVerifierImpl.java @@ -207,7 +207,7 @@ public SimpleResult verify(Bag bag) { if (bag.getFile() != null) { DirNode bagDirNode; try { - bagDirNode = FileSystemFactory.getDirNodeForBag(bag.getFile()); + bagDirNode = FileSystemFactory.getDirNodeForBag(bag.getFile(), bag.getBagFactory()); } catch (UnknownFormatException e) { throw new RuntimeException(e); } catch (UnsupportedFormatException e) { diff --git a/src/main/java/gov/loc/repository/bagit/writer/impl/FileSystemWriter.java b/src/main/java/gov/loc/repository/bagit/writer/impl/FileSystemWriter.java index be387af9c..101c9c24a 100644 --- a/src/main/java/gov/loc/repository/bagit/writer/impl/FileSystemWriter.java +++ b/src/main/java/gov/loc/repository/bagit/writer/impl/FileSystemWriter.java @@ -149,7 +149,7 @@ public void startBag(Bag bag) { throw new RuntimeException(ex); } this.newBag = this.bagFactory.createBag(this.newBagDir, bag.getBagConstants().getVersion(), LoadOption.NO_LOAD); - this.fileSystem = new FileFileSystem(this.newBagDir); + this.fileSystem = new FileFileSystem(this.newBagDir, bag.getBagFactory().getDefaultNodeFilter()); this.fileCount = 0; this.fileTotal = bag.getTags().size() + bag.getPayload().size(); this.origBag = bag; diff --git a/src/test/java/gov/loc/repository/bagit/bag/CancelTriggeringBagDecorator.java b/src/test/java/gov/loc/repository/bagit/bag/CancelTriggeringBagDecorator.java index a19a690d2..bebcd78bd 100644 --- a/src/test/java/gov/loc/repository/bagit/bag/CancelTriggeringBagDecorator.java +++ b/src/test/java/gov/loc/repository/bagit/bag/CancelTriggeringBagDecorator.java @@ -6,15 +6,7 @@ import java.util.List; import java.util.Map; -import gov.loc.repository.bagit.Bag; -import gov.loc.repository.bagit.BagFile; -import gov.loc.repository.bagit.BagInfoTxt; -import gov.loc.repository.bagit.BagItTxt; -import gov.loc.repository.bagit.BagVisitor; -import gov.loc.repository.bagit.Cancellable; -import gov.loc.repository.bagit.FetchTxt; -import gov.loc.repository.bagit.Manifest; -import gov.loc.repository.bagit.ProgressListener; +import gov.loc.repository.bagit.*; import gov.loc.repository.bagit.BagFactory.Version; import gov.loc.repository.bagit.Manifest.Algorithm; import gov.loc.repository.bagit.transformer.Completer; @@ -96,6 +88,12 @@ public BagPartFactory getBagPartFactory() return realBag.getBagPartFactory(); } + @Override + public BagFactory getBagFactory() { + this.increment(); + return realBag.getBagFactory(); + } + public Map getChecksums(String filepath) { this.increment(); diff --git a/src/test/java/gov/loc/repository/bagit/filesystem/impl/FileFileSystemTest.java b/src/test/java/gov/loc/repository/bagit/filesystem/impl/FileFileSystemTest.java index 14e9fdcf9..4cd2096e8 100644 --- a/src/test/java/gov/loc/repository/bagit/filesystem/impl/FileFileSystemTest.java +++ b/src/test/java/gov/loc/repository/bagit/filesystem/impl/FileFileSystemTest.java @@ -1,5 +1,6 @@ package gov.loc.repository.bagit.filesystem.impl; +import gov.loc.repository.bagit.BagFactory; import gov.loc.repository.bagit.filesystem.FileSystem; import gov.loc.repository.bagit.utilities.ResourceHelper; @@ -21,7 +22,7 @@ public void setup() throws Exception { @Override FileSystem getFileSystem() { - return new FileFileSystem(this.rootFile); + return new FileFileSystem(this.rootFile, null); } } From 8c642be247985fc8dfad4abcdb011f61c8a7fbb4 Mon Sep 17 00:00:00 2001 From: Daniel Shimshoni Date: Mon, 22 Feb 2016 16:37:39 -0800 Subject: [PATCH 2/4] Avoid silly pmd warnings --- .../filter/DirNodeFileSystemNodeFilter.java | 12 +++--------- .../filter/FileNodeFileSystemNodeFilter.java | 11 +++-------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java b/src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java index 2113d0873..a07865d4c 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java @@ -6,20 +6,14 @@ import gov.loc.repository.bagit.filesystem.impl.AbstractFileNode; public class DirNodeFileSystemNodeFilter implements FileSystemNodeFilter { - @Override public boolean accept(FileSystemNode fileSystemNode) { if (fileSystemNode.getFileSystem().getDefaultNodeFilter() != null && - fileSystemNode instanceof AbstractFileNode) { - // additional checks are required - // - if (!fileSystemNode.getFileSystem().getDefaultNodeFilter().accept(fileSystemNode)) { - // excluded by the basic filter - // + fileSystemNode instanceof AbstractFileNode && + !fileSystemNode.getFileSystem().getDefaultNodeFilter().accept(fileSystemNode)) { + // excluded by the default filter return false; - } } return fileSystemNode instanceof DirNode; } - } diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/filter/FileNodeFileSystemNodeFilter.java b/src/main/java/gov/loc/repository/bagit/filesystem/filter/FileNodeFileSystemNodeFilter.java index c5e073802..3b9b30a6b 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/filter/FileNodeFileSystemNodeFilter.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/filter/FileNodeFileSystemNodeFilter.java @@ -10,16 +10,11 @@ public class FileNodeFileSystemNodeFilter implements FileSystemNodeFilter { @Override public boolean accept(FileSystemNode fileSystemNode) { if (fileSystemNode.getFileSystem().getDefaultNodeFilter() != null && - fileSystemNode instanceof AbstractFileNode) { - // additional checks are required - // - if (!fileSystemNode.getFileSystem().getDefaultNodeFilter().accept(fileSystemNode)) { - // excluded by the basic filter - // + fileSystemNode instanceof AbstractFileNode && + !fileSystemNode.getFileSystem().getDefaultNodeFilter().accept(fileSystemNode)) { + // excluded by the default filter return false; } - } - return fileSystemNode instanceof FileNode; } } From 31fa38a9b4e1a41c10ebd868317b0d4f5f718f7f Mon Sep 17 00:00:00 2001 From: Daniel Shimshoni Date: Tue, 23 Feb 2016 16:30:55 -0800 Subject: [PATCH 3/4] .keep is a hidden file. We should test with a null filter (not ignoring hidden files) in order to not break the unit test that expects the .keep file in the payload. --- .../repository/bagit/impl/PreBagImplTest.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/test/java/gov/loc/repository/bagit/impl/PreBagImplTest.java b/src/test/java/gov/loc/repository/bagit/impl/PreBagImplTest.java index 60f0a86a7..a08ad7358 100644 --- a/src/test/java/gov/loc/repository/bagit/impl/PreBagImplTest.java +++ b/src/test/java/gov/loc/repository/bagit/impl/PreBagImplTest.java @@ -20,15 +20,15 @@ public class PreBagImplTest { - BagFactory bagFactory = new BagFactory(); - @Test public void testBagInPlaceWithExistingDataDir() throws Exception { File testDir = createTestBag(true); assertTrue(testDir.exists()); File testDataDir = new File(testDir, "data"); assertTrue(testDataDir.exists()); - + + BagFactory bagFactory = new BagFactory(); + PreBag preBag = bagFactory.createPreBag(testDir); Bag bag = preBag.makeBagInPlace(BagFactory.LATEST, false); try { @@ -45,7 +45,8 @@ public void testBagInPlaceRetainingBaseDir() throws Exception { assertTrue(testDir.exists()); File testDataDir = new File(testDir, "data"); assertFalse(testDataDir.exists()); - + + BagFactory bagFactory = new BagFactory(); PreBag preBag = bagFactory.createPreBag(testDir); Bag bag = preBag.makeBagInPlace(BagFactory.LATEST, true); try { @@ -65,7 +66,8 @@ public void testBagInPlaceNotRetainingBaseDir() throws Exception { assertTrue(testDir.exists()); File testDataDir = new File(testDir, "data"); assertFalse(testDataDir.exists()); - + + BagFactory bagFactory = new BagFactory(); PreBag preBag = bagFactory.createPreBag(testDir); Bag bag = preBag.makeBagInPlace(BagFactory.LATEST, false); try { @@ -93,6 +95,7 @@ public void testBagInPlaceWithIgnoredExtraDir() throws Exception { FileUtils.write(extraFile, "extra"); assertTrue(extraFile.exists()); + BagFactory bagFactory = new BagFactory(); PreBag preBag = bagFactory.createPreBag(testDir); List ignoreDirs = new ArrayList(); ignoreDirs.add("extra"); @@ -128,6 +131,7 @@ public void testBagInPlaceWithDataDirAndTagDirPrev97() throws Exception { FileUtils.write(extraFile, "extra"); assertTrue(extraFile.exists()); + BagFactory bagFactory = new BagFactory(); PreBag preBag = bagFactory.createPreBag(testDir); preBag.makeBagInPlace(Version.V0_96, false); } @@ -146,6 +150,7 @@ public void testBagInPlaceWithDataDirAndTagDirPostv97() throws Exception { FileUtils.write(extraFile, "extra"); assertTrue(extraFile.exists()); + BagFactory bagFactory = new BagFactory(); PreBag preBag = bagFactory.createPreBag(testDir); Bag bag = preBag.makeBagInPlace(BagFactory.LATEST, false); try { @@ -175,7 +180,8 @@ public void testBagInPlaceWithEmptyDir() throws Exception { assertTrue(emptyDir.exists()); File testDataDir = new File(testDir, "data"); assertFalse(testDataDir.exists()); - + + BagFactory bagFactory = new BagFactory(); PreBag preBag = bagFactory.createPreBag(testDir); Bag bag = preBag.makeBagInPlace(BagFactory.LATEST, false); try { @@ -198,7 +204,9 @@ public void testBagInPlaceKeepEmptyDirectories() throws Exception { File emptyDir = new File(testDir, "empty_dir"); FileUtils.forceMkdir(emptyDir); assertTrue(emptyDir.exists()); - + + BagFactory bagFactory = new BagFactory(null); + PreBag preBag = bagFactory.createPreBag(testDir); Bag bag = preBag.makeBagInPlace(BagFactory.LATEST, true, true); try { From ce42831f92bd7e7a3dc4863b9b4cdff1239a45dc Mon Sep 17 00:00:00 2001 From: Daniel Shimshoni Date: Thu, 25 Feb 2016 12:43:22 -0800 Subject: [PATCH 4/4] More comment re-formatting (removed msft-style extra space comment line) --- .../gov/loc/repository/bagit/driver/CommandLineBagDriver.java | 2 -- .../bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java | 2 -- .../loc/repository/bagit/impl/AddFilesToPayloadOperation.java | 1 - 3 files changed, 5 deletions(-) diff --git a/src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java b/src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java index 3893cb79a..b67f0aa95 100644 --- a/src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java +++ b/src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java @@ -618,11 +618,9 @@ private int performOperation(Operation operation, JSAPResult config) { BagFactory bagFactory = null; if (config.getBoolean(PARAM_INCLUDE_HIDDEN)) { // null filter will include all nodes, including hidden ones - // bagFactory = new BagFactory(null); } else { // The default should filter out hidden files - // bagFactory = new BagFactory(new NotHiddenFileSystemNodeFilter()); } diff --git a/src/main/java/gov/loc/repository/bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java b/src/main/java/gov/loc/repository/bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java index 50ebedfb5..7fe1ea996 100644 --- a/src/main/java/gov/loc/repository/bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java +++ b/src/main/java/gov/loc/repository/bagit/filesystem/filter/NotHiddenFileSystemNodeFilter.java @@ -13,13 +13,11 @@ public class NotHiddenFileSystemNodeFilter implements FileSystemNodeFilter { public boolean accept(FileSystemNode fileSystemNode) { if (fileSystemNode instanceof AbstractFileNode) { // File-based node, check if resource is hidden - // AbstractFileNode fileNode = (AbstractFileNode) fileSystemNode; return !fileNode.getFile().isHidden(); } // Not a file-based node, accept without checking - // return true; } } diff --git a/src/main/java/gov/loc/repository/bagit/impl/AddFilesToPayloadOperation.java b/src/main/java/gov/loc/repository/bagit/impl/AddFilesToPayloadOperation.java index ef544467c..c4e0f519a 100644 --- a/src/main/java/gov/loc/repository/bagit/impl/AddFilesToPayloadOperation.java +++ b/src/main/java/gov/loc/repository/bagit/impl/AddFilesToPayloadOperation.java @@ -59,7 +59,6 @@ private int addPayload(File file, File rootDir, int count) { String filepath = child.getAbsolutePath(); // check filters - // if (nodeFilter != null) { FileSystemNode node = fileSystem.resolve(filepath); if (!nodeFilter.accept(node)) {