Skip to content

Commit

Permalink
Merge pull request #34 from danshim/master
Browse files Browse the repository at this point in the history 
Bagit-java issue 31 : Hidden files should be ignored.
  • Loading branch information
@johnscancella
johnscancella committed Mar 9, 2016
2 parents d78bbf2 + ce42831 commit 98080c2
Show file tree
Hide file tree
Showing 18 changed files with 178 additions and 73 deletions.
4 changes: 3 additions & 1 deletion src/main/java/gov/loc/repository/bagit/Bag.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,9 @@ enum Format {
BagConstants getBagConstants();

BagPartFactory getBagPartFactory();


BagFactory getBagFactory();

/**
* <p>Contains names for constants associated with a bag.
* BagIt defines and reserves several names, and some of those names
Expand Down
19 changes: 15 additions & 4 deletions src/main/java/gov/loc/repository/bagit/BagFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import gov.loc.repository.bagit.Bag.BagConstants;
import gov.loc.repository.bagit.Bag.BagPartFactory;
import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter;
import gov.loc.repository.bagit.filesystem.filter.NotHiddenFileSystemNodeFilter;
import gov.loc.repository.bagit.impl.PreBagImpl;

import java.io.File;
Expand All @@ -23,7 +25,7 @@
*
* @see Bag
*/
public class BagFactory {
public class BagFactory {

/**
* <p>Specifies the mechanism used to load a bag from disk.
Expand Down Expand Up @@ -73,7 +75,9 @@ public static Version valueOfString(String versionString) {
}

}


private FileSystemNodeFilter defaultNodeFilter;

/**
* The latest version of the BagIt spec. Currently, this
* is {@link Version#V0_97 0.97}.
Expand All @@ -84,7 +88,11 @@ public static Version valueOfString(String versionString) {
* Creates an instance of a bag factory.
*/
public BagFactory() {

this.defaultNodeFilter = new NotHiddenFileSystemNodeFilter();
}

public BagFactory(FileSystemNodeFilter defaultNodeFilter) {
this.defaultNodeFilter = defaultNodeFilter;
}

/**
Expand Down Expand Up @@ -300,5 +308,8 @@ public PreBag createPreBag(File dir) {
preBag.setFile(dir);
return preBag;
}


public FileSystemNodeFilter getDefaultNodeFilter() {
return this.defaultNodeFilter;
}
}
2 changes: 1 addition & 1 deletion src/main/java/gov/loc/repository/bagit/BagHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public class BagHelper {
public static String getVersion(File bagFile) {
DirNode bagFileDirNode = null;
try {
bagFileDirNode = FileSystemFactory.getDirNodeForBag(bagFile);
bagFileDirNode = FileSystemFactory.getDirNodeForBag(bagFile, new BagFactory());
log.trace(MessageFormat.format("BagFileDirNode has filepath {0} and is a {1}", bagFileDirNode.getFilepath(), bagFileDirNode.getClass().getSimpleName()));

FileNode bagItNode = bagFileDirNode.childFile(BAGIT);
Expand Down
63 changes: 39 additions & 24 deletions src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import java.util.List;
import java.util.Map;

import gov.loc.repository.bagit.filesystem.filter.NotHiddenFileSystemNodeFilter;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Expand Down Expand Up @@ -144,15 +145,16 @@ public class CommandLineBagDriver {
public static final String PARAM_FAIL_MODE = "failmode";
public static final String PARAM_COMPRESSION_LEVEL = "compressionlevel";
public static final String PARAM_MOVE = "move";

public static final String PARAM_INCLUDE_HIDDEN = "includehiddenfiles";

public static final String VALUE_WRITER_FILESYSTEM = Format.FILESYSTEM.name().toLowerCase();
public static final String VALUE_WRITER_ZIP = Format.ZIP.name().toLowerCase();

private static final Log log = LogFactory.getLog(CommandLineBagDriver.class);

private Map<String, Operation> operationMap = new HashMap<String, Operation>();
private BagFactory bagFactory = new BagFactory();


public static void main(String[] args) throws Exception {
CommandLineBagDriver driver = new CommandLineBagDriver();
int ret = driver.execute(args);
Expand Down Expand Up @@ -372,6 +374,7 @@ private void addOperation(String name, String help, Parameter[] params, String[]
jsap.registerParameter(new Switch( PARAM_HELP, JSAP.NO_SHORTFLAG, PARAM_HELP, "Prints help." ));
jsap.registerParameter(new Switch(PARAM_VERBOSE, JSAP.NO_SHORTFLAG, PARAM_VERBOSE, "Reports progress of the operation to the console."));
jsap.registerParameter(new Switch(PARAM_LOG_VERBOSE, JSAP.NO_SHORTFLAG, PARAM_LOG_VERBOSE, "Reports progress of the operation to the log."));
jsap.registerParameter(new Switch( PARAM_INCLUDE_HIDDEN, JSAP.NO_SHORTFLAG, PARAM_INCLUDE_HIDDEN, "Include hidden files." ));

this.operationMap.put(name, new Operation(name, jsap, help, examples));
}
Expand Down Expand Up @@ -571,7 +574,8 @@ private static String argsToString(String[] args) {
return sb.toString();
}

private Bag getBag(File sourceFile, Version version, LoadOption loadOption) {
private Bag getBag(BagFactory bagFactory, File sourceFile, Version version, LoadOption loadOption) {

if (version != null) {
if (sourceFile != null) {
return bagFactory.createBag(sourceFile, version, loadOption);
Expand Down Expand Up @@ -611,6 +615,15 @@ private int performOperation(Operation operation, JSAPResult config) {
destFile = new File(config.getString(PARAM_DESTINATION));
}

BagFactory bagFactory = null;
if (config.getBoolean(PARAM_INCLUDE_HIDDEN)) {
// null filter will include all nodes, including hidden ones
bagFactory = new BagFactory(null);
} else {
// The default should filter out hidden files
bagFactory = new BagFactory(new NotHiddenFileSystemNodeFilter());
}

Writer writer = null;
if (config.contains(PARAM_WRITER)) {
Format format = Format.valueOf(config.getString(PARAM_WRITER).toUpperCase());
Expand Down Expand Up @@ -726,7 +739,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
ValidVerifierImpl verifier = new ValidVerifierImpl(completeVerifier, checksumVerifier);
verifier.addProgressListener(listener);
verifier.setFailMode(FailMode.valueOf(config.getString(PARAM_FAIL_MODE).toUpperCase()));
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
try {
SimpleResult result = verifier.verify(bag);
log.info(result.toString());
Expand All @@ -745,7 +758,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
completeVerifier.setIgnoreSymlinks(config.getBoolean(PARAM_EXCLUDE_SYMLINKS));
completeVerifier.addProgressListener(listener);
completeVerifier.setFailMode(FailMode.valueOf(config.getString(PARAM_FAIL_MODE).toUpperCase()));
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
try {
SimpleResult result = completeVerifier.verify(bag);
log.info(result.toString());
Expand All @@ -758,7 +771,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
bag.close();
}
} else if (OPERATION_VERIFY_TAGMANIFESTS.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
try {
ParallelManifestChecksumVerifier verifier = new ParallelManifestChecksumVerifier();
verifier.addProgressListener(listener);
Expand All @@ -774,7 +787,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
bag.close();
}
} else if (OPERATION_VERIFY_PAYLOADMANIFESTS.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
try {
SimpleResult result;
if (bag.getPayloadManifests().size() == 0) {
Expand All @@ -795,7 +808,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
bag.close();
}
} else if (OPERATION_MAKE_COMPLETE.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES);
try {
Bag newBag = completer.complete(bag);
try {
Expand All @@ -807,7 +820,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
bag.close();
}
} else if (OPERATION_UPDATE.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES);
try {
UpdateCompleter updateCompleter = new UpdateCompleter(bagFactory);
updateCompleter.setTagManifestAlgorithm(Algorithm.valueOfBagItAlgorithm(config.getString(PARAM_TAG_MANIFEST_ALGORITHM, Algorithm.MD5.bagItAlgorithm)));
Expand All @@ -825,7 +838,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
}
}
else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES);
try {
TagManifestCompleter tagManifestCompleter = new TagManifestCompleter(bagFactory);
tagManifestCompleter.setTagManifestAlgorithm(Algorithm.valueOfBagItAlgorithm(config.getString(PARAM_TAG_MANIFEST_ALGORITHM, Algorithm.MD5.bagItAlgorithm)));
Expand All @@ -842,8 +855,9 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
bag.close();
}
} else if (OPERATION_UPDATE_PAYLOAD_OXUM.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES);
try {

UpdatePayloadOxumCompleter updatePayloadOxumCompleter = new UpdatePayloadOxumCompleter(bagFactory);
Bag newBag = updatePayloadOxumCompleter.complete(bag);
try {
Expand All @@ -860,7 +874,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
bag.close();
}
} else if (OPERATION_BAG_IN_PLACE.equals(operation.name)) {
PreBag preBag = this.bagFactory.createPreBag(sourceFile);
PreBag preBag = bagFactory.createPreBag(sourceFile);
if (config.contains(PARAM_BAGINFOTXT)) {
File bagInfoTxtFile = config.getFile(PARAM_BAGINFOTXT);
if (! bagInfoTxtFile.getName().equals(bagFactory.getBagConstants().getBagInfoTxt())) {
Expand All @@ -875,7 +889,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
}
preBag.makeBagInPlace(version != null ? version : BagFactory.LATEST, config.getBoolean(PARAM_RETAIN_BASE_DIR, false), config.getBoolean(PARAM_KEEP_EMPTY_DIRS, false), completer);
} else if (OPERATION_CREATE.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, null);
Bag bag = this.getBag(bagFactory, sourceFile, version, null);
try {
for(String filepath : config.getStringArray(PARAM_PAYLOAD)) {
if (filepath.endsWith(File.separator + "*")) {
Expand Down Expand Up @@ -918,7 +932,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {

} else if (OPERATION_MAKE_HOLEY.equals(operation.name)) {
HolePuncherImpl puncher = new HolePuncherImpl(bagFactory);
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
try {
Bag newBag = puncher.makeHoley(bag, config.getString(PARAM_BASE_URL), ! config.getBoolean(PARAM_EXCLUDE_PAYLOAD_DIR, false), false, config.getBoolean(PARAM_RESUME));
try {
Expand All @@ -930,7 +944,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
bag.close();
}
} else if (OPERATION_GENERATE_PAYLOAD_OXUM.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
try {
String oxum = BagHelper.generatePayloadOxum(bag);
log.info("Payload-Oxum is " + oxum);
Expand All @@ -939,7 +953,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
bag.close();
}
} else if (OPERATION_CHECK_PAYLOAD_OXUM.equals(operation.name)) {
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
try {
String genOxum = BagHelper.generatePayloadOxum(bag);
BagInfoTxt bagInfo = bag.getBagInfoTxt();
Expand Down Expand Up @@ -971,7 +985,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
}
} else if (OPERATION_FILL_HOLEY.equals(operation.name)) {
FileSystemFileDestination dest = new FileSystemFileDestination(sourceFile);
Bag bag = this.getBag(sourceFile, version, null);
Bag bag = this.getBag(bagFactory, sourceFile, version, null);
try {
SimpleResult result = fetcher.fetch(bag, dest, config.getBoolean(PARAM_RESUME), config.getBoolean(PARAM_VERIFY));
log.info(result.toString());
Expand All @@ -996,7 +1010,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
|| OPERATION_SPLIT_BAG_BY_FILE_TYPE.equals(operation.name)
|| OPERATION_SPLIT_BAG_BY_SIZE_AND_FILE_TYPE.equals(operation.name)) {

Bag srcBag = this.bagFactory.createBag(sourceFile, BagFactory.LoadOption.BY_FILES);
Bag srcBag = bagFactory.createBag(sourceFile, BagFactory.LoadOption.BY_FILES);
try {
Double sourceBagSize = null;
if(srcBag.getBagInfoTxt() != null && srcBag.getBagInfoTxt().getPayloadOxum() != null){
Expand Down Expand Up @@ -1030,7 +1044,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
return RETURN_FAILURE;
}

Splitter splitter = new SplitBySize(this.bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs);
Splitter splitter = new SplitBySize(bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs);
List<Bag> splitBags = splitter.split(srcBag);
try {
this.completeAndWriteBagToDisk(splitBags, completer, writer, srcBag, destBagFile, true);
Expand All @@ -1044,7 +1058,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
return RETURN_FAILURE;
}

Splitter splitter = new SplitByFileType(this.bagFactory, fileExtensionsIn, excludeDirs);
Splitter splitter = new SplitByFileType(bagFactory, fileExtensionsIn, excludeDirs);
List<Bag> splitBags = splitter.split(srcBag);
try {
this.completeAndWriteBagToDisk(splitBags, completer, writer, srcBag, destBagFile, false);
Expand All @@ -1058,9 +1072,9 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
return RETURN_FAILURE;
}

Splitter splitter1 = new SplitByFileType(this.bagFactory, fileExtensionsIn, excludeDirs);
Splitter splitter1 = new SplitByFileType(bagFactory, fileExtensionsIn, excludeDirs);
List<Bag> bags = splitter1.split(srcBag);
Splitter splitter2 = new SplitBySize(this.bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs);
Splitter splitter2 = new SplitBySize(bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs);
try {
for(Bag bag : bags) {
List<Bag> bagsUnderMaxSize = new ArrayList<Bag>();
Expand Down Expand Up @@ -1104,7 +1118,8 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {

private void completeAndWriteBagToDisk(List<Bag> bags, Completer completer, Writer writer, Bag srcBag, File destBagFile, boolean appendNumber){

int i = 0;
BagFactory bagFactory = srcBag.getBagFactory();
int i = 0;
for(Bag bag : bags) {
Bag newBag = completer.complete(bag);

Expand Down
3 changes: 2 additions & 1 deletion src/main/java/gov/loc/repository/bagit/filesystem/FileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@
import java.io.Closeable;
import java.io.File;

public interface FileSystem extends Closeable {
public interface FileSystem extends Closeable {
DirNode getRoot();
/*
* The file that represents the file system.
* This may be a file or directory depending on the type of file system.
*/
File getFile();
FileNode resolve(String filepath);
FileSystemNodeFilter getDefaultNodeFilter();
void closeQuietly();
}
9 changes: 5 additions & 4 deletions src/main/java/gov/loc/repository/bagit/filesystem/FileSystemFactory.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package gov.loc.repository.bagit.filesystem;

import gov.loc.repository.bagit.Bag.Format;
import gov.loc.repository.bagit.BagFactory;
import gov.loc.repository.bagit.filesystem.impl.FileFileSystem;
import gov.loc.repository.bagit.filesystem.impl.ZipFileSystem;
import gov.loc.repository.bagit.utilities.FormatHelper;
Expand All @@ -11,7 +12,7 @@

public class FileSystemFactory {

public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatException, UnsupportedFormatException {
public static DirNode getDirNodeForBag(File fileForBag, BagFactory bagFactory) throws UnknownFormatException, UnsupportedFormatException {
assert fileForBag != null;

if (! fileForBag.exists()) {
Expand All @@ -21,7 +22,7 @@ public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatExce
Format format = FormatHelper.getFormat(fileForBag);
FileSystem fs = null;
if (Format.FILESYSTEM == format) {
fs = new FileFileSystem(fileForBag);
fs = new FileFileSystem(fileForBag, bagFactory.getDefaultNodeFilter());
} else if (Format.ZIP == format) {
fs = new ZipFileSystem(fileForBag);
} else {
Expand All @@ -30,11 +31,11 @@ public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatExce

DirNode root = fs.getRoot();
if (format.isSerialized) {
if (root.listChildren().size() != 1) {
if (root.listChildren(fs.getDefaultNodeFilter()).size() != 1) {
root.getFileSystem().closeQuietly();
throw new RuntimeException("Unable to find bag_dir in serialized bag");
}
FileSystemNode bagDirNode = root.listChildren().iterator().next();
FileSystemNode bagDirNode = root.listChildren(fs.getDefaultNodeFilter()).iterator().next();
if (! (bagDirNode instanceof DirNode)) {
root.getFileSystem().closeQuietly();
throw new RuntimeException("Unable to find bag_dir in serialized bag");
Expand Down
9 changes: 7 additions & 2 deletions src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,17 @@
import gov.loc.repository.bagit.filesystem.DirNode;
import gov.loc.repository.bagit.filesystem.FileSystemNode;
import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter;
import gov.loc.repository.bagit.filesystem.impl.AbstractFileNode;

public class DirNodeFileSystemNodeFilter implements FileSystemNodeFilter {

@Override
public boolean accept(FileSystemNode fileSystemNode) {
if (fileSystemNode.getFileSystem().getDefaultNodeFilter() != null &&
fileSystemNode instanceof AbstractFileNode &&
!fileSystemNode.getFileSystem().getDefaultNodeFilter().accept(fileSystemNode)) {
// excluded by the default filter
return false;
}
return fileSystemNode instanceof DirNode;
}

}
Loading

0 comments on commit 98080c2

Please sign in to comment.