diff --git a/src/main/java/gov/loc/repository/bagit/impl/AbstractBag.java b/src/main/java/gov/loc/repository/bagit/impl/AbstractBag.java
index 50861ee83..3b65eccce 100644
--- a/src/main/java/gov/loc/repository/bagit/impl/AbstractBag.java
+++ b/src/main/java/gov/loc/repository/bagit/impl/AbstractBag.java
@@ -2,16 +2,19 @@
 
 import java.io.Closeable;
 import java.io.File;
+import java.io.IOException;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -541,24 +544,25 @@ public void removePayloadDirectory(String filepath) {
 			filepath = bagConstants.getDataDirectory() + "/" + filepath;
 		}
 		
-		if ((bagConstants.getDataDirectory() + "/").equals(filepath)) {
-			return;
-		}
-
-		log.debug("Removing payload directory " + filepath);
-		
+		log.debug("remove all the files under " + filepath);
+    //remove all the files. Need to create list so there is no concurrentModificationException
 		List<String> deleteFilepaths = new ArrayList<String>();
+    for(BagFile bagFile : this.getPayload()) {
+      if (bagFile.getFilepath().startsWith(filepath)) {
+        deleteFilepaths.add(bagFile.getFilepath());
+      }
+    }
+    for(String deleteFilepath : deleteFilepaths) {
+      log.debug("Removing " + deleteFilepath);
+      this.removeBagFile(deleteFilepath);     
+    }
 		
-		for(BagFile bagFile : this.getPayload()) {
-			if (bagFile.getFilepath().startsWith(filepath)) {
-				deleteFilepaths.add(bagFile.getFilepath());
-			}
-		}
-		
-		for(String deleteFilepath : deleteFilepaths) {
-			log.debug("Removing " + deleteFilepath);
-			this.removeBagFile(deleteFilepath);			
-		}
+    //now remove the empty directories
+		try {
+      FileUtils.deleteDirectory(new File(filepath));
+    } catch (IOException e) {
+      log.error("Could not delete payload directory [" + filepath + "]!", e);
+    }
 	}
 
 	@Override
@@ -570,21 +574,27 @@ public void removeTagDirectory(String filepath) {
 		if (filepath.startsWith(bagConstants.getDataDirectory())) {
 			throw new RuntimeException("Trying to remove payload");
 		}
-		
-		log.debug("Removing tag directory " + filepath);
-		
+
+		log.debug("remove all the files under " + filepath);
+		//remove all the files. Need to create list so there is no concurrentModificationException
 		List<String> deleteFilepaths = new ArrayList<String>();
+    for(BagFile bagFile : this.getTags()) {
+      if (bagFile.getFilepath().startsWith(filepath)) {
+        deleteFilepaths.add(bagFile.getFilepath());
+      }
+    }
+    for(String deleteFilepath : deleteFilepaths) {
+      log.debug("Removing " + deleteFilepath);
+      this.removeBagFile(deleteFilepath);     
+    }
 		
-		for(BagFile bagFile : this.getTags()) {
-			if (bagFile.getFilepath().startsWith(filepath)) {
-				deleteFilepaths.add(bagFile.getFilepath());
-			}
-		}
-		
-		for(String deleteFilepath : deleteFilepaths) {
-			log.debug("Removing " + deleteFilepath);
-			this.removeBagFile(deleteFilepath);			
-		}
+		log.debug("Removing tag directory " + filepath);
+		//now remove the empty directories
+		try {
+      FileUtils.deleteDirectory(new File(filepath));
+    } catch (IOException e) {
+      log.error("Could not delete tag directory [" + filepath + "]!", e);
+    }
 	}
 
 	
diff --git a/src/test/java/gov/loc/repository/bagit/impl/AbstractBagImplTest.java b/src/test/java/gov/loc/repository/bagit/impl/AbstractBagImplTest.java
index d9d7056d7..db526b836 100644
--- a/src/test/java/gov/loc/repository/bagit/impl/AbstractBagImplTest.java
+++ b/src/test/java/gov/loc/repository/bagit/impl/AbstractBagImplTest.java
@@ -549,9 +549,6 @@ public void testRemoveDirectory() throws Exception {
 	
 			bag.removePayloadDirectory("data/test1.txt");
 			assertNotNull(bag.getBagFile("data/test1.txt"));
-	
-			bag.removePayloadDirectory("data");
-			assertNotNull(bag.getBagFile("data/test1.txt"));
 		} finally {
 			bag.close();
 		}
diff --git a/src/test/java/gov/loc/repository/bagit/v0_96/BagInABagTests.java b/src/test/java/gov/loc/repository/bagit/v0_96/BagInABagTests.java
index 606315a30..e34c294d7 100644
--- a/src/test/java/gov/loc/repository/bagit/v0_96/BagInABagTests.java
+++ b/src/test/java/gov/loc/repository/bagit/v0_96/BagInABagTests.java
@@ -41,9 +41,6 @@ public void testRemoveDirectory() throws Exception
 	
 			bag.removePayloadDirectory("data/bag/data/test1.txt");
 			assertNotNull(bag.getBagFile("data/bag/data/test1.txt"));
-	
-			bag.removePayloadDirectory("data");
-			assertNotNull(bag.getBagFile("data/bag/manifest-md5.txt"));
 		} finally {
 			bag.close();
 		}