Adds container lister tool, which is used to perform listing of conta…

…iners in SDSC's OpenStack swift environment.
duracloud · Aug 28, 2015 · ae38247 · ae38247
1 parent 9505821
commit ae38247
Show file tree

Hide file tree

Showing 4 changed files with 337 additions and 12 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,12 +1,9 @@
-*.class
-
-# Mobile Tools for Java (J2ME)
-.mtj.tmp/
-
-# Package Files #
-*.jar
-*.war
-*.ear
-
-# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
-hs_err_pid*
+.classpath
+.idea
+.project
+.settings
+*/target
+*.iml
+*.ipr
+*.iws
+.DS_STORE
diff --git a/container-lister-tool/pom.xml b/container-lister-tool/pom.xml
@@ -0,0 +1,109 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.duracloud</groupId>
+  <artifactId>containerlistertool</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0</version>
+  <name>Container Lister Tool</name>
+
+  <repositories>
+
+    <repository>
+      <id>central</id>
+      <name>Maven Repository Switchboard</name>
+      <url>http://repo1.maven.org/maven2</url>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
+    </repository>
+
+    <repository>
+      <id>duraspace-releases</id>
+      <name>DuraSpace Release Maven Repository</name>
+      <url>https://m2.duraspace.org/content/repositories/releases</url>
+    </repository>
+
+  </repositories>
+
+  <properties>
+    <jclouds.version>1.8.0</jclouds.version>
+  </properties>
+
+  <build>
+
+    <plugins>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+          <encoding>UTF-8</encoding>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>2.3</version>
+        <configuration>
+          <descriptor>src/main/assembly/dep.xml</descriptor>
+          <includeSite>false</includeSite>
+          <archive>
+            <manifest>
+              <mainClass>org.duraspace.tools.storage.ContainerListerTool</mainClass>
+            </manifest>
+          </archive>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+    </plugins>
+
+  </build>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <version>1.2</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.jclouds</groupId>
+      <artifactId>jclouds-core</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.jclouds.api</groupId>
+      <artifactId>swift</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.jclouds</groupId>
+      <artifactId>jclouds-blobstore</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.jclouds.driver</groupId>
+      <artifactId>jclouds-enterprise</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>    
+
+  </dependencies>
+</project>
diff --git a/container-lister-tool/src/main/assembly/dep.xml b/container-lister-tool/src/main/assembly/dep.xml
@@ -0,0 +1,15 @@
+<assembly>
+  <id>driver</id>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <formats>
+    <format>jar</format>
+  </formats>
+
+  <dependencySets>
+    <dependencySet>
+      <unpack>true</unpack>
+    </dependencySet>
+  </dependencySets>
+
+</assembly>
+
diff --git a/container-lister-tool/src/main/java/org/duraspace/tools/storage/ContainerListerTool.java b/container-lister-tool/src/main/java/org/duraspace/tools/storage/ContainerListerTool.java
@@ -0,0 +1,204 @@
+package org.duraspace.tools.storage;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.inject.Module;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.jclouds.Constants;
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
+import org.jclouds.openstack.swift.SwiftApiMetadata;
+import org.jclouds.openstack.swift.SwiftClient;
+import org.jclouds.openstack.swift.domain.ObjectInfo;
+import org.jclouds.openstack.swift.options.ListContainerOptions;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.util.Properties;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/*
+ * Container Lister Tool - Provides a way to get a listing of
+ *   all items in an OpenStack swift container using JClouds
+ *   to talk directly to SDSC
+ *
+ * @author: Bill Branan
+ * Date: Aug 28, 2015
+ */
+public class ContainerListerTool {
+
+    private static final String authUrl =
+        "https://duracloud.auth.cloud.sdsc.edu/auth/v1.0";
+
+    private String spaceName;
+    private String username;
+    private String password;
+    private int maxSize;
+    private String outputFilePath;
+
+    private static Options cmdOptions;
+
+    public ContainerListerTool(String spaceName,
+                               String username,
+                               String password,
+                               int maxSize,
+                               String outputFile) {
+        this.spaceName = spaceName;
+        this.username = username;
+        this.password = password;
+        this.maxSize = maxSize;
+        this.outputFilePath = outputFile;
+    }
+
+    public void run() throws Exception {
+        System.out.println("Running Space Lister Tool (JClouds) with config:" +
+                           "\nspace name=" + spaceName +
+                           "\nuser name=" + username +
+                           "\nmax size=" + maxSize);
+
+        System.out.println("Setting up tool...");
+
+        String trimmedAuthUrl = // JClouds expects authURL with no version
+            authUrl.substring(0, authUrl.lastIndexOf("/"));
+
+        ListeningExecutorService useExecutor = createThreadPool();
+        ListeningExecutorService ioExecutor = createThreadPool();
+
+        Iterable<Module> modules = ImmutableSet.<Module> of(
+            new EnterpriseConfigurationModule(useExecutor, ioExecutor));
+        Properties properties = new Properties();
+        properties.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER,
+                               "true");
+        SwiftClient swiftClient = ContextBuilder.newBuilder(new SwiftApiMetadata())
+                                    .endpoint(trimmedAuthUrl)
+                                    .credentials(username, password)
+                                    .modules(modules)
+                                    .overrides(properties)
+                                    .buildApi(SwiftClient.class);
+
+        doList(swiftClient, spaceName);
+
+        System.out.println("Space Lister Tool process complete.");
+    }
+
+    protected ListeningExecutorService createThreadPool() {
+        return MoreExecutors.listeningDecorator(
+            new ThreadPoolExecutor(0,
+                                   Integer.MAX_VALUE,
+                                   5L,
+                                   TimeUnit.SECONDS,
+                                   new SynchronousQueue<Runnable>()));
+    }
+
+    private void doList(SwiftClient swiftClient, String spaceId)
+        throws Exception {
+
+        File outputFile = new File(spaceId + "-content-listing-sdsc-jclouds.txt");
+
+        if(outputFilePath != null){
+            outputFile = new File(outputFilePath);
+            outputFile.getParentFile().mkdirs();
+        }
+
+        System.out.println("Writing space listing to: " + outputFile.getAbsolutePath());
+        Writer writer = new FileWriter(outputFile);
+
+        String marker = null;
+        PageSet<ObjectInfo> objects = listObjects(swiftClient, spaceName, maxSize, marker);
+        int itemsInList = objects.size();
+        System.out.println("Items in returned set: " + objects.size());
+        while(objects.size() > 0) {
+            for (ObjectInfo object : objects) {
+                marker = object.getName();
+                writer.write(marker + "\n");
+            }
+            objects = listObjects(swiftClient, spaceName, maxSize, marker);
+            itemsInList += objects.size();
+            System.out.println("Items in returned set: " + objects.size() +
+                               ". Total: " + itemsInList);
+        }
+
+        writer.flush();
+        writer.close();
+    }
+
+    private PageSet<ObjectInfo> listObjects(SwiftClient swiftClient,
+                                            String containerName,
+                                            int limit,
+                                            String marker) {
+        ListContainerOptions containerOptions =
+            ListContainerOptions.Builder.maxResults(limit);
+        if(marker != null) containerOptions.afterMarker(marker);
+        return swiftClient.listObjects(containerName, containerOptions);
+    }
+
+    public static void main(String[] args) throws Exception {
+        cmdOptions = new Options();
+
+        Option spaceNameOption =
+            new Option("s", "spacename", true, "the space name to list");
+        spaceNameOption.setRequired(true);
+        cmdOptions.addOption(spaceNameOption);
+
+        Option usernameOption =
+           new Option("u", "username", true,
+                      "the username necessary to perform writes to DuraStore");
+        usernameOption.setRequired(true);
+        cmdOptions.addOption(usernameOption);
+
+        Option passwordOption =
+           new Option("p", "password", true,
+                      "the password necessary to perform writes to DuraStore");
+        passwordOption.setRequired(true);
+        cmdOptions.addOption(passwordOption);
+
+        Option maxSizeOption =
+            new Option("m", "maxsize", true,
+                       "the max size of request results from open stack");
+        maxSizeOption.setRequired(true);
+        cmdOptions.addOption(maxSizeOption);
+
+        Option outputFileOption =
+                new Option("o", "output-file", true,
+                           "The file to output results to");
+             outputFileOption.setRequired(false);
+             cmdOptions.addOption(outputFileOption);
+
+        CommandLine cmd = null;
+        try {
+            CommandLineParser parser = new PosixParser();
+            cmd = parser.parse(cmdOptions, args);
+        } catch(ParseException e) {
+            System.out.println(e.getMessage());
+            usage();
+        }
+
+        String spacename = cmd.getOptionValue("s");
+        String username = cmd.getOptionValue("u");
+        String password = cmd.getOptionValue("p");
+        int maxSize = Integer.parseInt(cmd.getOptionValue("m"));
+        String outputFile = cmd.getOptionValue("o");
+
+        ContainerListerTool tool =
+            new ContainerListerTool(spacename, username, password, maxSize, outputFile);
+        tool.run();
+    }
+
+    private static void usage() {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp("Running the Space Lister Tool (JClouds)", cmdOptions);
+        System.exit(1);
+    }
+
+}