moved classes into different packages. upgraded SDK. tested table create request factory

Author: Alexander Patrikalakis

README.md

@@ -17,21 +17,31 @@ The CLI's usage follows with required parameters marked by asterisks.
       Use this flag to use strongly consistent scan. If the flag is not used 
       it will default to eventually consistent scan
       Default: false
-    --createDestination
-      Create destination table if it does not exist
-      Default: false
     --copyStreamSpecificationWhenCreating
       Use the source table stream specification for the destination table 
       during its creation.
       Default: false
+    --createAllGsi
+      Create all GSI in destination table
+      Default: false
+    --createAllLsi
+      Create all LSI in destination table
+      Default: false
+    --createDestination
+      Create destination table if it does not exist
+      Default: false
     --destinationEndpoint
       Endpoint of the destination table
-  * --destinationRegion
+  * --destinationSigningRegion
       Signing region for the destination endpoint
   * --destinationTable
       Name of the destination table
     --help
       Display usage information
+    --includeGsi
+      Include the following GSI in the destination table
+    --includeLsi
+      Include the following LSI in the destination table
     --maxWriteThreads
       Number of max threads to write to destination table
       Default: 1024
@@ -44,7 +54,7 @@ The CLI's usage follows with required parameters marked by asterisks.
       Default: 0
     --sourceEndpoint
       Endpoint of the source table
-  * --sourceRegion
+  * --sourceSigningRegion
       Signing region for the source endpoint
   * --sourceTable
       Name of the source table
@@ -87,8 +97,8 @@ To transfer to a different region, create two AmazonDynamoDBClients
 with different endpoints to pass into the DynamoDBBootstrapWorker and the DynamoDBConsumer.
 
 ```java
-import com.amazonaws.dynamodb.bootstrap.DynamoDBBootstrapWorker;
-import com.amazonaws.dynamodb.bootstrap.DynamoDBConsumer;
+import DynamoDBBootstrapWorker;
+import com.amazonaws.dynamodb.bootstrap.consumer.DynamoDBConsumer;
 import com.amazonaws.dynamodb.bootstrap.exception.NullReadCapacityException;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
@@ -98,28 +108,22 @@ import java.util.concurrent.Executors;
 
 class TransferDataFromOneTableToAnother {
     public static void main(String[] args) {
-        AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
+        final AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
                 .withRegion(com.amazonaws.regions.Regions.US_WEST_1).build();
-        DynamoDBBootstrapWorker worker = null;
         try {
             // 100.0 read operations per second. 4 threads to scan the table.
-            worker = new DynamoDBBootstrapWorker(client,
+            final DynamoDBBootstrapWorker worker = new DynamoDBBootstrapWorker(client,
                     100.0, "mySourceTable", 4);
+            // 50.0 write operations per second. 8 threads to scan the table.
+            final DynamoDBConsumer consumer = new DynamoDBConsumer(client, "myDestinationTable",
+                    50.0, Executors.newFixedThreadPool(8));
+            worker.pipe(consumer);
         } catch (NullReadCapacityException e) {
             System.err.println("The DynamoDB source table returned a null read capacity.");
             System.exit(1);
-        }
-        // 50.0 write operations per second. 8 threads to scan the table.
-        DynamoDBConsumer consumer = new DynamoDBConsumer(client, "myDestinationTable", 50.0,
-                Executors.newFixedThreadPool(8));
-        try {
-            worker.pipe(consumer);
-        } catch (ExecutionException e) {
+        } catch (ExecutionException | InterruptedException e) {
             System.err.println("Encountered exception when executing transfer: " + e.getMessage());
             System.exit(1);
-        } catch (InterruptedException e){
-            System.err.println("Interrupted when executing transfer: " + e.getMessage());
-            System.exit(1);
         }
     }
 }
@@ -128,13 +132,15 @@ class TransferDataFromOneTableToAnother {
 
 ### 2. Transfer Data From one DynamoDB Table to a Blocking Queue.
 
-The below example will read from a DynamoDB table and export to an array blocking queue. This is useful for when another application would like to consume
-the DynamoDB entries but does not have a setup application for it. They can just retrieve the queue (consumer.getQueue()) and then continually pop() from it
+The below example will read from a DynamoDB table and export to an array blocking queue.
+This is useful for when another application would like to consume
+the DynamoDB entries but does not have a setup application for it.
+They can just retrieve the queue (consumer.getQueue()) and then continually pop() from it
 to then process the new entries.
 
 ```java
-import com.amazonaws.dynamodb.bootstrap.BlockingQueueConsumer;
-import com.amazonaws.dynamodb.bootstrap.DynamoDBBootstrapWorker;
+import com.amazonaws.dynamodb.bootstrap.consumer.BlockingQueueConsumer;
+import DynamoDBBootstrapWorker;
 import com.amazonaws.dynamodb.bootstrap.exception.NullReadCapacityException;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
@@ -143,29 +149,20 @@ import java.util.concurrent.ExecutionException;
 
 class TransferDataFromOneTableToBlockingQueue {
     public static void main(String[] args) {
-        AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
+        final AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
                 .withRegion(com.amazonaws.regions.Regions.US_WEST_1).build();
-
-        DynamoDBBootstrapWorker worker = null;
-
         try {
             // 100.0 read operations per second. 4 threads to scan the table.
-            worker = new DynamoDBBootstrapWorker(client, 100.0, "mySourceTable", 4);
+            final DynamoDBBootstrapWorker worker = new DynamoDBBootstrapWorker(client, 100.0,
+                    "mySourceTable", 4);
+            final BlockingQueueConsumer consumer = new BlockingQueueConsumer(8);
+            worker.pipe(consumer);
         } catch (NullReadCapacityException e) {
             System.err.println("The DynamoDB source table returned a null read capacity.");
             System.exit(1);
-        }
-
-        BlockingQueueConsumer consumer = new BlockingQueueConsumer(8);
-
-        try {
-            worker.pipe(consumer);
-        } catch (ExecutionException e) {
+        } catch (ExecutionException | InterruptedException e) {
             System.err.println("Encountered exception when executing transfer: " + e.getMessage());
             System.exit(1);
-        } catch (InterruptedException e){
-            System.err.println("Interrupted when executing transfer: " + e.getMessage());
-            System.exit(1);
         }
     }
 }

pom.xml

@@ -11,7 +11,7 @@
         <url>https://github.com/awslabs/dynamodb-import-export-tool.git</url>
     </scm>
     <properties>
-        <jdk.version>1.7</jdk.version>
+        <jdk.version>1.8</jdk.version>
         <aws.java.sdk.version>1.11.123</aws.java.sdk.version>
         <powermock.version>1.6.6</powermock.version>
         <jcommander.version>1.69</jcommander.version>

src/main/java/com/amazonaws/dynamodb/bootstrap/AbstractLogProvider.java

@@ -22,6 +22,7 @@
 import org.apache.log4j.Logger;
 
 import com.amazonaws.dynamodb.bootstrap.constants.BootstrapConstants;
+import com.amazonaws.dynamodb.bootstrap.consumer.AbstractLogConsumer;
 
 /**
  * Abstract class to send inputs from a source to a consumer.
@@ -31,25 +32,22 @@ public abstract class AbstractLogProvider {
     /**
      * Logger for the DynamoDBBootstrapWorker.
      */
-    private static final Logger LOGGER = LogManager
-            .getLogger(AbstractLogProvider.class);
+    private static final Logger LOGGER = LogManager.getLogger(AbstractLogProvider.class);
 
     protected ExecutorService threadPool;
 
     /**
      * Begins to read log results and transfer them to the consumer who will
      * write the results.
      */
-    public abstract void pipe(final AbstractLogConsumer consumer)
-            throws ExecutionException, InterruptedException;
+    public abstract void pipe(final AbstractLogConsumer consumer) throws ExecutionException, InterruptedException;
 
     /**
      * Shuts the thread pool down.
-     * 
-     * @param <awaitTermination>
-     *            If true, this method waits for the threads in the pool to
-     *            finish. If false, this thread pool shuts down without
-     *            finishing their current tasks.
+     *
+     * @param <awaitTermination> If true, this method waits for the threads in the pool to
+     *                           finish. If false, this thread pool shuts down without
+     *                           finishing their current tasks.
      */
     public void shutdown(boolean awaitTermination) {
         if (awaitTermination) {
@@ -61,8 +59,7 @@ public void shutdown(boolean awaitTermination) {
                 }
             } catch (InterruptedException e) {
                 interrupted = true;
-                LOGGER.warn("Threadpool was interrupted when trying to shutdown: "
-                        + e.getMessage());
+                LOGGER.warn("Threadpool was interrupted when trying to shutdown: " + e.getMessage());
             } finally {
                 if (interrupted)
                     Thread.currentThread().interrupt();

src/main/java/com/amazonaws/dynamodb/bootstrap/AttributeValueMixIn.java

@@ -14,8 +14,11 @@
  */
 package com.amazonaws.dynamodb.bootstrap;
 
+import java.util.List;
+
 import com.amazonaws.dynamodb.bootstrap.constants.BootstrapConstants;
 import com.beust.jcommander.Parameter;
+
 import lombok.Getter;
 
 /**
@@ -60,6 +63,22 @@ public boolean getHelp() {
     @Parameter(names = CREATE_DESTINATION_TABLE_IF_MISSING, description = "Create destination table if it does not exist")
     private boolean createDestinationTableIfMissing;
 
+    public static final String CREATE_ALL_LSI = "--createAllLsi";
+    @Parameter(names = CREATE_ALL_LSI, description = "Create all LSI in destination table")
+    private boolean createAllLsi;
+
+    public static final String CREATE_ALL_GSI = "--createAllGsi";
+    @Parameter(names = CREATE_ALL_GSI, description = "Create all GSI in destination table")
+    private boolean createAllGsi;
+
+    public static final String INCLUDE_LSI = "--includeLsi";
+    @Parameter(names = INCLUDE_LSI, description = "Include the following LSI in the destination table")
+    private List<String> includeLsi;
+
+    public static final String INCLUDE_GSI = "--includeGsi";
+    @Parameter(names = INCLUDE_GSI, description = "Include the following GSI in the destination table")
+    private List<String> includeGsi;
+
     public static final String COPY_STREAM_SPECIFICATION_WHEN_CREATING = "--copyStreamSpecificationWhenCreating";
     @Parameter(names = COPY_STREAM_SPECIFICATION_WHEN_CREATING, description = "Use the source table stream specification for the destination table during its creation.")
     private boolean copyStreamSpecification;
@@ -83,7 +102,7 @@ public boolean getHelp() {
     public static final String SECTION = "--section";
     @Parameter(names = SECTION, description = "Section number to scan when running multiple programs concurrently [0, 1... totalSections-1]", required = false)
     private int section = 0;
-    
+
     public static final String CONSISTENT_SCAN = "--consistentScan";
     @Parameter(names = CONSISTENT_SCAN, description = "Use this flag to use strongly consistent scan. If the flag is not used it will default to eventually consistent scan")
     private boolean consistentScan = false;