Merge pull request localstack#220 from localstack/feat/fat-jar

Refactor install script and put java libs into a single jar

Author: whummer

Makefile

@@ -22,7 +22,7 @@ publish:           ## Publish the library to the central PyPi repository
 	($(VENV_RUN) && ./setup.py sdist upload)
 
 publish-maven:     ## Publish artifacts to Maven Central
-	(cd localstack/ext/java/; mvn clean javadoc:jar source:jar deploy)
+	(cd localstack/ext/java/; mvn -Pfatjar clean javadoc:jar source:jar package deploy)
 
 coveralls:         ## Publish coveralls metrics
 	($(VENV_RUN); coveralls)

README.md

@@ -295,6 +295,7 @@ localstack web
 
 ## Change Log
 
+* v0.7.3: Extract proxy listeners into (sub-)classes; put java libs into a single "fat" jar; fix issue with non-daemonized threads; refactor code to start flask services
 * v0.7.2: Fix DATA_DIR config when running in Docker; fix Maven dependencies; return 'ConsumedCapacity' from DynamoDB get-item; use Queue ARN instead of URL for S3 bucket notifications
 * v0.7.1: Fix S3 API to GET bucket notifications; release Java artifacts to Maven Central; fix S3 file access from Spark; create DDB stream on UpdateTable; remove AUI dependency, optimize size of Docker image
 * v0.7.0: Support for Kinesis in CloudFormation; extend and integrate Java tests in CI; publish Docker image under new name; update READMEs and license agreements

localstack/constants.py

@@ -2,7 +2,7 @@
 import sys
 
 # LocalStack version
-VERSION = '0.7.2'
+VERSION = '0.7.3'
 
 # default AWS region
 if 'DEFAULT_REGION' not in os.environ:

localstack/ext/java/pom.xml

@@ -5,7 +5,7 @@
     <groupId>cloud.localstack</groupId>
     <artifactId>localstack-utils</artifactId>
     <packaging>jar</packaging>
-    <version>0.1.2</version>
+    <version>0.1.3</version>
     <name>localstack-utils</name>
 
     <description>Java utilities for the LocalStack platform.</description>
@@ -105,6 +105,19 @@
         </dependency>
     </dependencies>
 
+    <profiles>
+        <profile>
+            <id>fatjar</id>
+            <dependencies>
+                <dependency>
+                    <groupId>com.amazonaws</groupId>
+                    <artifactId>aws-java-sdk</artifactId>
+                    <version>1.11.86</version>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
+
     <build>
         <plugins>
             <plugin>
@@ -115,22 +128,10 @@
                     <additionalparam>-Xdoclint:none</additionalparam>
                 </configuration>
             </plugin>
-            <!-- <plugin>
+            <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
                 <version>3.0.0</version>
-                <configuration>
-                    <minimizeJar>true</minimizeJar>
-                    <outputFile>${project.artifactId}-${project.version}-shaded</outputFile>
-                    <finalName>${project.artifactId}-${project.version}-shaded</finalName>
-                    <createDependencyReducedPom>false</createDependencyReducedPom>
-                    <relocations>
-                        <relocation>
-                            <pattern>com.amazonaws</pattern>
-                            <shadedPattern>shaded.com.amazonaws</shadedPattern>
-                        </relocation>
-                    </relocations>
-                </configuration>
                 <executions>
                     <execution>
                         <phase>package</phase>
@@ -139,7 +140,28 @@
                         </goals>
                     </execution>
                 </executions>
-            </plugin> -->
+                <configuration>
+                    <minimizeJar>true</minimizeJar>
+                    <createDependencyReducedPom>false</createDependencyReducedPom>
+                    <shadedArtifactAttached>true</shadedArtifactAttached>
+                    <shadedClassifierName>fat</shadedClassifierName>
+                    <artifactSet>
+                        <includes>
+                            <include>com.amazonaws:aws-java-sdk-lambda</include>
+                            <include>com.amazonaws:aws-lambda-java-events</include>
+                            <include>com.amazonaws:aws-lambda-java-core</include>
+                            <include>commons-*:*</include>
+                            <include>net.*:*</include>
+                            <include>org.*:*</include>
+                            <include>junit:junit</include>
+                            <include>com.fasterxml.*:*</include>
+                            <include>joda-time:*</include>
+                            <include>com.jayway.*:*</include>
+                            <include>software.*:*</include>
+                        </includes>
+                    </artifactSet>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
@@ -170,7 +192,6 @@
                         </goals>
                     </execution>
                 </executions>
-
             </plugin>
             <plugin>
                 <groupId>org.sonatype.plugins</groupId>

localstack/ext/java/src/main/java/cloud/localstack/LambdaExecutor.java

@@ -28,7 +28,7 @@ public class LambdaExecutor {
 	public static void main(String[] args) throws Exception {
 		if(args.length < 2) {
 			System.err.println("Usage: java " + LambdaExecutor.class.getSimpleName() +
-					"<lambdaClass> <recordsFilePath>");
+					" <lambdaClass> <recordsFilePath>");
 			System.exit(1);
 		}
 

localstack/services/awslambda/lambda_api.py

@@ -22,7 +22,7 @@
 from localstack import config
 from localstack.constants import *
 from localstack.services import generic_proxy
-from localstack.services.install import M2_HOME, JAR_DEPENDENCIES, INSTALL_PATH_LOCALSTACK_JAR
+from localstack.services.install import INSTALL_PATH_LOCALSTACK_FAT_JAR
 from localstack.utils.common import *
 from localstack.utils.aws import aws_stack
 from localstack.utils.analytics import event_publisher
@@ -34,7 +34,7 @@
 ARCHIVE_FILE_PATTERN = '%s/lambda.handler.*.jar' % config.TMP_FOLDER
 EVENT_FILE_PATTERN = '%s/lambda.event.*.json' % config.TMP_FOLDER
 LAMBDA_SCRIPT_PATTERN = '%s/lambda_script_*.py' % config.TMP_FOLDER
-LAMBDA_EXECUTOR_JAR = INSTALL_PATH_LOCALSTACK_JAR
+LAMBDA_EXECUTOR_JAR = INSTALL_PATH_LOCALSTACK_FAT_JAR
 LAMBDA_EXECUTOR_CLASS = 'cloud.localstack.LambdaExecutor'
 
 LAMBDA_RUNTIME_PYTHON27 = 'python2.7'
@@ -430,9 +430,6 @@ def execute(event, context):
             TMP_FILES.append(event_file)
             class_name = lambda_arn_to_handler[arn].split('::')[0]
             classpath = '%s:%s' % (LAMBDA_EXECUTOR_JAR, main_file)
-            for jar in JAR_DEPENDENCIES:
-                jar_path = '%s/repository/%s' % (M2_HOME, jar)
-                classpath += ':%s' % (jar_path)
             cmd = 'java -cp %s %s %s %s' % (classpath, LAMBDA_EXECUTOR_CLASS, class_name, event_file)
             result, log_output = run_lambda_executor(cmd)
             LOG.info('Lambda output: %s' % log_output.replace('\n', '\n> '))

localstack/services/install.py

@@ -17,38 +17,17 @@
 INSTALL_DIR_ES = '%s/elasticsearch' % INSTALL_DIR_INFRA
 INSTALL_DIR_DDB = '%s/dynamodb' % INSTALL_DIR_INFRA
 INSTALL_DIR_KCL = '%s/amazon-kinesis-client' % INSTALL_DIR_INFRA
-INSTALL_PATH_LOCALSTACK_JAR = '%s/localstack-utils.jar' % INSTALL_DIR_INFRA
+INSTALL_PATH_LOCALSTACK_FAT_JAR = '%s/localstack-utils-fat.jar' % INSTALL_DIR_INFRA
 TMP_ARCHIVE_ES = os.path.join(tempfile.gettempdir(), 'localstack.es.zip')
 TMP_ARCHIVE_DDB = os.path.join(tempfile.gettempdir(), 'localstack.ddb.zip')
 TMP_ARCHIVE_STS = os.path.join(tempfile.gettempdir(), 'aws-java-sdk-sts.jar')
 URL_STS_JAR = 'http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar'
-URL_LOCALSTACK_JAR = ('http://central.maven.org/maven2/' +
-    'cloud/localstack/localstack-utils/0.1.2/localstack-utils-0.1.2.jar')
+URL_LOCALSTACK_FAT_JAR = ('http://central.maven.org/maven2/' +
+    'cloud/localstack/localstack-utils/0.1.3/localstack-utils-0.1.3-fat.jar')
 
 # list of additional pip packages to install
 EXTENDED_PIP_LIBS = ['amazon-kclpy==1.4.5']
 
-# local maven repository path
-M2_HOME = os.path.expanduser('~/.m2')
-
-# hack required for Docker because our base image uses $HOME/.m2 as a volume (see Dockerfile)
-# TODO still needed?
-if '/root/.m2_persistent' in os.environ.get('MAVEN_OPTS', ''):
-    M2_HOME = '/root/.m2_persistent'
-
-# TODO: temporary hack! Remove all hardcoded paths (and move to lamb-ci Docker for Java once it's available)
-JAR_DEPENDENCIES = [
-    'com/amazonaws/aws-lambda-java-core/1.1.0/aws-lambda-java-core-1.1.0.jar',
-    'com/amazonaws/aws-lambda-java-events/1.3.0/aws-lambda-java-events-1.3.0.jar',
-    'com/amazonaws/aws-java-sdk-kinesis/1.11.86/aws-java-sdk-kinesis-1.11.86.jar',
-    'com/fasterxml/jackson/core/jackson-databind/2.6.6/jackson-databind-2.6.6.jar',
-    'com/fasterxml/jackson/core/jackson-core/2.6.6/jackson-core-2.6.6.jar',
-    'com/fasterxml/jackson/core/jackson-annotations/2.6.0/jackson-annotations-2.6.0.jar',
-    'commons-codec/commons-codec/1.9/commons-codec-1.9.jar',
-    'commons-io/commons-io/2.5/commons-io-2.5.jar',
-    'org/apache/commons/commons-lang3/3.5/commons-lang3-3.5.jar'
-]
-
 # set up logger
 LOGGER = logging.getLogger(os.path.basename(__file__))
 
@@ -124,22 +103,12 @@ def install_amazon_kinesis_libs():
     class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH
     if not glob.glob(class_files):
         run('javac -cp "%s" %s' % (classpath, java_files))
-    # TODO needed?
-    ext_java_dir = '%s/ext/java' % ROOT_PATH
-    if not glob.glob('%s/target/*.jar' % ext_java_dir):
-        run('cd "%s"; mvn -DskipTests package' % (ext_java_dir))
 
 
 def install_lambda_java_libs():
-    for jar in JAR_DEPENDENCIES:
-        jar_path = '%s/repository/%s' % (M2_HOME, jar)
-        if not os.path.exists(jar_path):
-            jar_url = ('http://central.maven.org/maven2/%s' % jar)
-            mkdir(os.path.dirname(jar_path))
-            download(jar_url, jar_path)
-    # install LocalStack JAR file
-    if not os.path.exists(INSTALL_PATH_LOCALSTACK_JAR):
-        download(URL_LOCALSTACK_JAR, INSTALL_PATH_LOCALSTACK_JAR)
+    # install LocalStack "fat" JAR file (contains all dependencies)
+    if not os.path.exists(INSTALL_PATH_LOCALSTACK_FAT_JAR):
+        download(URL_LOCALSTACK_FAT_JAR, INSTALL_PATH_LOCALSTACK_FAT_JAR)
 
 
 def install_component(name):