refactor configuration; make base installation more light-weight; lazy load libraries (localstack#177)

Author: whummer

LICENSE.txt

@@ -1,4 +1,5 @@
-Copyright @ 2016 - 2016 Atlassian Pty Ltd
+Copyright (c) 2017 LocalStack contributors
+Copyright (c) 2016 Atlassian Pty Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

Makefile

@@ -3,41 +3,20 @@ IMAGE_NAME_BASE ?= localstack/java-maven-node-python
 IMAGE_TAG ?= $(shell cat localstack/constants.py | grep '^VERSION =' | sed "s/VERSION = ['\"]\(.*\)['\"].*/\1/")
 VENV_DIR ?= .venv
 VENV_RUN = . $(VENV_DIR)/bin/activate
-AWS_STS_URL = http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar
-AWS_STS_TMPFILE = $(TMPDIR)aws-java-sdk-sts.jar
-LOCALSTACK_JAR_URL = https://bitbucket.org/atlassian/localstack/raw/mvn/release/com/atlassian/localstack-utils/1.0-SNAPSHOT/localstack-utils-1.0-SNAPSHOT.jar
-LOCALSTACK_JAR_PATH = localstack/infra/localstack-utils.jar
 PIP_CMD ?= pip
 
 usage:             ## Show this help
 	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//'
 
-install:           ## Install npm/pip dependencies, compile code
-	make setup-venv && \
-		make install-libs && \
-		make compile
-
-setup-venv:        # Setup virtualenv
+install:           ## Install dependencies in virtualenv
 	(test `which virtualenv` || $(PIP_CMD) install --user virtualenv) && \
 		(test -e $(VENV_DIR) || virtualenv $(VENV_OPTS) $(VENV_DIR)) && \
 		($(VENV_RUN) && $(PIP_CMD) install --upgrade pip) && \
 		(test ! -e requirements.txt || ($(VENV_RUN); $(PIP_CMD) install six==1.10.0 ; $(PIP_CMD) install -r requirements.txt))
 
-install-libs:      ## Install npm/pip dependencies
-	(test -e localstack/infra/amazon-kinesis-client/aws-java-sdk-sts.jar || \
-			{ (test -e $(AWS_STS_TMPFILE) || curl -o $(AWS_STS_TMPFILE) $(AWS_STS_URL)); \
-				mkdir -p localstack/infra/amazon-kinesis-client; \
-				cp $(AWS_STS_TMPFILE) localstack/infra/amazon-kinesis-client/aws-java-sdk-sts.jar; }) && \
-		(test -e $(LOCALSTACK_JAR_PATH) || curl -o $(LOCALSTACK_JAR_PATH) $(LOCALSTACK_JAR_URL))
-
 install-web:       ## Install npm dependencies for dashboard Web UI
 	(cd localstack/dashboard/web && (test ! -e package.json || npm install --silent > /dev/null))
 
-compile:           ## Compile Java code (KCL library utils, Java Lambda executor)
-	echo "Compiling"
-	javac -cp $(shell $(VENV_RUN); python -c 'from localstack.utils.kinesis import kclipy_helper; print(kclipy_helper.get_kcl_classpath())') localstack/utils/kinesis/java/com/atlassian/*.java
-	(test ! -e localstack/ext/java || (cd localstack/ext/java && mvn -q -DskipTests package))
-
 publish:           ## Publish the library to the central PyPi repository
 	# build and upload archive
 	($(VENV_RUN) && ./setup.py sdist upload)
@@ -111,7 +90,5 @@ clean:             ## Clean up (npm dependencies, downloaded infrastructure code
 	rm -rf localstack/node_modules/
 	rm -rf $(VENV_DIR)
 	rm -f localstack/utils/kinesis/java/com/atlassian/*.class
-	rm -f $(AWS_STS_TMPFILE)
-	rm -f $(TMPDIR)localstack.es.zip
 
-.PHONY: usage compile clean install web install-web infra test install-libs
+.PHONY: usage compile clean install web install-web infra test

README.md

@@ -291,6 +291,7 @@ localstack web
 
 ## Change Log
 
+* v0.6.2: Major refactoring of installation process, lazy loading of dependencies
 * v0.6.1: Add CORS headers; platform compatibility fixes (remove shell commands and sh module); add CloudFormation validate-template; fix Lambda execution in Docker; basic domain handling in ES API; API Gateway authorizers
 * v0.6.0: Load services as plugins; fix service default ports; fix SQS->SNS and MD5 of message attributes; fix Host header for S3
 * v0.5.5: Enable SSL encryption for all service endpoints (`USE_SSL` config); create Docker base image; fix issue with DATA_DIR
@@ -353,9 +354,12 @@ individual developer who contributes code to this repository. Please refer to th
 
 ## License
 
+Copyright (c) 2017 *LocalStack* contributors.
+
 Copyright (c) 2016 Atlassian and others.
 
-*LocalStack* is released under the Apache License, Version 2.0 (see LICENSE.txt).
+This version of *LocalStack* is released under the Apache License,
+Version 2.0 (see LICENSE.txt).
 
 We build on a number of third-party software tools, with the following licenses:
 
@@ -377,8 +381,6 @@ pep8						| Expat license
 requests					| Apache License 2.0
 subprocess32				| PSF License
 **Node.js/npm modules:**	|
-dynalite					| MIT License
 kinesalite					| MIT License
 **Other tools:**			|
 Elasticsearch 				| Apache License 2.0
-

localstack/constants.py

@@ -2,7 +2,7 @@
 import sys
 
 # LocalStack version
-VERSION = '0.6.1'
+VERSION = '0.6.2'
 
 # default AWS region
 if 'DEFAULT_REGION' not in os.environ:

localstack/services/firehose/firehose_api.py

@@ -16,7 +16,6 @@
 from localstack.constants import *
 from localstack.services.generic_proxy import GenericProxy
 from localstack.utils.common import short_uid, to_str
-from localstack.utils.testutil import get_s3_client
 from localstack.utils.aws.aws_stack import *
 from six import iteritems
 

localstack/services/infra.py

@@ -12,7 +12,7 @@
 import six
 import warnings
 import pkgutil
-from localstack import constants
+from localstack import constants, config
 from localstack.config import *
 from localstack.utils.aws import aws_stack
 from localstack.utils import common, persistence
@@ -92,16 +92,14 @@ def load_plugins():
 
     loaded_files = []
     for module in pkgutil.iter_modules():
+        file_path = None
         if six.PY3 and not isinstance(module, tuple):
             file_path = '%s/%s/plugins.py' % (module.module_finder.path, module.name)
-            if file_path not in loaded_files:
-                load_plugin_from_path(file_path)
-                loaded_files.append(file_path)
         elif six.PY3 or isinstance(module[0], pkgutil.ImpImporter):
             file_path = '%s/%s/plugins.py' % (module[0].path, module[1])
-            if file_path not in loaded_files:
-                load_plugin_from_path(file_path)
-                loaded_files.append(file_path)
+        if file_path and file_path not in loaded_files:
+            load_plugin_from_path(file_path)
+            loaded_files.append(file_path)
 
 
 # -----------------
@@ -315,7 +313,7 @@ def start_infra(async=False, apis=None):
         LOGGER.setLevel(logging.INFO)
 
         if not apis:
-            apis = list(SERVICE_PORTS.keys())
+            apis = list(config.SERVICE_PORTS.keys())
         # set environment
         os.environ['AWS_REGION'] = DEFAULT_REGION
         os.environ['ENV'] = ENV_DEV

localstack/services/install.py

@@ -2,11 +2,12 @@
 
 import os
 import sys
+import glob
+import shutil
 import logging
 from localstack.constants import DEFAULT_SERVICE_PORTS, ELASTICSEARCH_JAR_URL, DYNAMODB_JAR_URL
 from localstack.config import *
-from localstack.utils.common import download, parallelize, run
-
+from localstack.utils.common import download, parallelize, run, mkdir
 
 THIS_PATH = os.path.dirname(os.path.realpath(__file__))
 ROOT_PATH = os.path.realpath(os.path.join(THIS_PATH, '..'))
@@ -15,9 +16,17 @@
 INSTALL_DIR_NPM = '%s/node_modules' % ROOT_PATH
 INSTALL_DIR_ES = '%s/elasticsearch' % INSTALL_DIR_INFRA
 INSTALL_DIR_DDB = '%s/dynamodb' % INSTALL_DIR_INFRA
+INSTALL_DIR_KCL = '%s/amazon-kinesis-client' % INSTALL_DIR_INFRA
+INSTALL_PATH_LOCALSTACK_JAR = '%s/localstack-utils.jar' % INSTALL_DIR_INFRA
 TMP_ARCHIVE_ES = os.path.join(tempfile.gettempdir(), 'localstack.es.zip')
 TMP_ARCHIVE_DDB = os.path.join(tempfile.gettempdir(), 'localstack.ddb.zip')
+TMP_ARCHIVE_STS = os.path.join(tempfile.gettempdir(), 'aws-java-sdk-sts.jar')
+URL_STS_JAR = 'http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar'
+URL_LOCALSTACK_JAR = ('https://bitbucket.org/atlassian/localstack/raw/mvn/release/' +
+    'com/atlassian/localstack-utils/1.0-SNAPSHOT/localstack-utils-1.0-SNAPSHOT.jar')
 
+# list of additional pip packages to install
+EXTENDED_PIP_LIBS = ['amazon_kclpy==1.4.4']
 
 # set up logger
 LOGGER = logging.getLogger(os.path.basename(__file__))
@@ -34,10 +43,6 @@ def install_elasticsearch():
         for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'):
             cmd = 'cd %s && mkdir -p %s && chmod -R 777 %s'
             run(cmd % (INSTALL_DIR_ES, dir_name, dir_name))
-        # install plugins
-        # TODO remove
-        # cmd = 'echo y | %s/bin/elasticsearch-plugin install x-pack'
-        # run(cmd % INSTALL_DIR_ES)
 
 
 def install_kinesalite():
@@ -58,7 +63,7 @@ def is_alpine():
 def install_dynamodb_local():
     if not os.path.exists(INSTALL_DIR_DDB):
         LOGGER.info('Downloading and installing local DynamoDB server. This may take some time.')
-        run('mkdir -p %s' % INSTALL_DIR_DDB)
+        mkdir(INSTALL_DIR_DDB)
         if not os.path.exists(TMP_ARCHIVE_DDB):
             download(DYNAMODB_JAR_URL, TMP_ARCHIVE_DDB)
         cmd = 'cd %s && cp %s ddb.zip && unzip -q ddb.zip && rm ddb.zip'
@@ -74,18 +79,46 @@ def install_dynamodb_local():
             run("curl -L -o %s/DynamoDBLocal_lib/sqlite4java.jar '%s'" % (INSTALL_DIR_DDB, patched_jar))
 
 
+def install_amazon_kinesis_libs():
+    # install KCL/STS JAR files
+    if not os.path.exists(INSTALL_DIR_KCL):
+        mkdir(INSTALL_DIR_KCL)
+        if not os.path.exists(TMP_ARCHIVE_STS):
+            download(URL_STS_JAR, TMP_ARCHIVE_STS)
+        shutil.copy(TMP_ARCHIVE_STS, INSTALL_DIR_KCL)
+    # install LocalStack JAR file
+    if not os.path.exists(INSTALL_PATH_LOCALSTACK_JAR):
+        download(URL_LOCALSTACK_JAR, INSTALL_PATH_LOCALSTACK_JAR)
+    # install extended libs
+    try:
+        from amazon_kclpy import kcl
+    except Exception as e:
+        for lib in EXTENDED_PIP_LIBS:
+            run('pip install %s' % lib)
+    # Compile Java files
+    from localstack.utils.kinesis import kclipy_helper
+    classpath = kclipy_helper.get_kcl_classpath()
+    java_files = '%s/utils/kinesis/java/com/atlassian/*.java' % ROOT_PATH
+    class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH
+    if not glob.glob(class_files):
+        run('javac -cp "%s" %s' % (classpath, java_files))
+    ext_java_dir = '%s/ext/java' % ROOT_PATH
+    if not glob.glob('%s/target/*.jar' % ext_java_dir):
+        run('cd "%s"; mvn -DskipTests package' % (ext_java_dir))
+
+
 def install_component(name):
     if name == 'kinesis':
         install_kinesalite()
     elif name == 'dynamodb':
-        # install_dynalite()
         install_dynamodb_local()
     elif name == 'es':
         install_elasticsearch()
 
 
 def install_components(names):
     parallelize(install_component, names)
+    install_amazon_kinesis_libs()
 
 
 def install_all_components():

localstack/utils/aws/aws_stack.py

@@ -198,6 +198,14 @@ def render_velocity_template(template, context, as_json=False):
     return replaced
 
 
+def get_s3_client():
+    return boto3.resource('s3',
+        endpoint_url=config.TEST_S3_URL,
+        config=boto3.session.Config(
+            s3={'addressing_style': 'path'}),
+        verify=False)
+
+
 def get_account_id(account_id=None, env=None):
     if account_id:
         return account_id

localstack/utils/common.py

@@ -47,7 +47,7 @@
 LOGGER = logging.getLogger(__name__)
 
 
-# Helper class to convert JSON documents with datetime or decimals.
+# Helper class to convert JSON documents with datetime, decimals, or bytes.
 class CustomEncoder(json.JSONEncoder):
     def default(self, o):
         if isinstance(o, decimal.Decimal):
@@ -57,6 +57,8 @@ def default(self, o):
                 return int(o)
         if isinstance(o, datetime):
             return str(o)
+        if isinstance(o, six.binary_type):
+            return to_str(o)
         return super(CustomEncoder, self).default(o)
 
 

localstack/utils/testutil.py

@@ -183,14 +183,6 @@ def find_recursive(key, value, obj):
         return False
 
 
-def get_s3_client():
-    return boto3.resource('s3',
-        endpoint_url=TEST_S3_URL,
-        config=boto3.session.Config(
-            s3={'addressing_style': 'path'}),
-        verify=False)
-
-
 def list_all_s3_objects():
     return map_all_s3_objects().values()
 
@@ -203,7 +195,7 @@ def download_s3_object(s3, bucket, path):
 
 
 def map_all_s3_objects(to_json=True):
-    s3_client = get_s3_client()
+    s3_client = aws_stack.get_s3_client()
     result = {}
     for bucket in s3_client.buckets.all():
         for key in bucket.objects.all():

requirements.txt

@@ -1,5 +1,4 @@
 airspeed==0.5.5.dev20160812
-amazon_kclpy==1.4.4
 awscli==1.11.86
 boto==2.46.1
 boto3==1.4.4
@@ -10,7 +9,7 @@ flask==0.10.1
 flask-cors==3.0.3
 flask_swagger==0.2.12
 jsonpath-rw==1.4.0
-localstack-ext==0.6.1.4
+localstack-ext
 moto-ext==1.0.1.4
 nose==1.3.7
 pep8==1.7.0

setup.py

@@ -5,11 +5,9 @@
 import os
 import sys
 import re
-import subprocess
-import setuptools
 from setuptools import find_packages, setup
-from setuptools.command.install_lib import install_lib
 
+# parameter variables
 install_requires = []
 dependency_links = []
 package_data = {}
@@ -21,9 +19,9 @@
 version = re.search(r'^\s*VERSION\s*=\s*[\'"](.+)[\'"]\s*$', constants, re.MULTILINE).group(1)
 
 
+# determine requirements
 with open('requirements.txt') as f:
     requirements = f.read()
-
 for line in re.split('\n', requirements):
     if line and line[0] == '#' and '#egg=' in line:
         line = re.search(r'#\s*(.*)', line).group(1)
@@ -32,37 +30,6 @@
             install_requires.append(line)
 
 
-def do_make_install(workdir=None):
-    if workdir:
-        prev_workdir = os.getcwd()
-        os.chdir(workdir)
-    try:
-        subprocess.check_output('make install', shell=True)
-    except subprocess.CalledProcessError as e:
-        print(e.output)
-        raise e
-    if workdir:
-        os.chdir(prev_workdir)
-
-
-class InstallLibCommand(install_lib):
-
-    def run(self):
-        install_lib.run(self)
-        # prepare filesystem
-        main_dir_name = 'localstack'
-        target_dir = '%s/%s' % (self.install_dir, main_dir_name)
-        infra_dir = '%s/infra' % (main_dir_name)
-        # delete existing directory
-        subprocess.check_output('rm -r %s' % (main_dir_name), shell=True)
-        # create symlink
-        subprocess.check_output('ln -s %s %s' % (target_dir, main_dir_name), shell=True)
-        # delete infra directory (if it exists) to force re-install
-        subprocess.check_output('rm -rf %s' % (infra_dir), shell=True)
-        # run 'make install'
-        do_make_install()
-
-
 package_data = {
     '': ['Makefile', '*.md'],
     'localstack': [
@@ -94,9 +61,6 @@ def run(self):
         dependency_links=dependency_links,
         test_suite='tests',
         license='Apache License 2.0',
-        cmdclass={
-            'install_lib': InstallLibCommand
-        },
         zip_safe=False,
         classifiers=[
             'Programming Language :: Python :: 2',