use json for settings, remove stats, add tracing

Author: mugurrus

.gitignore

@@ -25,4 +25,4 @@ __pycache__
 /*.tar
 /*.7z
 /*.tgz
-
+/settings/collections.json

docker-compose.yml

@@ -1,8 +1,5 @@
 version: "3.3"
 
-volumes:
-  snoop-stats-es-data: {}
-
 services:
   snoop-rabbitmq:
     image: rabbitmq:3.7.3

src/common.py

@@ -9,7 +9,8 @@
     collection_allowed_chars, validate_collection_name, validate_collection_data_dir, \
     create_settings_dir, write_collection_docker_file, volumes_dir_name, \
     write_python_settings_file, default_snoop_image, write_env_file, \
-    InvalidCollectionName, exit_msg, DOCKER_HOOVER_SNOOP_STATS
+    InvalidCollectionName, exit_msg, init_collection_settings,\
+    write_collections_settings
 
 steps_file_name = 'collection-%s-steps.txt'
 steps_script_name = 'init-%s.sh'
@@ -51,17 +52,12 @@ def get_args():
                         help='Add development settings to the docker file')
     parser.add_argument('-p', '--profiling', action='store_const', const=True, default=False,
                         help='Add profiling settings for the new collection.')
+    parser.add_argument('-t', '--tracing', action='store_const', const=True, default=False,
+                        help='Add tracing settings for the new collection.')
     parser.add_argument('-m', '--manual-indexing', action='store_const', const=True, default=False,
                         help='Do not add the option to start indexing automatically.')
-    parser.add_argument('--stats', action='store_const', const=True, default=False,
-                        help='Enable kibana stats.')
     args = parser.parse_args()
 
-    try:
-        validate_collection_name(args.collection)
-    except InvalidCollectionName as e:
-        exit_msg(str(e))
-
     return args
 
 
@@ -72,26 +68,37 @@ def create_pg_dir(collection):
 
 
 def create_collection(args):
-    data = get_collections_data(args.collection)
+    data = get_collections_data()
+    try:
+        validate_collection_name(args.collection, data['collections'])
+    except InvalidCollectionName as e:
+        exit_msg(str(e))
     if len(data['collections']):
         validate_collections(data['collections'])
     validate_collection_data_dir(args.collection)
 
     try:
+        init_collection_settings(data['collections'], args.collection, {
+            'autoindex': not args.manual_indexing,
+            'image': args.snoop_image,
+            'profiling': args.profiling,
+            'tracing': args.tracing,
+            'for_dev': args.dev,
+            'snoop_port': data['snoop_port'],
+            'flower_port': data['flower_port'] if not args.manual_indexing else None,
+            'pg_port': data['pg_port'] if args.dev else None
+        })
         create_pg_dir(args.collection)
         settings_dir = create_settings_dir(args.collection)
 
-        data['collections'][args.collection] = {'profiling': args.profiling, 'for_dev': args.dev}
         ordered_collections = OrderedDict(sorted(data['collections'].items(), key=lambda t: t[0]))
 
-        stats = args.stats or data['collections'].get('env', {}).get(DOCKER_HOOVER_SNOOP_STATS, False)
-
-        write_collection_docker_file(args.collection, args.snoop_image, settings_dir, data['snoop_port'],
-                                     args.profiling, args.dev, data['pg_port'], not args.manual_indexing,
-                                     stats, data['flower_port'])
-        write_env_file(settings_dir, {'DOCKER_HOOVER_SNOOP_STATS': args.stats})
-        write_python_settings_file(args.collection, settings_dir, args.profiling, args.dev)
-        write_global_docker_file(ordered_collections, args.dev or bool(data['dev_instances']), stats)
+        write_collection_docker_file(args.collection, settings_dir,
+                                     data['collections'][args.collection])
+        write_env_file(settings_dir)
+        write_python_settings_file(args.collection, settings_dir, data['collections'][args.collection])
+        write_global_docker_file(ordered_collections, args.dev or bool(data['dev_instances']))
+        write_collections_settings(data)
     except Exception as e:
         print('Error creating collection: %s' % e)
         cleanup(args.collection)

src/createcollection.py

@@ -2,7 +2,7 @@
 from copy import deepcopy
 import json
 
-from src.common import get_collections_data, DOCKER_HOOVER_SNOOP_STATS
+from src.common import get_collections_data
 
 
 def get_args():
@@ -16,23 +16,22 @@ def print_collections(collections):
     index = 1
     for collection, settings in collections.items():
         print('%d. %s' % (index, collection))
-        print('  - profiling: %s' % settings['profiling'])
-        print('  - development: %s' % settings['for_dev'])
-        print('  - auto-indexing: %s' % settings['autoindex'])
+        print('  - auto-indexing: %s' % settings.get('autoindex', False))
+        if settings.get('autoindex') and 'flower_url' in settings:
+            print('  - flower URL: %s' % settings['flower_url'])
         print('  - image: %s' % settings['image'])
-        print('  - stats: %s' % settings['stats'])
         print('  - snoop admin URL: %s' % settings['snoop_url'])
-        if settings['autoindex'] and 'flower_url' in settings:
-            print('  - flower URL: %s' % settings['flower_url'])
+        print('  - profiling: %s' % settings.get('profiling', False))
+        print('  - tracing: %s' % settings.get('tracing', False))
+        print('  - development: %s' % settings.get('for_dev', False))
         index += 1
 
 
 def prepare_data(collections):
     data = deepcopy(collections)
     for settings in data.values():
-        settings['stats'] = 'enabled' if settings['env'].get(DOCKER_HOOVER_SNOOP_STATS, False) \
-            else 'disabled'
-        del settings['env']
+        if 'env' in settings:
+            del settings['env']
         settings['snoop_url'] = 'http://localhost:%d' % settings['snoop_port']
         if 'flower_port' in settings:
             settings['flower_url'] = 'http://localhost:%d' % settings['flower_port']

src/listcollections.py

@@ -6,27 +6,22 @@
 
 from src.common import get_collections_data, validate_collections, cleanup, \
     write_global_docker_file, collection_allowed_chars, validate_collection_name, \
-    volumes_dir_name, blobs_dir_name, InvalidCollectionName, exit_msg, \
-    DOCKER_HOOVER_SNOOP_STATS
+    volumes_dir_name, blobs_dir_name, InvalidCollectionName, exit_msg,\
+    write_collections_settings
 
 
 def get_args():
     parser = argparse.ArgumentParser(description='Remove collection.')
     parser.add_argument('-c', '--collection', required=True,
                         help='Collection name; allowed characters: ' + collection_allowed_chars)
-    parser.add_argument('--skip-index', action='store_const', const=True,
+    parser.add_argument('-i', '--skip-index', action='store_const', const=True,
                         help='Skip the index removal from search.')
     parser.add_argument('-b', '--remove-blobs', action='store_const', const=True, default=False,
                         help='Remove blobs')
     parser.add_argument('-y', '--yes', action='store_const', const=True, default=False,
                         help='Force yes answer to all interactive user inputs.')
     args = parser.parse_args()
 
-    try:
-        validate_collection_name(args.collection)
-    except InvalidCollectionName as e:
-        exit_msg(str(e))
-
     return args
 
 
@@ -63,21 +58,24 @@ def remove_blobs(collection_name, force_yes=False):
 
 def remove_collection(args):
     data = get_collections_data()
+    try:
+        validate_collection_name(args.collection, data['collections'], new=False)
+    except InvalidCollectionName as e:
+        exit_msg(str(e))
+
     if args.collection not in data['collections']:
         print('Invalid collection %s' % args.collection)
         exit(1)
 
     validate_collections(data['collections'], exit_on_errors=False)
     if data['dev_instances'] == 1 and data['collections'][args.collection]['for_dev']:
         data['dev_instances'] = 0
-    if data['stats_clients'] == 1 and \
-            data['collections'][args.collection]['env'][DOCKER_HOOVER_SNOOP_STATS]:
-        data['stats_clients'] = 0
 
     if not args.skip_index:
         remove_index(args.collection)
     del data['collections'][args.collection]
-    write_global_docker_file(data['collections'], bool(data['dev_instances']), bool(data['stats_clients']))
+    write_global_docker_file(data['collections'], bool(data['dev_instances']))
+    write_collections_settings(data['collections'])
 
     cleanup(args.collection)
     remove_pg_dir(args.collection)

src/removecollection.py

@@ -1,7 +1,10 @@
 import argparse
 
 from src.common import validate_collections, get_collections_data, write_global_docker_file, \
-    write_collections_docker_files, write_python_settings_files, write_env_files
+    write_collections_docker_files, write_python_settings_files, write_env_files,\
+    update_collections_settings, write_collections_settings,\
+    get_collections_data_old, collections_settings_file_name
+import os
 
 
 def get_args():
@@ -24,6 +27,14 @@ def get_args():
                            help='Remove profiling settings for the given collections. ' +
                                 'If no collections were specified profiling will be disabled for all.')
 
+    profiling = parser.add_mutually_exclusive_group()
+    profiling.add_argument('-t', '--tracing', action='append', nargs='*',
+                           help='Add tracing settings for the given collections. ' +
+                                'If no collections were specified the tracing will be enabled for all.')
+    profiling.add_argument('-z', '--no-tracing', action='append', nargs='*',
+                           help='Remove tracing settings for the given collections. ' +
+                                'If no collections were specified the tracing will be disabled for all.')
+
     autoindex = parser.add_mutually_exclusive_group()
     autoindex.add_argument('-a', '--autoindex', action='append', nargs='*',
                            help='Enable automatic indexing for the given collections. ' +
@@ -32,13 +43,6 @@ def get_args():
                            help='Enable automatic indexing for the given collections. ' +
                                 'If no collections were specified auto-indexing will be disabled for all.')
 
-    stats = parser.add_mutually_exclusive_group()
-    stats.add_argument('--enable-stats', action='append', nargs='*',
-                       help='Enable kibana stats for the given collections. ' +
-                            'If no collections were specified kibana will be enabled for all.')
-    stats.add_argument('--disable-stats', action='append', nargs='*',
-                       help='Disable kibana stats for the given collections. ' +
-                            'If no collections were specified kibana will be disabled for all.')
     return parser.parse_args()
 
 
@@ -58,24 +62,38 @@ def read_collections_arg(add_list, remove_list, all_collections):
 
 
 def update_settings(args):
-    collections = get_collections_data()['collections']
+    if not os.path.isfile(collections_settings_file_name):
+        data = get_collections_data_old()
+    else:
+        data = get_collections_data()
+
+    collections = data['collections']
     if len(collections):
         validate_collections(collections)
 
     collections_names = set(collections.keys())
+
+    indexing, disable_indexing = read_collections_arg(args.autoindex, args.manual_indexing,
+                                                      collections_names)
+    update_collections_settings(data, {'autoindex': not disable_indexing}, indexing)
+
     profiling, remove_profiling = read_collections_arg(args.profiling, args.no_profiling,
                                                        collections_names)
+    update_collections_settings(data, {'profiling': not remove_profiling}, profiling)
+
+    tracing, disable_tracing = read_collections_arg(args.tracing, args.no_tracing,
+                                                    collections_names)
+    update_collections_settings(data, {'tracing': not disable_tracing}, tracing)
+
     for_dev, remove_dev = read_collections_arg(args.dev, args.remove_dev, collections_names)
-    indexing, disable_indexing = read_collections_arg(args.autoindex, args.manual_indexing,
-                                                      collections_names)
-    stats, disable_stats = read_collections_arg(args.enable_stats, args.disable_stats, collections_names)
-
-    stats_clients = write_env_files(collections, stats, disable_stats)
-    write_python_settings_files(collections, profiling, remove_profiling, for_dev, remove_dev)
-    dev_instances = write_collections_docker_files(collections, args.snoop_image, profiling,
-                                                   remove_profiling, for_dev, remove_dev,
-                                                   indexing, disable_indexing, stats, disable_stats)
-    write_global_docker_file(collections, bool(dev_instances), bool(stats_clients))
+    update_collections_settings(data, {'for_dev': not remove_dev}, for_dev)
+
+    write_env_files(collections)
+
+    write_python_settings_files(collections)
+    dev_instances = write_collections_docker_files(collections)
+    write_global_docker_file(collections, bool(dev_instances))
+    write_collections_settings(collections)
 
     print('Restart docker-compose:')
     print('  $ docker-compose down && docker-compose up -d')

src/updatesettings.py

@@ -40,6 +40,6 @@
       WAIT_HOSTS_TIMEOUT: 60
     depends_on:
       - snoop-rabbitmq
-      - snoop-tika{{ snoop_stats }}
+      - snoop-tika
       - search-es
       - snoop-pg--{{ collection_name }}

templates/docker-collection.yml

@@ -0,0 +1,11 @@
+
+TRACING_ENABLED = False
+
+TRACING_SERVICE = 'snoop'
+
+TRACING_HOST = 'zipkin'
+
+TRACING_PORT = 9411
+
+TRACING_API = '/api/v2/spans'
+

templates/snoop-settings-tracing.py

@@ -1,5 +1,4 @@
 DOCKER_HOOVER_SNOOP_SECRET_KEY={{ DOCKER_HOOVER_SNOOP_SECRET_KEY }}
 DOCKER_HOOVER_SNOOP_DEBUG={{ DOCKER_HOOVER_SNOOP_DEBUG }}
 DOCKER_HOOVER_SNOOP_BASE_URL={{ DOCKER_HOOVER_SNOOP_BASE_URL }}
-DOCKER_HOOVER_SNOOP_STATS={{ DOCKER_HOOVER_SNOOP_STATS }}