From cb250d17213ba32d5b8d1ef7b08f36c88e0c6668 Mon Sep 17 00:00:00 2001 From: Antoine Date: Thu, 2 May 2019 22:30:44 +0200 Subject: [PATCH 01/19] Added different config templates for exporters. --- watchme/config/__init__.py | 9 +++++---- watchme/config/templates/pushgateway/watchme.cfg | 6 ++++++ 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 watchme/config/templates/pushgateway/watchme.cfg diff --git a/watchme/config/__init__.py b/watchme/config/__init__.py index 2495286..8ac7d3f 100644 --- a/watchme/config/__init__.py +++ b/watchme/config/__init__.py @@ -26,15 +26,16 @@ # CONFIG TEMPLATES ############################################################# -def get_configfile_template(): +def get_configfile_template(watcher_type=None): '''return the full path to the default configuration file ''' - return _get_config('watchme.cfg') + return _get_config('watchme.cfg', watcher_type) -def _get_config(name): +def _get_config(name, watcher_type): '''shared function to return a file in the config directory ''' - return os.path.abspath(os.path.join(get_installdir(), 'config', name)) + watcher_path = watcher_type or '' + return os.path.abspath(os.path.join(get_installdir(), 'config', watcher_path, name)) # ACTIVE CONFIG FILES ########################################################## diff --git a/watchme/config/templates/pushgateway/watchme.cfg b/watchme/config/templates/pushgateway/watchme.cfg new file mode 100644 index 0000000..b0b4b3a --- /dev/null +++ b/watchme/config/templates/pushgateway/watchme.cfg @@ -0,0 +1,6 @@ +[watcher] +active = false +type = urls + +[pushgateway] +url = localhost:9091 \ No newline at end of file From cd3eeac9700ba887f60d364d9dd65b8efa89daf5 Mon Sep 17 00:00:00 2001 From: Antoine Date: Thu, 2 May 2019 22:58:56 +0200 Subject: [PATCH 02/19] Added the exporter option to the create command. --- watchme/client/__init__.py | 4 ++++ watchme/client/create.py | 2 +- watchme/command/create.py | 4 ++-- watchme/config/__init__.py | 14 +++++++------- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/watchme/client/__init__.py b/watchme/client/__init__.py index ae926a8..b4db39a 100644 --- a/watchme/client/__init__.py +++ b/watchme/client/__init__.py @@ -110,6 +110,10 @@ def get_parser(): create.add_argument('watchers', nargs="*", help='watchers to create (default: single watcher)') + create.add_argument('--exporter', dest="exporter", + help="where to export the data", + default=None) + # add add = subparsers.add_parser("add", diff --git a/watchme/client/create.py b/watchme/client/create.py index cf7e73b..eb92018 100644 --- a/watchme/client/create.py +++ b/watchme/client/create.py @@ -19,5 +19,5 @@ def main(args, extra): watchers = ['watcher'] for watcher in watchers: - create_watcher(watcher) + create_watcher(watcher, exporter=args.exporter) diff --git a/watchme/command/create.py b/watchme/command/create.py index 18c8efa..f745c1a 100644 --- a/watchme/command/create.py +++ b/watchme/command/create.py @@ -18,7 +18,7 @@ import os -def create_watcher(name=None, watcher_type=None, base=None): +def create_watcher(name=None, watcher_type=None, base=None, exporter=None): '''create a watcher, meaning a folder with a configuration and initialized git repo. @@ -47,7 +47,7 @@ def create_watcher(name=None, watcher_type=None, base=None): run_command("git --git-dir=%s/.git config commit.gpgsign false" % repo) # Add the watcher configuration file - generate_watcher_config(repo, watcher_type) + generate_watcher_config(repo, watcher_type, exporter) run_command("git -C %s add watchme.cfg" % repo) return repo diff --git a/watchme/config/__init__.py b/watchme/config/__init__.py index 8ac7d3f..0f9ddc3 100644 --- a/watchme/config/__init__.py +++ b/watchme/config/__init__.py @@ -26,16 +26,16 @@ # CONFIG TEMPLATES ############################################################# -def get_configfile_template(watcher_type=None): +def get_configfile_template(exporter=None): '''return the full path to the default configuration file ''' - return _get_config('watchme.cfg', watcher_type) + return _get_config('watchme.cfg', exporter) -def _get_config(name, watcher_type): +def _get_config(name, exporter): '''shared function to return a file in the config directory ''' - watcher_path = watcher_type or '' - return os.path.abspath(os.path.join(get_installdir(), 'config', watcher_path, name)) + exporter_path = exporter or '' + return os.path.abspath(os.path.join(get_installdir(), 'config', 'templates', exporter_path, name)) # ACTIVE CONFIG FILES ########################################################## @@ -72,7 +72,7 @@ def read_config(filename): # WATCHER CONFIG ############################################################### -def generate_watcher_config(path, watcher_type=None): +def generate_watcher_config(path, watcher_type=None, exporter=None): '''generate a watcher config, meaning a watcher folder in the watchme base folder. @@ -81,7 +81,7 @@ def generate_watcher_config(path, watcher_type=None): path: the path to the watcher repository ''' check_exists(path) - configfile = get_configfile_template() + configfile = get_configfile_template(exporter) watcher_config = os.path.join(path, 'watchme.cfg') if not os.path.exists(watcher_config): bot.info('Generating watcher config %s' % watcher_config) From cc5764367c13dd7a5d32b3308373e375f5e2568d Mon Sep 17 00:00:00 2001 From: Antoine Date: Thu, 2 May 2019 23:02:00 +0200 Subject: [PATCH 03/19] Moved the watchme template base file to the templates folder. --- watchme/config/{ => templates}/watchme.cfg | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename watchme/config/{ => templates}/watchme.cfg (100%) diff --git a/watchme/config/watchme.cfg b/watchme/config/templates/watchme.cfg similarity index 100% rename from watchme/config/watchme.cfg rename to watchme/config/templates/watchme.cfg From f7ea2503093cd699b817097f708aca959f9e06dc Mon Sep 17 00:00:00 2001 From: Antoine Date: Sat, 4 May 2019 17:36:53 +0200 Subject: [PATCH 04/19] Added a regex option when performing a get_task --- watchme/watchers/urls/tasks.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/watchme/watchers/urls/tasks.py b/watchme/watchers/urls/tasks.py index 04d749d..8f6254f 100644 --- a/watchme/watchers/urls/tasks.py +++ b/watchme/watchers/urls/tasks.py @@ -19,6 +19,7 @@ import os import tempfile import requests +import re def get_task(url, **kwargs): @@ -36,15 +37,23 @@ def get_task(url, **kwargs): headers = get_headers(kwargs) for params in paramsets: + response = requests.get(url, params=params, headers=headers) if response.status_code == 200: - save_as = kwargs.get('save_as') + save_as = kwargs.get('save_as') + regex = kwargs.get('regex') + # Returning the result as json will detect dictionary, and save json if save_as == "json": result = response.json() + # If the user defined a regex, capture the first group matching + elif regex: + matching = re.search(regex, response.text) + result = matching.group() + # Otherwise, we return text else: result = response.text From 88329c7d4fbe9bd2025ab19b77f25af585f693a4 Mon Sep 17 00:00:00 2001 From: Antoine Date: Sun, 5 May 2019 17:50:39 +0200 Subject: [PATCH 05/19] First round of changes to add exporters --- .../config/templates/pushgateway/watchme.cfg | 5 +- watchme/exporters/__init__.py | 50 +++++ watchme/exporters/pushgateway/__init__.py | 22 ++ watchme/tasks/__init__.py | 18 ++ watchme/watchers/__init__.py | 203 ++++++++++++------ 5 files changed, 230 insertions(+), 68 deletions(-) create mode 100644 watchme/exporters/__init__.py create mode 100644 watchme/exporters/pushgateway/__init__.py diff --git a/watchme/config/templates/pushgateway/watchme.cfg b/watchme/config/templates/pushgateway/watchme.cfg index b0b4b3a..ebf8ca1 100644 --- a/watchme/config/templates/pushgateway/watchme.cfg +++ b/watchme/config/templates/pushgateway/watchme.cfg @@ -2,5 +2,6 @@ active = false type = urls -[pushgateway] -url = localhost:9091 \ No newline at end of file +[exporter-pushgateway] +url = localhost:9091 +active = true \ No newline at end of file diff --git a/watchme/exporters/__init__.py b/watchme/exporters/__init__.py new file mode 100644 index 0000000..74e1c50 --- /dev/null +++ b/watchme/exporters/__init__.py @@ -0,0 +1,50 @@ +''' + +Exporter class : defines properties of an exporter object +Note : No validation is currently done for the exporters. + +Author : Antoine Solnichkin +Date : 04/05/2019 + + +''' + +class ExporterBase(object): + + required_params = [] + + def __init__(self, name, params={}, **kwargs): + + self.name = name + self.valid = False + self.params = {} + self.set_params(params) + self.validate() + + # Parameters + + def set_params(self, params): + '''iterate through parameters, set into dictionary. + + Parameters + ========== + params: a list of key@value pairs to set. + ''' + for key,value in params.items(): + key = key.lower() + self.params[key] = value + + # Validation + + # For now, it should always be valid as no required parameters are defined. + def validate(self): + '''validate the parameters set for the Exporter. Exit if there are any + errors. Ensure required parameters are defined, and have correct + values. + ''' + self.valid = True + + for param in self.required_params: + if param not in self.params: + bot.error('Missing required parameter: %s' % param) + self.valid = False \ No newline at end of file diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py new file mode 100644 index 0000000..eb426fe --- /dev/null +++ b/watchme/exporters/pushgateway/__init__.py @@ -0,0 +1,22 @@ +''' + +Copyright (C) 2019 Antoine Solnichkin. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +''' + +from watchme.exporters import ExporterBase + +class Exporter(ExporterBase): + + required_params = ['url'] + + def __init__(self, name, params={}, **kwargs): + + self.type = 'pushgateway' + + super(Exporter, self).__init__(name, params, **kwargs) \ No newline at end of file diff --git a/watchme/tasks/__init__.py b/watchme/tasks/__init__.py index 43e5ec6..f6704cb 100644 --- a/watchme/tasks/__init__.py +++ b/watchme/tasks/__init__.py @@ -13,6 +13,7 @@ write_file, write_json ) +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway import shutil import os @@ -217,3 +218,20 @@ def _save_text_list(self, results, repo): ''' file_name = self.params.get('file_name', 'result.txt') return self._save_list(results, repo, self._save_text, file_name) + + def _save_pushgateway_list(self, name, results): + '''for a list of general text results, send them to a pushgateway. + + Parameters + ========== + results: list of string results to write to the pushgateway + ''' + registry = CollectorRegistry() + for r in range(len(results)): + result = results[r] + g = Gauge(name.replace('-', ':'), '', registry=registry) + g.set(result) + push_to_gateway('localhost:9091', job='watchme', registry=registry) + + + diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index 8945f92..d9270a0 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -8,7 +8,7 @@ ''' -from watchme.logger import ( bot, RobotNamer ) +from watchme.logger import (bot, RobotNamer) from watchme.version import __version__ from watchme.defaults import ( WATCHME_BASE_DIR, @@ -70,12 +70,12 @@ class Watcher(object): - repo=None - configfile=None + repo = None + configfile = None - def __init__(self, name=None, - base=None, - create=False, **kwargs): + def __init__(self, name=None, + base=None, + create=False, **kwargs): '''the watcher base loads configuration files for the user (in $HOME) and module, and then stores any arguments given from the caller @@ -94,7 +94,6 @@ def __init__(self, name=None, # Load the configuration self.load_config() - def _set_base(self, base=None, create=False): ''' set the base for the watcher, ensuring that it exists. @@ -114,9 +113,10 @@ def _set_base(self, base=None, create=False): # If the watcher doesn't exist and we need to create: if not os.path.exists(self.repo) or not os.path.exists(self.configfile): if create is True: - create_watcher(self.name) + create_watcher(self.name) else: - bot.exit('Watcher %s does not exist. Use watchme create.' % self.name) + bot.exit( + 'Watcher %s does not exist. Use watchme create.' % self.name) # Config @@ -125,7 +125,6 @@ def save(self): '''save the configuration to file.''' write_config(self.configfile, self.config) - def edit_task(self, name, action, key, value=None): '''edit a task, meaning doing an addition (add), update (update), or "remove", All actions require a value other than remove. @@ -139,7 +138,7 @@ def edit_task(self, name, action, key, value=None): ''' if not self.has_task(name): - bot.exit('%s is not a task defined by %s' %(name, self.name)) + bot.exit('%s is not a task defined by %s' % (name, self.name)) if action not in ['update', 'add', 'remove']: bot.exit('Action must be update, add, or remove') @@ -149,7 +148,7 @@ def edit_task(self, name, action, key, value=None): # Add, and it doesn't exist so it's okay if action == "add" and key not in self.config[name]: - bot.info('Adding %s:%s to %s' %(key, value, name)) + bot.info('Adding %s:%s to %s' % (key, value, name)) self.set_setting(name, key, value) # Already exists, encourage user to update @@ -158,7 +157,7 @@ def edit_task(self, name, action, key, value=None): # Update, and it's a valid choice elif action == 'update' and key in self.config[name]: - bot.info('Updating %s to %s in %s' %(key, value, name)) + bot.info('Updating %s to %s in %s' % (key, value, name)) self.set_setting(name, key, value) # Update, and it's not a valid choice @@ -167,7 +166,7 @@ def edit_task(self, name, action, key, value=None): # Remove, and it's a valid choice elif action == "remove" and key in self.config[name]: - bot.info('Removing %s' % key ) + bot.info('Removing %s' % key) del self.config[name][key] # Remove, and it's not a valid choice @@ -175,7 +174,6 @@ def edit_task(self, name, action, key, value=None): bot.exit('%s is not found in config, cannot be removed.' % key) self.save() - def has_section(self, name): '''returns True or False to indicate if the watcher has a specified section. To get a task, use self.has_task. @@ -187,10 +185,9 @@ def has_section(self, name): self.load_config() if name in self.config._sections: return True - bot.warning('%s not found for watcher %s' %(name, self.name)) + bot.warning('%s not found for watcher %s' % (name, self.name)) return False - def load_config(self): '''load a configuration file, and set the active setting for the watcher if the file doesn't exist, the function will exit and prompt the user @@ -198,7 +195,7 @@ def load_config(self): it will be written with a default active status set to false. ''' if not hasattr(self, 'config'): - + # Load the configuration file if it exists (will exit if not found) if self.configfile != None: self.config = read_config(self.configfile) @@ -211,7 +208,6 @@ def load_config(self): # Only update the config if we've changed it self.save() - def _get_params_dict(self, pairs): '''iterate through parameters, make keys lowercase, and ensure valid format. @@ -224,7 +220,7 @@ def _get_params_dict(self, pairs): for pair in pairs: if "@" not in pair: bot.exit('incorrectly formatted param, must be key@value') - key,value = pair.split('@', 1) + key, value = pair.split('@', 1) key = key.lower() # All tasks are not allowed to have default params @@ -249,12 +245,12 @@ def add_task(self, task, task_type, params, force=False, active="true"): params: list of parameters to be validated (key@value) force: if task already exists, overwrite active: add the task as active (default "true") - + ''' # Check again, in case user calling from client if not task.startswith('task'): bot.exit('Task name must start with "task" (e.g., task-reddit)') - + # Ensure it's a valid type if task_type not in WATCHME_TASK_TYPES: bot.exit('%s is not a valid type: %s' % WATCHME_TASK_TYPES) @@ -272,7 +268,7 @@ def add_task(self, task, task_type, params, force=False, active="true"): # Convert list to dictionary params = self._get_params_dict(params) - + # Creating the task will validate parameters newtask = Task(task, params=params) @@ -283,7 +279,6 @@ def add_task(self, task, task_type, params, force=False, active="true"): # Write to file (all tasks get active = True added, and type) self._add_task(newtask, force, active) - def _add_task(self, task, force=False, active='true'): '''add a new task to the watcher, meaning we: @@ -322,7 +317,8 @@ def _add_task(self, task, force=False, active='true'): git_add(self.repo, task.name) # Commit changes - git_commit(repo=self.repo, task=self.name, message="ADD task %s" % task.name) + git_commit(repo=self.repo, task=self.name, + message="ADD task %s" % task.name) # Delete @@ -335,7 +331,8 @@ def delete(self): if self.is_frozen(): bot.exit('watcher %s is frozen, unfreeze to delete.' % self.name) elif self.is_protected(): - bot.exit('watcher %s is protected, turn off protection to delete.' % self.name) + bot.exit( + 'watcher %s is protected, turn off protection to delete.' % self.name) repo = os.path.dirname(self.configfile) @@ -344,8 +341,7 @@ def delete(self): bot.info('Removing watcher %s' % self.name) shutil.rmtree(repo) else: - bot.exit("%s:%s doesn't exist" %(self.name, repo)) - + bot.exit("%s:%s doesn't exist" % (self.name, repo)) def remove_task(self, task): '''remove a task from the watcher repo, if it exists, and the @@ -373,12 +369,12 @@ def remove_task(self, task): # Inspect - + def inspect(self, tasks=None, create_command=False): '''inspect a watcher, or one or more tasks belonging to it. This means printing the configuration for the entire watcher (if tasks is None) or just for one or more tasks. - + Parameters ========== tasks: one or more tasks to inspect (None will show entire file) @@ -403,7 +399,6 @@ def inspect(self, tasks=None, create_command=False): else: self.print_add_task(task) - def list(self, quiet=False): '''list the watchers. If quiet is True, don't print to the screen.''' watchers = get_watchers(base=self.base, quiet=quiet) @@ -415,12 +410,11 @@ def protect(self, status="on"): '''protect a watcher, meaning that it cannot be deleted. This does not influence removing a task. To freeze the entire watcher, use the freeze() function. - ''' + ''' self._set_status('watcher', 'protected', status) git_commit(self.repo, self.name, "PROTECT %s" % status) self.print_section('watcher') - def freeze(self): '''freeze a watcher, meaning that it along with its tasks cannot be deleted. This does not prevent the user from manual editing. @@ -450,7 +444,6 @@ def _set_status(self, section, setting, value): bot.exit('Status must be "on" or "off"') self.set_setting(section, setting, value) self.save() - def is_protected(self): '''return a boolean to indicate if the watcher is protected or frozen. @@ -462,7 +455,6 @@ def is_protected(self): if self.get_setting('watcher', status) == "on": protected = True return protected - def is_frozen(self): '''return a boolean to indicate if the watcher is frozen. @@ -478,7 +470,7 @@ def is_frozen(self): def _active_status(self, status='true', name=None): '''a general function to change the status, used by activate and deactivate. - + Parameters ========== status: must be one of true, false @@ -492,7 +484,7 @@ def _active_status(self, status='true', name=None): # Cut out early if section not in config if name not in self.config._sections: - bot.exit('%s is not a valid task or section' % name) + bot.exit('%s is not a valid task or section' % name) if status not in ['true', 'false']: bot.exit('status must be true or false.') @@ -508,9 +500,9 @@ def _active_status(self, status='true', name=None): # Add the task name if name != None: - message = "%s task %s" %(message, name) + message = "%s task %s" % (message, name) - bot.info('[%s|%s] active: %s' % (name, self.name, status)) + bot.info('[%s|%s] active: %s' % (name, self.name, status)) return message def activate(self, task=None): @@ -518,7 +510,6 @@ def activate(self, task=None): ''' message = self._active_status('true', task) git_commit(self.repo, self.name, message) - def deactivate(self, task=None): '''turn the active status of a watcher to false. If a task is provided, @@ -528,7 +519,6 @@ def deactivate(self, task=None): message = self._active_status('false', task) git_commit(self.repo, self.name, message) - def is_active(self, task=None): '''determine if the watcher is active by reading from the config directly if a task name is provided, check the active status of the task @@ -551,7 +541,6 @@ def has_task(self, name): return True return False - def get_task(self, name, save=False): '''get a particular task, based on the name. This is where each type of class should check the "type" parameter from the config, and @@ -590,7 +579,6 @@ def get_task(self, name, save=False): return task - def _task_selected(self, task, regexp=None): '''check if a task is active and (if defined) passes user provided task names or regular expressions. @@ -600,7 +588,7 @@ def _task_selected(self, task, regexp=None): task: the task object to check regexp: an optional regular expression (or name) to check ''' - selected = True + selected = True # A task can be None if it wasn't found if task == None: @@ -611,7 +599,7 @@ def _task_selected(self, task, regexp=None): if active == "false": bot.info('Task %s is not active.' % task) selected = False - + # The user wants to search for a custom task name if regexp != None: if not re.search(regexp, task): @@ -620,7 +608,6 @@ def _task_selected(self, task, regexp=None): return selected - def get_tasks(self, regexp=None): '''get the tasks for a watcher, possibly matching a regular expression. A list of dictionaries is returned, each holding the parameters for @@ -645,8 +632,90 @@ def get_tasks(self, regexp=None): tasks.append(task) bot.info('Found %s contender tasks.' % len(tasks)) - return tasks + return tasks + +# Get Exporters + + def get_exporter(self, name, save=False): + '''get a particular exporter, based on the name. We could have + an exporter type (similarly to Tasks), but it is quite useless right now. + Parameters + ========== + name: the name of the exporter to load + save: if saving, will be True + ''' + self.load_config() + + exporter = None + + # Only sections that start with exporter- are considered exporters + if name in self.config._sections and name.startswith('exporter'): + + print("Elligible exporter " + name) + + params = self.config._sections[name] + + from watchme.exporters import ExporterBase + + exporter = ExporterBase(name, params) + return exporter + + def get_exporters(self, regexp=None): + '''get the exporters for a watcher, possibly matching a regular expression. + A list of dictionaries is returned, each holding the parameters for + an exporter. An exporter has an active attribute. + + Parameters + ========== + regexp: if supplied, the user wants to export to destinations + that only match the expression specified. + ''' + self.load_config() + + exporters = [] + + for section in self.config._sections: + + # Get the exporter based on the section name + exporter = self.get_exporter(section) + + # Check that the exporter should be used, and is valid + if exporter != None: + if self._exporter_selected(exporter, regexp) and exporter.valid: + exporters.append(exporter) + + bot.info('Found %s contender exporters.' % len(exporters)) + return exporters + + def _exporter_selected(self, exporter, regexp=None): + '''check if an exporter is active and (if defined) passes user provided + exporter names or regular expressions. + + Parameters + ========== + exporter: the exporter object to check + regexp: an optional regular expression (or name) to check + ''' + selected = True + + # A exporter can be None if it wasn't found + if exporter == None: + selected = False + + # Is the exporter not active (undefined is active)? + active = exporter.params.get('active', 'true') + if active == "false": + bot.info('Exporter %s is not active.' % exporter) + selected = False + + # The user wants to search for a custom task name + if regexp != None: + if not re.search(regexp, exporter): + bot.info('Exporter %s is selected for data export.' % exporter) + selected = False + + return selected # Running Tasks @@ -668,10 +737,10 @@ def run_tasks(self, queue, parallel=True, show_progress=True): {'task-reddit-hpc': [('url', 'https://www.reddit.com/r/hpc'), ('active', 'true'), ('type', 'urls')]} - ''' + ''' if parallel is True: return self._run_parallel(queue, show_progress) - + # Otherwise, run in serial results = {} @@ -686,11 +755,10 @@ def run_tasks(self, queue, parallel=True, show_progress=True): else: bot.info('Running %s' % prefix) results[task.name] = task.run() - progress+=1 + progress += 1 return results - def _run_parallel(self, queue, show_progress=True): ''' run tasks in parallel using the Workers class. Returns a dictionary (lookup) wit results, with the key being the task name @@ -707,14 +775,13 @@ def _run_parallel(self, queue, show_progress=True): for task in queue: - # Export parameters and functions + # Export parameters and functions funcs[task.name] = task.export_func() tasks[task.name] = task.export_params() workers = Workers(show_progress=show_progress) return workers.run(funcs, tasks) - def run(self, regexp=None, parallel=True, test=False, show_progress=True): '''run the watcher, which should be done via the crontab, including: @@ -745,10 +812,13 @@ def run(self, regexp=None, parallel=True, test=False, show_progress=True): # Step 2: get the tasks associated with the run, a list of param dicts tasks = self.get_tasks() - # Step 3: Run the tasks. This means preparing a list of funcs/params, + # Step 3 : get the exporters if any were declared in the watcher configuration. + exporters = self.get_exporters() + + # Step 4: Run the tasks. This means preparing a list of funcs/params, # and then submitting with multiprocessing results = self.run_tasks(tasks, parallel, show_progress) - + # Finally, finish the runs. if test is False: self.finish_runs(results) @@ -756,13 +826,15 @@ def run(self, regexp=None, parallel=True, test=False, show_progress=True): # or print results to the screen print(json.dumps(results, indent=4)) - def finish_runs(self, results): '''finish runs should take a dictionary of results, with keys as the folder name, and for each, depending on the result type, write the result to file (or update file) and then commit to git. + Eventually, for active exporters, it should try to export it to + destinations defined. + Parameters ========== results: a dictionary of tasks, with keys as the task name, and @@ -781,8 +853,7 @@ def finish_runs(self, results): git_add(self.repo, task_folder) # Case 1. The result is a list - if isinstance(result, list): - + if isinstance(result, list): # Get rid of Nones, if the user accidentally added result = [r for r in result if r] @@ -801,16 +872,16 @@ def finish_runs(self, results): # Otherwise, sniff for list of paths elif os.path.exists(result[0]): bot.debug('Found list of paths...') - files += task._save_files_list(result, self.repo) + files += task._save_files_list(result, self.repo) # Finally, assume just writing text to file else: bot.debug('Saving content from list to file...') - files += task._save_text_list(result, self.repo) + files += task._save_text_list(result, self.repo) + task._save_pushgateway_list(name, result) # Case 2. The result is a string elif isinstance(result, str): - # if it's a path to a file, just save to repository if os.path.exists(result): files.append(task._save_file(result, self.repo)) @@ -835,18 +906,18 @@ def finish_runs(self, results): # Add files to git, and commit files.append(write_timestamp(repo=self.repo, task=name)) git_add(repo=self.repo, files=files) - git_commit(repo=self.repo, - task=self.name, + git_commit(repo=self.repo, + task=self.name, message="ADD results %s" % name) # Identification def __repr__(self): - return "[watcher|%s]" %self.name + return "[watcher|%s]" % self.name def __str__(self): - return "[watcher|%s]" %self.name + return "[watcher|%s]" % self.name # Settings @@ -859,7 +930,7 @@ def __str__(self): Watcher.print_section = print_section Watcher.print_add_task = print_add_task -# Schedule +# Schedule Watcher.remove_schedule = remove_schedule Watcher.get_crontab = get_crontab From a9755e2a09e76f08f6fc05a3e669225a348ec920 Mon Sep 17 00:00:00 2001 From: Antoine Date: Sun, 5 May 2019 18:02:40 +0200 Subject: [PATCH 06/19] Infering exporter instance given its declared type --- watchme/config/templates/pushgateway/watchme.cfg | 3 ++- watchme/watchers/__init__.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/watchme/config/templates/pushgateway/watchme.cfg b/watchme/config/templates/pushgateway/watchme.cfg index ebf8ca1..42ab161 100644 --- a/watchme/config/templates/pushgateway/watchme.cfg +++ b/watchme/config/templates/pushgateway/watchme.cfg @@ -4,4 +4,5 @@ type = urls [exporter-pushgateway] url = localhost:9091 -active = true \ No newline at end of file +type = pushgateway +active = true diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index d9270a0..8ab25ab 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -654,8 +654,15 @@ def get_exporter(self, name, save=False): print("Elligible exporter " + name) params = self.config._sections[name] + + # Instantiate the correct exporter given the type + exporter_type = params['type'] + + if exporter_type == 'pushgateway': + from watchme.exporters import ExporterBase - from watchme.exporters import ExporterBase + else: + bot.exit('exporter type %s does not exist' % exporter_type) exporter = ExporterBase(name, params) From a2d724f450b26b16e8012991fa84c7c3ffd887d6 Mon Sep 17 00:00:00 2001 From: Antoine Date: Sun, 5 May 2019 18:12:33 +0200 Subject: [PATCH 07/19] Helper function to retrieve the exporter type --- watchme/exporters/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/watchme/exporters/__init__.py b/watchme/exporters/__init__.py index 74e1c50..787e6ff 100644 --- a/watchme/exporters/__init__.py +++ b/watchme/exporters/__init__.py @@ -21,6 +21,12 @@ def __init__(self, name, params={}, **kwargs): self.set_params(params) self.validate() + + def get_type(self): + '''get the exporter type. + ''' + return self.type + # Parameters def set_params(self, params): @@ -47,4 +53,5 @@ def validate(self): for param in self.required_params: if param not in self.params: bot.error('Missing required parameter: %s' % param) - self.valid = False \ No newline at end of file + self.valid = False + From 5d9168e559ca6ae425214e623e82535c83737c83 Mon Sep 17 00:00:00 2001 From: Antoine Date: Sun, 5 May 2019 18:36:54 +0200 Subject: [PATCH 08/19] Removed exporter specific function from task classes --- watchme/exporters/pushgateway/__init__.py | 18 ++++++++++++- watchme/tasks/__init__.py | 16 ----------- watchme/watchers/__init__.py | 33 ++++++++++++++++++----- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py index eb426fe..02b64ef 100644 --- a/watchme/exporters/pushgateway/__init__.py +++ b/watchme/exporters/pushgateway/__init__.py @@ -11,6 +11,8 @@ from watchme.exporters import ExporterBase +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway + class Exporter(ExporterBase): required_params = ['url'] @@ -19,4 +21,18 @@ def __init__(self, name, params={}, **kwargs): self.type = 'pushgateway' - super(Exporter, self).__init__(name, params, **kwargs) \ No newline at end of file + super(Exporter, self).__init__(name, params, **kwargs) + + def _save_text_list(self, name, results): + '''for a list of general text results, send them to a pushgateway. + + Parameters + ========== + results: list of string results to write to the pushgateway + ''' + registry = CollectorRegistry() + for r in range(len(results)): + result = results[r] + g = Gauge(name.replace('-', ':'), '', registry=registry) + g.set(result) + push_to_gateway(self.params['url'], job='watchme', registry=registry) \ No newline at end of file diff --git a/watchme/tasks/__init__.py b/watchme/tasks/__init__.py index f6704cb..e44635e 100644 --- a/watchme/tasks/__init__.py +++ b/watchme/tasks/__init__.py @@ -13,7 +13,6 @@ write_file, write_json ) -from prometheus_client import CollectorRegistry, Gauge, push_to_gateway import shutil import os @@ -219,19 +218,4 @@ def _save_text_list(self, results, repo): file_name = self.params.get('file_name', 'result.txt') return self._save_list(results, repo, self._save_text, file_name) - def _save_pushgateway_list(self, name, results): - '''for a list of general text results, send them to a pushgateway. - - Parameters - ========== - results: list of string results to write to the pushgateway - ''' - registry = CollectorRegistry() - for r in range(len(results)): - result = results[r] - g = Gauge(name.replace('-', ':'), '', registry=registry) - g.set(result) - push_to_gateway('localhost:9091', job='watchme', registry=registry) - - diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index 8ab25ab..1b2440f 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -659,12 +659,12 @@ def get_exporter(self, name, save=False): exporter_type = params['type'] if exporter_type == 'pushgateway': - from watchme.exporters import ExporterBase + from watchme.exporters.pushgateway import Exporter else: bot.exit('exporter type %s does not exist' % exporter_type) - exporter = ExporterBase(name, params) + exporter = Exporter(name, params) return exporter @@ -829,9 +829,32 @@ def run(self, regexp=None, parallel=True, test=False, show_progress=True): # Finally, finish the runs. if test is False: self.finish_runs(results) + self.export_runs(results, exporters) else: # or print results to the screen print(json.dumps(results, indent=4)) + + def export_runs(self, results, exporters): + ''' export data retrieved to the set of exporters defined and active. + maybe an export flag could be set to choose to run + export? + ''' + for name, result in results.items(): + + task = self.get_task(name, save=True) + # Case 1. The result is a list + if isinstance(result, list): + + # Get rid of Nones, if the user accidentally added + result = [r for r in result if r] + + if len(result) == 0: + bot.error('%s returned empty list of results.' % name) + + # for a json, or a list of paths, ignore for now. + elif not(task.params.get('save_as') == 'json' or os.path.exists(result[0])): + for exporter in exporters: + bot.debug('Exporting list to ' + exporter.name) + exporter._save_text_list(name, result) def finish_runs(self, results): '''finish runs should take a dictionary of results, with keys as the @@ -839,9 +862,6 @@ def finish_runs(self, results): write the result to file (or update file) and then commit to git. - Eventually, for active exporters, it should try to export it to - destinations defined. - Parameters ========== results: a dictionary of tasks, with keys as the task name, and @@ -885,8 +905,7 @@ def finish_runs(self, results): else: bot.debug('Saving content from list to file...') files += task._save_text_list(result, self.repo) - task._save_pushgateway_list(name, result) - + # Case 2. The result is a string elif isinstance(result, str): # if it's a path to a file, just save to repository From 8a8d1559a8e81f224248659b82c86f31b50d8fe1 Mon Sep 17 00:00:00 2001 From: Antoine Date: Sun, 5 May 2019 18:45:14 +0200 Subject: [PATCH 09/19] Handling multiple export format in the exporter --- watchme/exporters/pushgateway/__init__.py | 26 +++++++++++++++++------ watchme/watchers/__init__.py | 10 +++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py index 02b64ef..b358aeb 100644 --- a/watchme/exporters/pushgateway/__init__.py +++ b/watchme/exporters/pushgateway/__init__.py @@ -20,7 +20,9 @@ class Exporter(ExporterBase): def __init__(self, name, params={}, **kwargs): self.type = 'pushgateway' - + + self.registry = CollectorRegistry() + super(Exporter, self).__init__(name, params, **kwargs) def _save_text_list(self, name, results): @@ -30,9 +32,21 @@ def _save_text_list(self, name, results): ========== results: list of string results to write to the pushgateway ''' - registry = CollectorRegistry() for r in range(len(results)): - result = results[r] - g = Gauge(name.replace('-', ':'), '', registry=registry) - g.set(result) - push_to_gateway(self.params['url'], job='watchme', registry=registry) \ No newline at end of file + self._write_to_pushgateway(results[r]) + + + def _save_text(self, result): + '''exports the text to the exporter + + Parameters + ========== + result: the result object to save, not a path to a file in this case + ''' + self._write_to_pushgateway(result) + + def _write_to_pushgateway(self, result): + g = Gauge(self.name.replace('-', ':'), '', registry=self.registry) + g.set(result) + push_to_gateway(self.params['url'], job='watchme', registry=self.registry) + \ No newline at end of file diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index 1b2440f..ba4474d 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -841,6 +841,7 @@ def export_runs(self, results, exporters): for name, result in results.items(): task = self.get_task(name, save=True) + # Case 1. The result is a list if isinstance(result, list): @@ -856,6 +857,15 @@ def export_runs(self, results, exporters): bot.debug('Exporting list to ' + exporter.name) exporter._save_text_list(name, result) + # Case 2. The result is a string + elif isinstance(result, str): + + # if it's a path to a file, ignore it. + if not(os.path.exists(result)): + exporter._save_text(result) + + # Case 3. The result is a dictionary, ignore it for now. + def finish_runs(self, results): '''finish runs should take a dictionary of results, with keys as the folder name, and for each, depending on the result type, From cce92979706b04ceed33ae688aaa18c8de4b0cbb Mon Sep 17 00:00:00 2001 From: Antoine Date: Sun, 5 May 2019 18:52:34 +0200 Subject: [PATCH 10/19] Catching exceptions thrown when exporters are not available. --- watchme/exporters/pushgateway/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py index b358aeb..40a12aa 100644 --- a/watchme/exporters/pushgateway/__init__.py +++ b/watchme/exporters/pushgateway/__init__.py @@ -8,7 +8,7 @@ ''' - +from watchme.logger import bot from watchme.exporters import ExporterBase from prometheus_client import CollectorRegistry, Gauge, push_to_gateway @@ -46,7 +46,16 @@ def _save_text(self, result): self._write_to_pushgateway(result) def _write_to_pushgateway(self, result): + ''' writes data to the pushgateway + + Parameters + ========== + result: the result object to save + ''' g = Gauge(self.name.replace('-', ':'), '', registry=self.registry) g.set(result) - push_to_gateway(self.params['url'], job='watchme', registry=self.registry) - \ No newline at end of file + + try: + push_to_gateway(self.params['url'], job='watchme', registry=self.registry) + except: + bot.error('An exception occurred while trying to export data using %s' % self.name) \ No newline at end of file From 1d4b89059dac306c4e64cb264aa122b8c30c4e01 Mon Sep 17 00:00:00 2001 From: Antoine Date: Sun, 5 May 2019 20:07:47 +0200 Subject: [PATCH 11/19] Handling regex on get_url_selection + error message when exporter does not exist --- watchme/config/__init__.py | 9 ++++++++- watchme/watchers/urls/helpers.py | 11 +++++++++-- watchme/watchers/urls/tasks.py | 6 +++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/watchme/config/__init__.py b/watchme/config/__init__.py index 0f9ddc3..82ca908 100644 --- a/watchme/config/__init__.py +++ b/watchme/config/__init__.py @@ -35,7 +35,14 @@ def _get_config(name, exporter): '''shared function to return a file in the config directory ''' exporter_path = exporter or '' - return os.path.abspath(os.path.join(get_installdir(), 'config', 'templates', exporter_path, name)) + + template_path = os.path.join(get_installdir(), 'config', 'templates', exporter_path, name) + + if os.path.exists(template_path): + return os.path.abspath(template_path) + else: + bot.info('The exporter specified does not exist : %s. The task was created with no exporters. ' % exporter) + return os.path.abspath(os.path.join(get_installdir(), 'config', 'templates', name)) # ACTIVE CONFIG FILES ########################################################## diff --git a/watchme/watchers/urls/helpers.py b/watchme/watchers/urls/helpers.py index 84e65d6..229e624 100644 --- a/watchme/watchers/urls/helpers.py +++ b/watchme/watchers/urls/helpers.py @@ -11,6 +11,7 @@ import os import tempfile import requests +import re # Helper functions @@ -72,7 +73,7 @@ def get_headers(kwargs): return headers -def get_results(url, selector, func=None, attributes=None, params={}, get_text=False, headers={}): +def get_results(url, selector, func=None, attributes=None, params={}, get_text=False, headers={}, regex=None): '''given a url, a function, an optional selector, optional attributes, and a set (dict) of parameters, perform a request. @@ -84,6 +85,7 @@ def get_results(url, selector, func=None, attributes=None, params={}, get_text=F attributes: optional, a list of attributes params: a dictionary of parameters headers: a dictionary of header key value pairs + regex : an optional regex ''' from bs4 import BeautifulSoup @@ -105,7 +107,12 @@ def get_results(url, selector, func=None, attributes=None, params={}, get_text=F # Does the user want to get text? elif get_text == True: - results.append(entry.text) + # Does the user want to capture a certain value? + if regex != None: + matching = re.search(regex, entry.text) + results.append(matching.group()) + else: + results.append(entry.text) # Otherwise, return the entire thing else: diff --git a/watchme/watchers/urls/tasks.py b/watchme/watchers/urls/tasks.py index 8f6254f..634816b 100644 --- a/watchme/watchers/urls/tasks.py +++ b/watchme/watchers/urls/tasks.py @@ -188,6 +188,9 @@ def get_url_selection(url, **kwargs): if kwargs.get('get_text') != None: get_text = True + # Does the user want to capture a certain value? + regex = kwargs.get('regex') + # Does the user want to get one or more attributes? attributes = kwargs.get('attributes', None) if attributes != None: @@ -207,7 +210,8 @@ def get_url_selection(url, **kwargs): headers=headers, attributes=attributes, params=params, - get_text=get_text) + get_text=get_text, + regex=regex) # No results if len(results) == 0: From 3a08ca4b5ce199761e0d76eddbda36a1105a373b Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Mon, 6 May 2019 13:50:10 -0400 Subject: [PATCH 12/19] initial bulk changes working on exporter Signed-off-by: Vanessa Sochat --- .github/CODE_OF_CONDUCT.md | 59 +++++++++++++ .github/CONTRIBUTING.md | 87 ++----------------- .gitignore | 1 + CHANGELOG.md | 1 + docs/_docs/getting-started/index.md | 12 ++- docs/_docs/install/index.md | 38 +++++++- docs/_docs/watcher-tasks/psutils.md | 2 +- docs/_docs/watcher-tasks/urls.md | 2 +- setup.py | 20 ++++- watchme/client/__init__.py | 26 +++++- watchme/client/export.py | 4 +- watchme/command/create.py | 5 +- watchme/config/__init__.py | 20 ++--- .../config/templates/pushgateway/watchme.cfg | 8 -- watchme/exporters/__init__.py | 11 ++- watchme/exporters/pushgateway/__init__.py | 18 ++-- watchme/version.py | 19 +++- 17 files changed, 203 insertions(+), 130 deletions(-) create mode 100644 .github/CODE_OF_CONDUCT.md delete mode 100644 watchme/config/templates/pushgateway/watchme.cfg diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..a366695 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,59 @@ +# WatchMe Code of Conduct v1.0 + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + + * Using welcoming and inclusive language + * Being respectful of differing viewpoints and experiences + * Gracefully accepting constructive criticism + * Focusing on what is best for the community + * Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + + * The use of sexualized language or imagery and unwelcome sexual attention or advances + * Trolling, insulting/derogatory comments, and personal or political attacks + * Public or private harassment + * Publishing others' private information, such as a physical or electronic address, without explicit permission + * Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting @vsoch directly. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. @vsoch is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +## Thanks + +This code of conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org/), version 1.4, +available at http://contributor-covenant.org/version/1/4. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 0303edd..feac720 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -2,14 +2,12 @@ This code is licensed under the MPL 2.0 [LICENSE](LICENSE). # Contributing -When contributing to the SIF Python Client, it is important to properly communicate the +When contributing to the WatchMe Python Client, it is important to properly communicate the gist of the contribution. If it is a simple code or editorial fix, simply explaining this within the GitHub Pull Request (PR) will suffice. But if this is a larger fix or Enhancement, it should be first discussed with the project -leader or developers. - -Please note we have a code of conduct, described below. Please follow it in -all your interactions with the project members and users. +leader or developers. Please also note that we have a [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) +that should be followed for all interactions with the project members and users. ## Pull Request Process @@ -33,78 +31,7 @@ all your interactions with the project members and users. done by the project lead, @vsoch (or approved by her). -# Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -### Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project leader (@vsoch). All -complaints will be reviewed and investigated and will result in a response -that is deemed necessary and appropriate to the circumstances. The project -team is obligated to maintain confidentiality with regard to the reporter of -an incident. Further details of specific enforcement policies may be posted -separately. - -Project maintainers, contributors and users who do not follow or enforce the -Code of Conduct in good faith may face temporary or permanent repercussions -with their involvement in the project as determined by the project's leader(s). - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version] - -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ +When you contribute to the project, you agree to add code under the provided +licensing terms, and also we ask that you add your name to the [AUTHORS](AUTHORS.md) +file. Any contribution in the way of documentation, features, or bug fixes is +greatly appreciated. diff --git a/.gitignore b/.gitignore index f4cbdd5..86cd0ef 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ watchme.egg-info/ dist build _site +.eggs __pycache__ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f1cd83..aa7fabd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Critical items to know are: - changed behaviour ## [master](https://github.com/vsoch/watchme/tree/master) + - Addition of exporters (0.0.17) - Adding option for regular expression for URL wachers, user agent header (0.0.16) - requests is missing from install dependencies (0.0.15) - small bug fixes (0.0.14) diff --git a/docs/_docs/getting-started/index.md b/docs/_docs/getting-started/index.md index 4fa7cd5..2dcd29c 100644 --- a/docs/_docs/getting-started/index.md +++ b/docs/_docs/getting-started/index.md @@ -637,7 +637,15 @@ as a single (loadable) json file: $ watchme export system task-cpu vanessa-thinkpad-t460s_vanessa.json --json ``` +### Advanced Export + +What if .git doesn't work for you, or you want to send data elsewhere? We had +a fantastic idea and contribution in [this thread](https://github.com/vsoch/watchme/issues/30) +that resulted in the creation of exporters, or additional exports to make +other than using GitHub. Each exporter is considered to be a plugin (akin to +a watcher) and can be optionally installed and used. For more information, +see the [exporters]({{ site.url }}/exporters) page. + ## Licenses -This code is licensed under the Affero GPL, version 3.0 or later [LICENSE](LICENSE). -The SIF Header format is licesed by [Sylabs](https://github.com/sylabs/sif/blob/master/pkg/sif/sif.go). +This code is licensed under the Mozilla, version 2.0 or later [LICENSE](LICENSE). diff --git a/docs/_docs/install/index.md b/docs/_docs/install/index.md index b72f63b..ed8cdc5 100644 --- a/docs/_docs/install/index.md +++ b/docs/_docs/install/index.md @@ -10,7 +10,8 @@ order: 1 The only dependency for watchme is to have [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [crontab](https://www.digitalocean.com/community/tutorials/how-to-use-cron-to-automate-tasks-on-a-vps) on your system. Git is used for version control of the pages you are watching, and crontab is -used for scheduling your watches. +used for scheduling your watches. If you want to install a custom exporter or +watcher, see [installing extras](#installing-extras) below. ## Install @@ -62,3 +63,38 @@ actions: If you have any questions or issues, please [open an issue]({{ site.repo }}/issues). + +## Installing Extras + +If you want to install all of watchme's exporters and watchers: + +```bash +$ pip install watchme[all] +``` + +To install all watchers only: + +```bash +$ pip install watchme[watchers] +``` + +To install all exporters only: + +```bash +$ pip install watchme[exporters] +``` + +To install a specific exporter: + +```bash +$ pip install watchme[exporter-pushgateway] +``` + +or a watcher: + +```bash +$ pip install watchme[watcher-urls-dynamic] +$ pip install watchme[watcher-psutils] +``` + +To see all of the choices, see [here](https://github.com/vsoch/watchme/blob/master/setup.py#L109) in the setup file. diff --git a/docs/_docs/watcher-tasks/psutils.md b/docs/_docs/watcher-tasks/psutils.md index 7bcadbe..0fe9ef4 100644 --- a/docs/_docs/watcher-tasks/psutils.md +++ b/docs/_docs/watcher-tasks/psutils.md @@ -11,7 +11,7 @@ basic python environment. If your python installation doesn't have the `psutil` module, install as follows: ```bash -pip install watchme[psutils] +pip install watchme[watcher-psutils] ``` Next, create a watcher for your tasks to live under: diff --git a/docs/_docs/watcher-tasks/urls.md b/docs/_docs/watcher-tasks/urls.md index 2cf607f..c7daf30 100644 --- a/docs/_docs/watcher-tasks/urls.md +++ b/docs/_docs/watcher-tasks/urls.md @@ -194,7 +194,7 @@ identified by a class or id) on a page. For this purpose, you can use the functi packages to do this: ```bash -$ pip install watchme[urls-dynamic] +$ pip install watchme[watcher-urls-dynamic] ``` This task will watch for changes based on a selection from a page. For example, diff --git a/setup.py b/setup.py index b45d663..8013243 100644 --- a/setup.py +++ b/setup.py @@ -76,10 +76,19 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): if __name__ == "__main__": + # Install all exporters and/or watchers INSTALL_REQUIRES = get_requirements(lookup) - URLS_DYNAMIC = get_requirements(lookup,'INSTALL_URLS_DYNAMIC') + INSTALL_ALL = get_requirements(lookup, 'INSTALL_ALL') + WATCHERS = get_requirements(lookup, 'INSTALL_WATCHERS') + EXPORTERS = get_requirements(lookup, 'INSTALL_EXPORTERS') + + # Watchers + URLS_DYNAMIC = get_requirements(lookup, 'INSTALL_URLS_DYNAMIC') PSUTILS = get_requirements(lookup, 'INSTALL_PSUTILS') + # Exporters + PUSHGATEWAY = get_requirements(lookup, 'INSTALL_PUSHGATEWAY') + setup(name=NAME, version=VERSION, author=AUTHOR, @@ -98,9 +107,12 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): tests_require=["pytest"], install_requires = INSTALL_REQUIRES, extras_require={ - 'all': [INSTALL_REQUIRES], - 'urls-dynamic': [URLS_DYNAMIC], - 'psutils': [PSUTILS] + 'all': [INSTALL_ALL], + 'watchers': [WATCHERS], + 'exporters': [EXPORTERS], + 'watcher-urls-dynamic': [URLS_DYNAMIC], + 'watcher-psutils': [PSUTILS] + 'exporter-pushgateway': [PUSHGATEWAY], }, classifiers=[ 'Intended Audience :: Science/Research', diff --git a/watchme/client/__init__.py b/watchme/client/__init__.py index b4db39a..f98aa68 100644 --- a/watchme/client/__init__.py +++ b/watchme/client/__init__.py @@ -101,6 +101,28 @@ def get_parser(): help="the output file (json) to export the data", default=None) + # push (intended for exporters) + + push = subparsers.add_parser("push", + help="push data to an extra exporter") + + push.add_argument('watcher', nargs=1, + help='the watcher export data from') + + push.add_argument('task', nargs=1, + help='the name of the task to push data for') + + push.add_argument('exporter', nargs=1, + help='the name of the exporter to push to') + + push.add_argument('--all', dest="all", + help="instead of last timepoint, push all temporal data.", + default=False, action='store_true') + + push.add_argument('--json', dest="json", + help="signal to load the file as json", + default=False, action='store_true') + # create @@ -111,9 +133,10 @@ def get_parser(): help='watchers to create (default: single watcher)') create.add_argument('--exporter', dest="exporter", - help="where to export the data", + help="an exporter to send the data", default=None) + # add add = subparsers.add_parser("add", @@ -319,6 +342,7 @@ def help(return_code=0): elif args.command == "inspect": from .inspect import main elif args.command == "list": from .ls import main elif args.command == "protect": from .protect import main + elif args.command == "push": from .push import main elif args.command == "remove": from .remove import main elif args.command == "run": from .run import main elif args.command == "schedule": from .schedule import main diff --git a/watchme/client/export.py b/watchme/client/export.py index 5d9b894..93418f8 100644 --- a/watchme/client/export.py +++ b/watchme/client/export.py @@ -15,7 +15,7 @@ import os def main(args, extra): - '''activate one or more watchers + '''export temporal data for a watcher ''' # Required - will print help if not provided name = args.watcher[0] @@ -23,7 +23,7 @@ def main(args, extra): filename = args.filename[0] if not task.startswith('task'): - example = 'watchme add watcher task-reddit url@https://www.reddit.com' + example = 'watchme export watcher task-reddit result.txt' bot.exit('Task name must start with "task", e.g., %s' % example) # Use the output file, or a temporary file diff --git a/watchme/command/create.py b/watchme/command/create.py index f745c1a..6c08d4e 100644 --- a/watchme/command/create.py +++ b/watchme/command/create.py @@ -18,7 +18,7 @@ import os -def create_watcher(name=None, watcher_type=None, base=None, exporter=None): +def create_watcher(name=None, watcher_type=None, base=None): '''create a watcher, meaning a folder with a configuration and initialized git repo. @@ -27,6 +27,7 @@ def create_watcher(name=None, watcher_type=None, base=None, exporter=None): name: the watcher to create, uses default or WATCHME_WATCHER watcher_type: the type of watcher to create. defaults to WATCHER_DEFAULT_TYPE + base: The watcher base to use (defaults to $HOME/.watchme) ''' if name == None: name = WATCHME_WATCHER @@ -47,7 +48,7 @@ def create_watcher(name=None, watcher_type=None, base=None, exporter=None): run_command("git --git-dir=%s/.git config commit.gpgsign false" % repo) # Add the watcher configuration file - generate_watcher_config(repo, watcher_type, exporter) + generate_watcher_config(repo, watcher_type) run_command("git -C %s add watchme.cfg" % repo) return repo diff --git a/watchme/config/__init__.py b/watchme/config/__init__.py index 82ca908..2495286 100644 --- a/watchme/config/__init__.py +++ b/watchme/config/__init__.py @@ -26,23 +26,15 @@ # CONFIG TEMPLATES ############################################################# -def get_configfile_template(exporter=None): +def get_configfile_template(): '''return the full path to the default configuration file ''' - return _get_config('watchme.cfg', exporter) + return _get_config('watchme.cfg') -def _get_config(name, exporter): +def _get_config(name): '''shared function to return a file in the config directory ''' - exporter_path = exporter or '' - - template_path = os.path.join(get_installdir(), 'config', 'templates', exporter_path, name) - - if os.path.exists(template_path): - return os.path.abspath(template_path) - else: - bot.info('The exporter specified does not exist : %s. The task was created with no exporters. ' % exporter) - return os.path.abspath(os.path.join(get_installdir(), 'config', 'templates', name)) + return os.path.abspath(os.path.join(get_installdir(), 'config', name)) # ACTIVE CONFIG FILES ########################################################## @@ -79,7 +71,7 @@ def read_config(filename): # WATCHER CONFIG ############################################################### -def generate_watcher_config(path, watcher_type=None, exporter=None): +def generate_watcher_config(path, watcher_type=None): '''generate a watcher config, meaning a watcher folder in the watchme base folder. @@ -88,7 +80,7 @@ def generate_watcher_config(path, watcher_type=None, exporter=None): path: the path to the watcher repository ''' check_exists(path) - configfile = get_configfile_template(exporter) + configfile = get_configfile_template() watcher_config = os.path.join(path, 'watchme.cfg') if not os.path.exists(watcher_config): bot.info('Generating watcher config %s' % watcher_config) diff --git a/watchme/config/templates/pushgateway/watchme.cfg b/watchme/config/templates/pushgateway/watchme.cfg deleted file mode 100644 index 42ab161..0000000 --- a/watchme/config/templates/pushgateway/watchme.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[watcher] -active = false -type = urls - -[exporter-pushgateway] -url = localhost:9091 -type = pushgateway -active = true diff --git a/watchme/exporters/__init__.py b/watchme/exporters/__init__.py index 787e6ff..c1c72b0 100644 --- a/watchme/exporters/__init__.py +++ b/watchme/exporters/__init__.py @@ -1,11 +1,11 @@ ''' -Exporter class : defines properties of an exporter object -Note : No validation is currently done for the exporters. - -Author : Antoine Solnichkin -Date : 04/05/2019 +Copyright (C) 2019 Vanessa Sochat +Copyright (C) 2019 Antoine Solnichkin +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. ''' @@ -54,4 +54,3 @@ def validate(self): if param not in self.params: bot.error('Missing required parameter: %s' % param) self.valid = False - diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py index 40a12aa..aca83ca 100644 --- a/watchme/exporters/pushgateway/__init__.py +++ b/watchme/exporters/pushgateway/__init__.py @@ -1,26 +1,31 @@ ''' -Copyright (C) 2019 Antoine Solnichkin. +Copyright (C) 2019 Vanessa Sochat +Copyright (C) 2019 Antoine Solnichkin This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - ''' + from watchme.logger import bot from watchme.exporters import ExporterBase -from prometheus_client import CollectorRegistry, Gauge, push_to_gateway - class Exporter(ExporterBase): required_params = ['url'] def __init__(self, name, params={}, **kwargs): + # Ensure that the user has installed PushGateway + try: + from prometheus_client import CollectorRegistry + except: + bot.error("prometheus_client module not found.") + bot.exit("pip install watchme[exporter-pushgateway]") + self.type = 'pushgateway' - self.registry = CollectorRegistry() super(Exporter, self).__init__(name, params, **kwargs) @@ -52,10 +57,11 @@ def _write_to_pushgateway(self, result): ========== result: the result object to save ''' + from prometheus_client import Gauge, push_to_gateway g = Gauge(self.name.replace('-', ':'), '', registry=self.registry) g.set(result) try: push_to_gateway(self.params['url'], job='watchme', registry=self.registry) except: - bot.error('An exception occurred while trying to export data using %s' % self.name) \ No newline at end of file + bot.error('An exception occurred while trying to export data using %s' % self.name) diff --git a/watchme/version.py b/watchme/version.py index 030bc84..a7ce3be 100644 --- a/watchme/version.py +++ b/watchme/version.py @@ -6,7 +6,7 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -__version__ = "0.0.16" +__version__ = "0.0.17" AUTHOR = 'Vanessa Sochat' AUTHOR_EMAIL = 'vsochat@stanford.edu' NAME = 'watchme' @@ -31,6 +31,21 @@ ('psutil', {'min_version': '5.4.3'}), ) +INSTALL_PUSHGATEWAY ( + ('prometheus_client', {'min_version': '0.6.0'}), +) + +# Install all watchers and exporters INSTALL_ALL = (INSTALL_REQUIRES + INSTALL_PSUTILS + - INSTALL_URLS_DYNAMIC) + INSTALL_URLS_DYNAMIC, + INSTALL_PUSHGATEWAY) + +# Install all watchers +INSTALL_WATCHERS = (INSTALL_REQUIRES + + INSTALL_PSUTILS + + INSTALL_URLS_DYNAMIC) + +# Install all exporters +INSTALL_EXPORTERS = (INSTALL_REQUIRES + + INSTALL_PUSHGATEWAY) From 434e5628afac0927acbaa5d68031291897b6c280 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Mon, 6 May 2019 15:55:47 -0400 Subject: [PATCH 13/19] first round of work to add exporters (nothing is tested yet!) Signed-off-by: Vanessa Sochat --- docs/_docs/exporters/index.md | 20 ++ docs/_docs/exporters/pushgateway.md | 18 ++ docs/_docs/getting-started/index.md | 4 +- watchme/client/__init__.py | 62 +++++-- watchme/client/activate.py | 3 +- watchme/client/{add.py => addtask.py} | 4 +- watchme/client/create.py | 2 +- watchme/client/deactivate.py | 1 + watchme/client/exporter.py | 60 ++++++ watchme/defaults.py | 1 + watchme/exporters/__init__.py | 10 + watchme/exporters/pushgateway/__init__.py | 11 +- watchme/watchers/README.md | 8 + watchme/watchers/__init__.py | 148 ++------------- watchme/watchers/data.py | 213 +++++++++++++++++++++- watchme/watchers/settings.py | 34 +++- 16 files changed, 436 insertions(+), 163 deletions(-) create mode 100644 docs/_docs/exporters/index.md create mode 100644 docs/_docs/exporters/pushgateway.md rename watchme/client/{add.py => addtask.py} (92%) create mode 100644 watchme/client/exporter.py diff --git a/docs/_docs/exporters/index.md b/docs/_docs/exporters/index.md new file mode 100644 index 0000000..5a6bfa1 --- /dev/null +++ b/docs/_docs/exporters/index.md @@ -0,0 +1,20 @@ +--- +title: WatchMe Exporters +category: Exporters +permalink: /exporters/index.html +order: 1 +--- + +Once you've followed the [getting started]({{ site.baseurl }}/getting-started/) +guide to [install]({{ site.baseurl }}/install/) watchme and have created your first +set of watchers, you might want to do more! Specifically, you can add +extra exporters to push your data to custom locations. The following exporters +are available: + + - [pushgateway]({{ site.baseurl }}/exporters/pushgateway/) to push data to a Prometheus gateway. + +And instructions for using a general exporter, either in sync with running a task +or separately, are provided below. + +#TODO write instructions + diff --git a/docs/_docs/exporters/pushgateway.md b/docs/_docs/exporters/pushgateway.md new file mode 100644 index 0000000..615d981 --- /dev/null +++ b/docs/_docs/exporters/pushgateway.md @@ -0,0 +1,18 @@ +--- +title: Pushgateway +category: Exporters +permalink: /exporters/pushgateway/ +order: 2 +--- + +The [pushgateway](https://github.com/prometheus/pushgateway) exporter +will allow you to export data for a watcher to a Prometheus gateway. +To install the exporter, you will need to install its dependencies: + +```bash +pip install watchme[exporter-pushgateway] +``` + +The dependencies include the [prometheus client](https://github.com/prometheus/client_python). + +**under development** diff --git a/docs/_docs/getting-started/index.md b/docs/_docs/getting-started/index.md index 2dcd29c..cf4ce1f 100644 --- a/docs/_docs/getting-started/index.md +++ b/docs/_docs/getting-started/index.md @@ -172,7 +172,7 @@ The configuration commands will vary based on the kind of task you want to add, and here is a quick example of adding a task to watch a url (the default task): ```bash -$ watchme add watcher task-singularity-release url@https://github.com/sylabs/singularity/releases +$ watchme add-task watcher task-singularity-release url@https://github.com/sylabs/singularity/releases [task-singularity-release] url = https://github.com/sylabs/singularity/releases active = true @@ -207,7 +207,7 @@ The task is active by default (after you set up its schedule) and you can disabl this with --active false: ```bash -$ watchme add watcher task-singularity-release url@https://github.com/sylabs/singularity/releases --active false +$ watchme add-task watcher task-singularity-release url@https://github.com/sylabs/singularity/releases --active false ``` The reason we save these parameters in the repo is that if you put it under version diff --git a/watchme/client/__init__.py b/watchme/client/__init__.py index f98aa68..7279ec2 100644 --- a/watchme/client/__init__.py +++ b/watchme/client/__init__.py @@ -14,6 +14,7 @@ from watchme.defaults import ( WATCHME_WATCHER, WATCHME_TASK_TYPES, + WATCHME_EXPORTERS, WATCHME_DEFAULT_TYPE ) import watchme @@ -132,33 +133,53 @@ def get_parser(): create.add_argument('watchers', nargs="*", help='watchers to create (default: single watcher)') - create.add_argument('--exporter', dest="exporter", - help="an exporter to send the data", - default=None) + # add task - # add + add_task = subparsers.add_parser("add-task", + help="add a task to a watcher.") - add = subparsers.add_parser("add", - help="add a task to a watcher.") + add_task.add_argument('watcher', nargs=1, + help='the watcher to add to') - add.add_argument('watcher', nargs=1, - help='the watcher to add to') + add_task.add_argument('task', nargs=1, + help='the name of the task to add.') - add.add_argument('task', nargs=1, - help='the name of the task to add. Must start with task') + add_task.add_argument('--type', dest="watcher_type", + choices=WATCHME_TASK_TYPES, + default=WATCHME_DEFAULT_TYPE) - add.add_argument('--type', dest="watcher_type", - choices=WATCHME_TASK_TYPES, - default=WATCHME_DEFAULT_TYPE) + add_task.add_argument('--active', dest="active", + choices=["true", "false"], + default="true") - add.add_argument('--active', dest="active", - choices=["true", "false"], - default="true") + add_task.add_argument('--force', dest="force", + help="force overwrite a task, if already exists.", + default=False, action='store_true') - add.add_argument('--force', dest="force", - help="force overwrite a task, if already exists.", - default=False, action='store_true') + + # add exporter + + add_exporter = subparsers.add_parser("add-exporter", + help="add an exporter to a watcher.") + + add_exporter.add_argument('watcher', nargs=1, + help='the watcher to add to') + + add_exporter.add_argument('name', nargs=1, + help='the name of the exporter to add.') + + add_exporter.add_argument('--type', dest="exporter_type", + choices=WATCHME_EXPORTERS, + default=WATCHME_EXPORTERS[0]) + + add_exporter.add_argument('--active', dest="active", + choices=["true", "false"], + default="true") + + add_exporter.add_argument('--force', dest="force", + help="force overwrite an exporter, if already exists.", + default=False, action='store_true') # inspect @@ -332,7 +353,8 @@ def help(return_code=0): sys.exit(0) if args.command == "activate": from .activate import main - elif args.command == "add": from .add import main + elif args.command == "add-task": from .addtask import main + elif args.command == "add-exporter": from .exporter import main elif args.command == "edit": from .edit import main elif args.command == "export": from .export import main elif args.command == "create": from .create import main diff --git a/watchme/client/activate.py b/watchme/client/activate.py index 67cab12..ed19687 100644 --- a/watchme/client/activate.py +++ b/watchme/client/activate.py @@ -12,7 +12,7 @@ from watchme.logger import bot def main(args, extra): - '''activate one or more watchers + '''activate one or more watchers, tasks, or exporters ''' # Doesn't work if watcher not provided watcher = args.watcher[0] @@ -22,6 +22,7 @@ def main(args, extra): if extra == None: watcher.activate() else: + # This can be used for tasks and exporters for name in extra: watcher.activate(name) diff --git a/watchme/client/add.py b/watchme/client/addtask.py similarity index 92% rename from watchme/client/add.py rename to watchme/client/addtask.py index 0199534..6edd619 100644 --- a/watchme/client/add.py +++ b/watchme/client/addtask.py @@ -12,14 +12,14 @@ from watchme.logger import bot def main(args, extra): - '''activate one or more watchers + '''add a task for a watcher ''' # Required - will print help if not provided name = args.watcher[0] task = args.task[0] if not task.startswith('task'): - example = 'watchme add watcher task-cpu func@cpu_task type@psutils' + example = 'watchme add-task watcher task-cpu func@cpu_task type@psutils' bot.exit('Task name must start with "task", e.g., %s' % example) # Exit if the user doesn't provide any parameters diff --git a/watchme/client/create.py b/watchme/client/create.py index eb92018..cf7e73b 100644 --- a/watchme/client/create.py +++ b/watchme/client/create.py @@ -19,5 +19,5 @@ def main(args, extra): watchers = ['watcher'] for watcher in watchers: - create_watcher(watcher, exporter=args.exporter) + create_watcher(watcher) diff --git a/watchme/client/deactivate.py b/watchme/client/deactivate.py index e683bb1..221c4dc 100644 --- a/watchme/client/deactivate.py +++ b/watchme/client/deactivate.py @@ -22,6 +22,7 @@ def main(args, extra): if extra == None: watcher.deactivate() else: + # This can be used for a task or an exporter for name in extra: watcher.deactivate(name) diff --git a/watchme/client/exporter.py b/watchme/client/exporter.py new file mode 100644 index 0000000..95fe7b2 --- /dev/null +++ b/watchme/client/exporter.py @@ -0,0 +1,60 @@ +''' + +Copyright (C) 2019 Vanessa Sochat. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' + +from watchme import get_watcher +from watchme.logger import bot +from watchme.defaults import WATCHME_EXPORTERS + +def main(args, extra): + '''add an exporter to a watcher, optionally with params and tasks + ''' + + # Required - will print help if not provided + name = args.watcher[0] + exporter = args.name[0] + + # Are we adding an extra task or exporter? + if not exporter.startswith('exporter'): + bot.error('Exporter name must start with exporter-') + bot.exit('watchme add-exporter watcher exporter-pushgateway task1 task2 task3') + + # Extra parameters are parameters for the exporter, or task names + if extra == None: + extra = [] + + # Type can also be an argument (required for exporter) + exporter_type = args.exporter_type + + # Go through extras and determine params and tasks + params = [] + tasks = [] + for param in extra: + if param.startswith('type@'): + exporter_type = param.replace('type@', '') + elif param.startswith('task'): + tasks.append(param) + else: + params.append(param) + + # Get the watcher to interact with, must already exist + watcher = get_watcher(name, base=args.base, create=False) + + # Double check the exporter type + if exporter_type not in WATCHME_EXPORTERS + choices = ','.join(WATCHME_EXPORTERS) + bot.exit("%s is not a valid exporter, %s" %(exporter_type, choices)) + + # Add the exporter, optionally with tasks + watcher.add_exporter(name=exporter, + exporter_type=exporter_type, + params=params, + tasks=tasks, + force=args.force, + active=args.active) diff --git a/watchme/defaults.py b/watchme/defaults.py index ad347e5..23fc0f5 100644 --- a/watchme/defaults.py +++ b/watchme/defaults.py @@ -57,6 +57,7 @@ def getenv(variable_key, default=None, required=False, silent=True): # The types of valid watchers (currently only urls). Corresponds with # a folder under "main/watchers" +WATCHME_EXPORTERS = ['pushgateway'] WATCHME_TASK_TYPES = ['urls', 'url', 'psutils'] WATCHME_DEFAULT_TYPE = "urls" diff --git a/watchme/exporters/__init__.py b/watchme/exporters/__init__.py index c1c72b0..6fb4493 100644 --- a/watchme/exporters/__init__.py +++ b/watchme/exporters/__init__.py @@ -40,6 +40,16 @@ def set_params(self, params): key = key.lower() self.params[key] = value + + def export_params(self, active="true"): + '''export parameters, meaning returning a dictionary of the task + parameters plus the addition of the task type and active status. + ''' + params = self.params.copy() + params['active'] = active + params['type'] = self.type + return params + # Validation # For now, it should always be valid as no required parameters are defined. diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py index aca83ca..5bfe2e0 100644 --- a/watchme/exporters/pushgateway/__init__.py +++ b/watchme/exporters/pushgateway/__init__.py @@ -23,7 +23,8 @@ def __init__(self, name, params={}, **kwargs): from prometheus_client import CollectorRegistry except: bot.error("prometheus_client module not found.") - bot.exit("pip install watchme[exporter-pushgateway]") + bot.error("pip install watchme[exporter-pushgateway]") + return self.type = 'pushgateway' self.registry = CollectorRegistry() @@ -50,6 +51,7 @@ def _save_text(self, result): ''' self._write_to_pushgateway(result) + def _write_to_pushgateway(self, result): ''' writes data to the pushgateway @@ -62,6 +64,11 @@ def _write_to_pushgateway(self, result): g.set(result) try: - push_to_gateway(self.params['url'], job='watchme', registry=self.registry) + push_to_gateway(self.params['url'], + job='watchme', + registry=self.registry) except: bot.error('An exception occurred while trying to export data using %s' % self.name) + + #TODO: disable task, and add a --test command + #TODO: need commands to add/remove exporters from a task diff --git a/watchme/watchers/README.md b/watchme/watchers/README.md index 5ed474a..4229105 100644 --- a/watchme/watchers/README.md +++ b/watchme/watchers/README.md @@ -6,3 +6,11 @@ Each of these is a Watcher that the user can request. - [psutils](psutils) to get basic system statistics More watchers will be added as the library is developed. + +## Watcher Base + +The watcher base is defined in the [init](__init__.py) file here. + + - [schedules](schedule.py) to interact with cronjobs + - [exporters](data.py) (extras) and the default export of temporal data + - [settings](settings.py) meaning add/remove from the watchme.cfg diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index ba4474d..603416c 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -27,12 +27,18 @@ from configparser import NoOptionError from .data import ( - export_dict + export_dict, + export_runs, + get_exporter, + add_exporter, + _add_exporter ) from .settings import ( get_setting, set_setting, + has_setting, + has_section, get_section, print_section, print_add_task, @@ -239,8 +245,7 @@ def add_task(self, task, task_type, params, force=False, active="true"): Parameters ========== - task: the Task object to add, should have a name and params and - be child of watchme.tasks.TaskBase + task: the task name to add, must start with task- task_type: must be in WATCHME_TASK_TYPES, meaning a client exists params: list of parameters to be validated (key@value) force: if task already exists, overwrite @@ -279,6 +284,7 @@ def add_task(self, task, task_type, params, force=False, active="true"): # Write to file (all tasks get active = True added, and type) self._add_task(newtask, force, active) + def _add_task(self, task, force=False, active='true'): '''add a new task to the watcher, meaning we: @@ -634,95 +640,6 @@ def get_tasks(self, regexp=None): bot.info('Found %s contender tasks.' % len(tasks)) return tasks -# Get Exporters - - def get_exporter(self, name, save=False): - '''get a particular exporter, based on the name. We could have - an exporter type (similarly to Tasks), but it is quite useless right now. - Parameters - ========== - name: the name of the exporter to load - save: if saving, will be True - ''' - self.load_config() - - exporter = None - - # Only sections that start with exporter- are considered exporters - if name in self.config._sections and name.startswith('exporter'): - - print("Elligible exporter " + name) - - params = self.config._sections[name] - - # Instantiate the correct exporter given the type - exporter_type = params['type'] - - if exporter_type == 'pushgateway': - from watchme.exporters.pushgateway import Exporter - - else: - bot.exit('exporter type %s does not exist' % exporter_type) - - exporter = Exporter(name, params) - - return exporter - - def get_exporters(self, regexp=None): - '''get the exporters for a watcher, possibly matching a regular expression. - A list of dictionaries is returned, each holding the parameters for - an exporter. An exporter has an active attribute. - - Parameters - ========== - regexp: if supplied, the user wants to export to destinations - that only match the expression specified. - ''' - self.load_config() - - exporters = [] - - for section in self.config._sections: - - # Get the exporter based on the section name - exporter = self.get_exporter(section) - - # Check that the exporter should be used, and is valid - if exporter != None: - if self._exporter_selected(exporter, regexp) and exporter.valid: - exporters.append(exporter) - - bot.info('Found %s contender exporters.' % len(exporters)) - return exporters - - def _exporter_selected(self, exporter, regexp=None): - '''check if an exporter is active and (if defined) passes user provided - exporter names or regular expressions. - - Parameters - ========== - exporter: the exporter object to check - regexp: an optional regular expression (or name) to check - ''' - selected = True - - # A exporter can be None if it wasn't found - if exporter == None: - selected = False - - # Is the exporter not active (undefined is active)? - active = exporter.params.get('active', 'true') - if active == "false": - bot.info('Exporter %s is not active.' % exporter) - selected = False - - # The user wants to search for a custom task name - if regexp != None: - if not re.search(regexp, exporter): - bot.info('Exporter %s is selected for data export.' % exporter) - selected = False - - return selected # Running Tasks @@ -819,52 +736,18 @@ def run(self, regexp=None, parallel=True, test=False, show_progress=True): # Step 2: get the tasks associated with the run, a list of param dicts tasks = self.get_tasks() - # Step 3 : get the exporters if any were declared in the watcher configuration. - exporters = self.get_exporters() - - # Step 4: Run the tasks. This means preparing a list of funcs/params, + # Step 3: Run the tasks. This means preparing a list of funcs/params, # and then submitting with multiprocessing results = self.run_tasks(tasks, parallel, show_progress) # Finally, finish the runs. if test is False: self.finish_runs(results) - self.export_runs(results, exporters) + self.export_runs(results) else: # or print results to the screen print(json.dumps(results, indent=4)) - def export_runs(self, results, exporters): - ''' export data retrieved to the set of exporters defined and active. - maybe an export flag could be set to choose to run + export? - ''' - for name, result in results.items(): - - task = self.get_task(name, save=True) - - # Case 1. The result is a list - if isinstance(result, list): - - # Get rid of Nones, if the user accidentally added - result = [r for r in result if r] - - if len(result) == 0: - bot.error('%s returned empty list of results.' % name) - - # for a json, or a list of paths, ignore for now. - elif not(task.params.get('save_as') == 'json' or os.path.exists(result[0])): - for exporter in exporters: - bot.debug('Exporting list to ' + exporter.name) - exporter._save_text_list(name, result) - - # Case 2. The result is a string - elif isinstance(result, str): - - # if it's a path to a file, ignore it. - if not(os.path.exists(result)): - exporter._save_text(result) - - # Case 3. The result is a dictionary, ignore it for now. def finish_runs(self, results): '''finish runs should take a dictionary of results, with keys as the @@ -961,6 +844,8 @@ def __str__(self): Watcher.remove_setting = remove_setting Watcher.get_setting = get_setting Watcher.get_section = get_section +Watcher.has_setting = has_setting +Watcher.has_section = has_section Watcher.set_setting = set_setting Watcher.remove_section = remove_section Watcher.print_section = print_section @@ -976,6 +861,9 @@ def __str__(self): Watcher.clear_schedule = clear_schedule Watcher.schedule = schedule -# Data - +# Exporters +Watcher.add_exporter = add_exporter +Watcher.get_exporter = get_exporter +Watcher._add_exporter = _add_exporter Watcher.export_dict = export_dict +Watcher.export_runs = export_runs diff --git a/watchme/watchers/data.py b/watchme/watchers/data.py index fb7b262..cb3864d 100644 --- a/watchme/watchers/data.py +++ b/watchme/watchers/data.py @@ -21,8 +21,7 @@ import os import json -# Data Exports - +# Default (git) Data Exports def export_dict(self, task, filename, @@ -86,3 +85,213 @@ def export_dict(self, task, result['dates'].append(git_date(repo=repo, commit=commit)) result['commits'].append(commit) return result + + +# Exporter Functions + + # Add the exporter, optionally with tasks + watcher.add_exporter(exporter=exporter, + exporter_type=exporter_type, + params=params, + tasks=tasks, + force=args.force, + active=args.active) + + +def add_exporter(self, name, exporter_type, params, tasks, force=False, active="true"): + '''add an exporter, meaning an extra plugin to push data to a remote. + + Parameters + ========== + name: the name of the exporter to add, should start with exporter- + exporter_type: must be in WATCHME_EXPORTERS, meaning a client exists + params: list of parameters to be validated (key@value) + tasks: list of task names to add the exporter to + force: if task already exists, overwrite + active: add the task as active (default "true") + ''' + # Check again, in case user calling from client + if not name.startswith('exporter'): + bot.exit('Exporter name must start with "exporter" (e.g., exporter-grafana)') + + # Ensure it's a valid type + if exporter_type not in WATCHME_EXPORTERS: + bot.exit('%s is not a valid type: %s' % WATCHME_EXPORTERS) + + # Validate variables provided for task + if task_type.startswith('pushgateway'): + from watchme.exporters.pushgateway import Exporter + + else: + bot.exit('exporter_type %s not installed' % exporter_type) + + # Convert list to dictionary + params = self._get_params_dict(params) + + # Creating the exporter will validate parameters + exporter = Exporter(name, params=params) + + # Exit if the exporter is not valid + if not exporter.valid: + bot.exit('%s is not valid, will not be added.' % exporter) + + # Write to file (all tasks get active = True added, and type) + self._add_exporter(exporter, force, active, tasks) + + + +def _add_exporter(self, exporter, force=False, active='true', tasks=[]): + '''add a new exporter to the watcher, meaning we: + + 1. Check first that the task doesn't already exist (if the task + exists, we only add if force is set to true) + 2. Validate the task (depends on the task) + 3. write the task to the helper config file, if valid. + + Parameters + ========== + task: the Task object to add, should have a name and params and + be child of watchme.tasks.TaskBase + force: if task already exists, overwrite + active: add the task as active (default "true") + ''' + self.load_config() + + if active not in ["true", "false"]: + bot.exit('Active must be "true" or "false"') + + # Don't overwrite a section that already exists + if exporter.name in self.config.sections(): + if not force: + bot.exit('%s exists, use --force to overwrite.' % task.name) + self.remove_section(exporter.name, save=False) + + # Add the new section + self.config[exporter.name] = exporter.export_params(active=active) + self.print_section(exporter.name) + + # Add the exporter to any tasks + for task in tasks: + if task in self.config.sections(): + exporters = self.get_setting(task, "exporters", default=[]) + if exporter.name not in exporters: + exporters.append(exporter.name) + self.set_setting(task, "exporters", exporters) + else: + bot.warning("Task %s not found installed, skipping adding exporter to it." % task) + + # Save all changes + self.save() + + # Commit changes + git_commit(repo=self.repo, task=self.name, + message="ADD exporter %s" % exporter.name) + +# Get Exporters + + +def get_exporter(self, name): + '''get a particular task, based on the name. This is where each type + of class should check the "type" parameter from the config, and + import the correct Task class. + + Parameters + ========== + name: the name of the task to load + save: if saving, will be True + ''' + self.load_config() + + exporter = None + + # Only sections that start with task- are considered tasks + if name in self.config._sections and name.startswith('exporter'): + + # Task is an ordered dict, key value pairs are entries + params = self.config._sections[name] + + # Get the task type (if removed, consider disabled) + exporter_type = params.get('type', '') + + # If we get here, validate and prepare the task + if exporter_type.startswith("pushgateway"): + from watchme.exporters.pushgateway import Exporter + + else: + bot.exit('%s is not a valid exporter type.' % exporter_type) + + # if not valid, will return None (or exit) + exporter = Exporter(name, params) + + return exporter + + + +def export_runs(self, results): + ''' export data retrieved to the set of exporters defined and active, + which can vary based on the task. The export will be run if: + + 1. The task has a list of one or more exporters to use. + 2. Any given exporter is defined also in the watchme.cfg + 3. The exporter is active. + + If the user does not wish to export data for a specific task, + the exporter can be turned off (active set to false), the exporter + name can be removed from "exporters" or the exporter- configuration + can be removed from the file entirely. + ''' + for name, result in results.items(): + + task = self.get_task(name, save=True) + + # Get exporters added to task + exporters = self.get_setting(task.name, 'exporters', []) + + # Validate each exporter exists and is active, then run. + for exporter in exporters: + + # Check 1: The exporter must be defined + if not self.has_section(exporter): + bot.warning("%s not defined in watchme.cfg for %s" %(exporter, task)) + continue + + # Check 2: It must be active + # Be more conservative to run an export, default to false + if self.get_setting(exporter, "active", default="false") == "false": + bot.warning("%s is defined but not active for %s" %(exporter, task)) + continue + + # If we get here, safe to instantiate the exporter. + try: + client = self.get_exporter(exporter) + except: + bot.warning("Check parameters for %s, client did not validate." % exporter) + continue + + # A client with "None" indicates a dependency is likely missing + if client == None: + bot.warning("Check dependencies for %s.." % exporter) + continue + + # Case 1. The result is a list + if isinstance(result, list): + + # Get rid of Nones, if the user accidentally added + result = [r for r in result if r] + + if len(result) == 0: + bot.error('%s returned empty list of results.' % name) + + # Only save if the export type is not json, and the result is a text string + elif not task.params.get('save_as') == 'json' and not os.path.exists(result[0])): + bot.debug('Exporting list to ' + client.name) + client._save_text_list(name, result) + + # Case 2. The result is a string + elif isinstance(result, str): + + # Only export if it's not a file path (so it's a string) + if not(os.path.exists(result)): + client._save_text(result) + + # Case 3. The result is a dictionary or a file, ignore for now. diff --git a/watchme/watchers/settings.py b/watchme/watchers/settings.py index 35d4c1e..7e4747d 100644 --- a/watchme/watchers/settings.py +++ b/watchme/watchers/settings.py @@ -97,8 +97,8 @@ def print_section(self, section): bot.exit('%s is not a valid section.' % section) def get_setting(self, section, name, default=None): - '''return a setting from the environment (first priority) and then - secrets (second priority) if one can be found. If not, return None. + '''return a setting from the config, if defined. Otherwise return + default (None or set by user) Parameters ========== @@ -119,6 +119,34 @@ def get_setting(self, section, name, default=None): return setting +def has_setting(self, section, name): + '''return a boolean if a config has a setting (or not) + + Parameters + ========== + section: the section in the config, defaults to self.name + name: they key (index) of the setting to look up + ''' + self.load_config() + + exists = False + if section in self.config: + if name.lower() in self.config[section]: + exists = True + + return exists + + +def has_section(self, section): + '''return a boolean if a config has a section (e.g., a task or exporter) + + Parameters + ========== + section: the section in the config + ''' + self.load_config() + return section in self.config + def set_setting(self, section, key, value): '''set a key value pair in a section, if the section exists. Returns @@ -145,6 +173,6 @@ def get_section(self, name): ''' section = None self.load_config() - if name in self.config.sections(): + if name in self.config: section = self.config[name] return section From 67f522113bf1d5e80b250948489c61e3a90a21d6 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Mon, 6 May 2019 16:06:53 -0400 Subject: [PATCH 14/19] tweaks in setup.py (still not tested) Signed-off-by: Vanessa Sochat --- MANIFEST.in | 4 ++-- setup.py | 2 +- watchme/client/exporter.py | 2 +- watchme/version.py | 8 ++++---- watchme/watchers/data.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index a92bced..574fc21 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,5 +3,5 @@ recursive-include watchme * recursive-exclude * __pycache__ recursive-exclude * *.pyc recursive-exclude * *.pyo -recursive-exclude .docs -recursive-exclude docs +recursive-exclude .docs * +recursive-exclude docs * diff --git a/setup.py b/setup.py index 8013243..8c726bb 100644 --- a/setup.py +++ b/setup.py @@ -111,7 +111,7 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): 'watchers': [WATCHERS], 'exporters': [EXPORTERS], 'watcher-urls-dynamic': [URLS_DYNAMIC], - 'watcher-psutils': [PSUTILS] + 'watcher-psutils': [PSUTILS], 'exporter-pushgateway': [PUSHGATEWAY], }, classifiers=[ diff --git a/watchme/client/exporter.py b/watchme/client/exporter.py index 95fe7b2..e62bbb1 100644 --- a/watchme/client/exporter.py +++ b/watchme/client/exporter.py @@ -47,7 +47,7 @@ def main(args, extra): watcher = get_watcher(name, base=args.base, create=False) # Double check the exporter type - if exporter_type not in WATCHME_EXPORTERS + if exporter_type not in WATCHME_EXPORTERS: choices = ','.join(WATCHME_EXPORTERS) bot.exit("%s is not a valid exporter, %s" %(exporter_type, choices)) diff --git a/watchme/version.py b/watchme/version.py index a7ce3be..923b3e4 100644 --- a/watchme/version.py +++ b/watchme/version.py @@ -11,8 +11,8 @@ AUTHOR_EMAIL = 'vsochat@stanford.edu' NAME = 'watchme' PACKAGE_URL = "http://www.github.com/vsoch/watchme" -KEYWORDS = 'web, changes, cron' -DESCRIPTION = "client to watch for webpage changes, and track over time" +KEYWORDS = 'web, changes, cron, reproducible, version-control' +DESCRIPTION = "reproducible monitoring client with exporters" LICENSE = "LICENSE" INSTALL_REQUIRES = ( @@ -31,14 +31,14 @@ ('psutil', {'min_version': '5.4.3'}), ) -INSTALL_PUSHGATEWAY ( +INSTALL_PUSHGATEWAY = ( ('prometheus_client', {'min_version': '0.6.0'}), ) # Install all watchers and exporters INSTALL_ALL = (INSTALL_REQUIRES + INSTALL_PSUTILS + - INSTALL_URLS_DYNAMIC, + INSTALL_URLS_DYNAMIC + INSTALL_PUSHGATEWAY) # Install all watchers diff --git a/watchme/watchers/data.py b/watchme/watchers/data.py index cb3864d..53921ce 100644 --- a/watchme/watchers/data.py +++ b/watchme/watchers/data.py @@ -283,7 +283,7 @@ def export_runs(self, results): bot.error('%s returned empty list of results.' % name) # Only save if the export type is not json, and the result is a text string - elif not task.params.get('save_as') == 'json' and not os.path.exists(result[0])): + elif not task.params.get('save_as') == 'json' and not os.path.exists(result[0]): bot.debug('Exporting list to ' + client.name) client._save_text_list(name, result) From 0b47358bd614ccae4d9105334549c434c6a712c8 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Mon, 6 May 2019 16:13:17 -0400 Subject: [PATCH 15/19] config directory does not have templates subfolder Signed-off-by: Vanessa Sochat --- watchme/config/{templates => }/watchme.cfg | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename watchme/config/{templates => }/watchme.cfg (100%) diff --git a/watchme/config/templates/watchme.cfg b/watchme/config/watchme.cfg similarity index 100% rename from watchme/config/templates/watchme.cfg rename to watchme/config/watchme.cfg From a1b1bbde973864b2c8cd18761cc9ccca6a68fbee Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Tue, 7 May 2019 14:51:06 -0400 Subject: [PATCH 16/19] adding bulk of new work for exporters, need to write push function and docs Signed-off-by: Vanessa Sochat --- docs/_data/links.yml | 2 + docs/_docs/exporters/index.md | 83 +++++- docs/_docs/exporters/pushgateway.md | 18 -- docs/_docs/exporters/pushgateway/pushed.png | Bin 0 -> 45208 bytes .../exporters/pushgateway/pushgateway.md | 243 ++++++++++++++++++ .../exporters/pushgateway/pushgateway.png | Bin 0 -> 10724 bytes setup.py | 23 +- watchme/client/__init__.py | 10 +- watchme/client/create.py | 1 - watchme/client/exporter.py | 9 +- watchme/client/ls.py | 25 +- watchme/client/push.py | 43 ++++ watchme/client/remove.py | 26 +- watchme/command/__init__.py | 4 +- watchme/command/utils.py | 20 +- watchme/exporters/__init__.py | 6 + watchme/exporters/pushgateway/__init__.py | 12 +- watchme/watchers/__init__.py | 26 +- watchme/watchers/data.py | 168 +++++++++--- watchme/watchers/urls/helpers.py | 1 + 20 files changed, 621 insertions(+), 99 deletions(-) delete mode 100644 docs/_docs/exporters/pushgateway.md create mode 100644 docs/_docs/exporters/pushgateway/pushed.png create mode 100644 docs/_docs/exporters/pushgateway/pushgateway.md create mode 100644 docs/_docs/exporters/pushgateway/pushgateway.png create mode 100644 watchme/client/push.py diff --git a/docs/_data/links.yml b/docs/_data/links.yml index 83b30a5..2b618d4 100644 --- a/docs/_data/links.yml +++ b/docs/_data/links.yml @@ -12,5 +12,7 @@ navigation: url: watchers/ - name: Examples url: examples/ +- name: Exporters + url: exporters/ - name: Contributing url: contributing/ diff --git a/docs/_docs/exporters/index.md b/docs/_docs/exporters/index.md index 5a6bfa1..ff56aff 100644 --- a/docs/_docs/exporters/index.md +++ b/docs/_docs/exporters/index.md @@ -16,5 +16,86 @@ are available: And instructions for using a general exporter, either in sync with running a task or separately, are provided below. -#TODO write instructions +## List Exporters + +To see exporters available, you can list them: + +```bash +$ watchme list --exporters +watchme: exporters +pushgateway +``` + +An exporter is like a task in that it can be activated +(or not), and you can define more than one for any particular watcher. + +## Add an Exporter + +The most basic thing to do is to add an exporter to a watcher. You should +read the documentation for your exporter of interest (linked above) to +know the required parameters for the exporter (it varies). +Adding an exporter is similar to adding a task! The command asks you to +name the exporter, and the watcher to which you are adding it: + +```bash +$ watchme add-exporter +``` + +Most likely the exporter requires parameters, so you add these in the same way +you added parameters to a task. Notice that "type" is essential to designate the +type that you want (the list at the top of the page): + +```bash +$ watchme add-exporter exporter- param@value type@ +``` + +If you already have tasks defined that you want the exporter added to, just add them to the command: + +```bash +$ watchme add-exporter exporter- param@value type@ +``` + +If you've already added the exporter to the watcher and want to add it to a task: + +```bash +$ watchme add-exporter +``` + +Watchme will know that you aren't adding a new exporter (but instead using +an existing one) because you haven't provided a `type@` +variable. If you want to re-create an exporter that already exists, you +can issue the same command but you'll need `--force`. + +```bash +$ watchme add-exporter exporter- param@value type@ --force +``` + +## Remove an Exporter + +If you change your mind, you can remove an exporter from a watcher entirely: + +```bash +$ watchme remove +``` + +You can also remove it from a specific task (but not from the watcher entirely) + +```bash +$ watchme remove +``` + + +## Run the Exporter + +By default, an exporter will run with a task. This means that you should +run the task to run the exporter. If the watcher is active, and the task +is active, and the exporter is defined for the task, it will run. + +```bash +$ watchme activate +$ watchme run +``` + +If an exporter doesn't run successfully, it will be disabled for the task. +This is in case you forget about it. diff --git a/docs/_docs/exporters/pushgateway.md b/docs/_docs/exporters/pushgateway.md deleted file mode 100644 index 615d981..0000000 --- a/docs/_docs/exporters/pushgateway.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: Pushgateway -category: Exporters -permalink: /exporters/pushgateway/ -order: 2 ---- - -The [pushgateway](https://github.com/prometheus/pushgateway) exporter -will allow you to export data for a watcher to a Prometheus gateway. -To install the exporter, you will need to install its dependencies: - -```bash -pip install watchme[exporter-pushgateway] -``` - -The dependencies include the [prometheus client](https://github.com/prometheus/client_python). - -**under development** diff --git a/docs/_docs/exporters/pushgateway/pushed.png b/docs/_docs/exporters/pushgateway/pushed.png new file mode 100644 index 0000000000000000000000000000000000000000..5692063d57b1d7e5e72320d219642918d5bbe43c GIT binary patch literal 45208 zcmdSARa9I}(=Zw%Mv%dsFt|f-4`FZ%?hxGFJt4>p8r&hlo!~M^aCZxCgUbMe+Z>)Z zdA|4i&st}#b8&9YzM0wGRbAEHRbAE_@=-w&?KRQsM~@z%NlSs0A3Z`zee~#Y!pkQQ zEz!;Wh=-r2&L5;zU%q@fzpA+M@Ji?+uIZv;Z|34|=w$lH+|J(Cl-b$X$<)-&*}~oh z{=0Kjn-r~sI;sc6ezIaxg4hb=bT`0@V&yS|lNPjJ2odISPSU%c!1pRrL877HJmD#GVkv;Lqz8r}Go z{g?-Xh5HZYHbau+`m__2#$ust3*}GSA_^ao17gMK>3?27mj#fZ{ponL?fHL023!0Z z`Wp7PyYcnXk{2of0!k9a{)6zH%#_S8?h^V9oI zj@B8zmZkeXibMRkahkS+f(!V(!OB$`3!&dDn8lxA-fQd(3KE?=OAc+3qRqcJ-KEqB z+V#thfnRSMhzQ);TO5>Wz@yo#jIQCxBN371X$myaqn80vuo(tiaZJG{dKJ|ZrmtNX zH)=g}1>r6YLQkT5M?~Haa&A0-WtZ5)`l~g&=rvhl$dlXSP^15l{5=2Bd~>Ye>bTXQ zwl{m*d&^i6R17>MzIK+LIzw(BE~;SK{ld6tmV84>pef0dwXGvrPACv*r%2!c+>1Ug zgp$gKg$UlTuBWn}p(sA1#pRz{bf3(w>KKG?AV;klc7tNzTk%5gRxz%Ll{c&pG3vt5 zAd!v!ag~{Y3vTn4*)1j)BWb+vL9pH2m!w7_Ru%3)>yV$nMf>$cY9K4M|7`BB((OjS zma;p`u@W6SVvn3o%))I+e&b3VLsME6h_FekeP?-EiQM#Vz=VAmgPf7*7 zd0*V)<}ytsm3yUWNoa9nrC)2H|53TQ5Pv9pqOaCzW>(Bob8l<1bS|zq0JvAg#mNXJ zB^?5NA)j*YAl0hF5%E{2{eBd~EUIHAS7Cd4pvR9N2fTgbb2J--y>DQGoY$QBseZq5 zOM6&&wU+Kr?0Ld_hjT#-iXavK*&4WOJ)FYWa&-cYW$_mQioOMYc_OKhBJ5b=^odnN zeoEyd`<5DMZ;%t&kb3Ha$@+HnXyCB$MSo0U0=%lG;aN#Y9ijM^VgqSHJ2`30+} z)!};UDYZfT{4Ih1Y_p;W;SO@~1$p9BC>VTg;eWFx)=}Kje}5C@3Icste&viOagu3| zoKe(;O5)IGZHWjko!JgcUZ>=}j27hIfIy3c2ktYi2`<-255{KM$$`KkR#pM5>-Qg$ zEDXeI#gX){F4W`6$-ZZ8^0aZg>vv%ND0CJFM%%9iZjWc|i|pr}U)V&OIWm7$X+l)% z8%GAWTIsS8irmfERfT}JJC$OG;5&u<@&*%M5N`P)&n`^MBH-5xYS-d2}AL)Hw1#5I!E!w#y(*lj3-h`m$qd`TGJ{LcVey!Z(Be|`l5K0@^@@Rz zpa*@Qh22+=FE(>1Rv=_c>jG^S$xu4C?Owd|{9mk5C!tBY%-mF9>;NruD zTq@kRL@haXNG!1(yVWrl8yaaXP?$22)B0e1N$e{w{2cjFeInA*~qu zrSzk6lPQA2ShC`b)=$_md+WGh?(Qmjls;CBSol0ea9tmP4=vtJ5?8>iIK_q0mIQq1dL|` zLPE|-k*6{FuD-;f2gTO6FhE~>o-pDjF@ee8!syoMH5c78``A_jU`3i z8O%4Yeva&d<~Z1k*hL@5OQvN3E<_5@d7*GP^0)VrJm>`mnZJvATFO2s z;O%A9K>H#ZAA^nxi)5%gc4 z1V_eA%)%9Pv2OYKvk(i4G^q+aR&l5Em0^A$P)GMt+jJ(~G508%Yt9h@o0UpVz1go* zZqnN6wplF`xgqttDY#}`f3D(A`r2c*`buOnWcbEeV(C03#P4qD_En)6gL+CVO^%&qlp7+4CAoiWOyoF4+!2B|oq@eTh2wYHlb_C>a zL-t)KIpg?{{LDDh-oG-EQSU*62da@PLJ*aVySwzBND@9DSm3OUko#O4_)a`g82OWq z?W#oa$BO^Ya-Jj&8qB5R}cxb0nc0iZm8hRLThWjMOM^2KE} z)rk>qb&0ZMTgs+KP!2UVS+4v#ovApz%;%G{;9MfZ$tH#C@gu9+%Leun@H9|mD$9_f z~RT;lWM^}v@$HK`*ZMdV5(gcWspId2Zi6sx510y^OU5cUtNpR zlDot-)7xD1e0=-8*6WFP6LO%90lZ*&7U>M%8E@;MGxQ=_$>y&G_yDCVk(pa+wSLrYUPSg%(8+jY$aeAO zVj?}6W-yk?x1L2g9PYc*9;VN= zl8;gIG?c|StuDKN4&1dhFEwW!4=NW~wA?jrCDi-ZO(O0c$HSZ^j0abHsEu zO>E8ZViQXAie-8>#mj)VzZqNZ?T?Hx0u_=0{9Rpoyr#msexD42rby||5%Ao@Pt-zv#cdfG- zvg3($B$*KHOP3nTIdpP9Vm@}0$eX6cyNAf;w}R*-}VW^vd6%-sji**_Af2Cr`ug{Q_3(cymTjrq?b5 zR`^@sH*-B?SBtW-ENR^2rEJ?6dM=?|Nz#A(xbL*va*+~4Q3n3{`twV3jG4%1$jvQ@ z8NLqMc{#Dk-H)zEz25=GlmXpK0zY8cK4S76Vbc6eH5;jlDY9p5yAS3txDtInId)5d z^vV-Szqb)xp#rySEl{~ZX4hTfY#n>_>Ul?kFJmP~>WzZY64}r<-p@4LP0t~wQ919? z8<1B3nQ6b1-UbEIigWPsY3Blipcw z=J|{88Lx`9`oV)YEZHb&yMoe|5vHc4JoE zmy>OX%BEOI3wDrvloj+>_l^!oo_164%5_*k+NtQlG~Hh zcvGHXr?)R7!xDLkXiUY)+9r;Cb`CiT4A!l#?l6KYxkB9T55n@Oz8L=a0y^&=58TYs zbFFd9(!WKSs7UF?*YW`Qs&!DEHOXQ=tIXs~&xKINGxYrYT(z9$^2(J~!HM7Xqts8U z_symkgo$r4_0GU9I}Tkty?yS1=)8*}yx#p8S@*nvH8{-&e@`#guc}Wt$wwA?Z|9q%II>X4K6Cf8()H z_+*j9bG33%&exFUEHmR~UEo#YJ)E>~$3?!JS+axLC?uq?hqQoBnD*oX8}ckl=g9b0 zFLEFV-R_i*L*s%U3~>GC zko*z*oOHX%T0h)R1uW`a(Bhx{39Ia!4&eNjrDfBioDPAi zE9naPvF?{IB8aDtb}>&hB!#YEvFygTqzZ`x8%|T10(Prlke`OoesZSb@29~b%Tqml z0(vZo6RU*6hLhjBjkFS|*Ay}~b?xfrH(;cGqS|V2l~jfyso-bPAzON6{O+e9G_*!W z;P2&C<}i2kWKP%a!Jb1651tm6b{FoqbhjCXU+{<1$|$oqXh-UQkKP;-!7544%(Qjy zz#v!22y@*N0kYAeC)RfQ!OsEF#_p?jJ#qLMZx-~niAUc`YMo1Ni1xvo5c{2u+3oq} zO*c*fP2IZUptCb7jEJaA-Xebi@eJl(1{b;6Q-%JHb%`LwnGC-@GT?7ulvSvcF< zUI|uk28TpO`cX+NMja#eIE)t9rv2^%?z+=4|LBa%4}}ejkCJ>u=h%|N^;r@TiBs^kOB?+ zla~SDeOL6X)r+n!^sB3oMTr3(19J51q3uBP*!}^(>c;2clL`a^1xqJg88V3%;9Bg2 z6zL1P=ac08j-r^)oYDmy6C}`#^~@%Yy+q$;6_Z7Sd-b&pJ=xSt^HT0`&C6&-#i+2j z?kx|YZQ3E+chgc}y ziwy0w`fsH#O#p@Qxp3B)OOHDm(k-i==O8UNBJFv`x+tN`sb{y#Oj~pZSEF?X!Ad!jzIup8K$GSmkIBxT-`nn< zKGk)bPhQ{fZniDF!nCeNcawTvTYy2cti+)mar(sJq@=_ zLEvzl&B)C?mQp5QJ)0&N1uX&PdHue*BH47}KA-YpngsjW>hjQ=J=dn~MMVY@5a=td z@Li6ATOKUE3BE_9Sh-G#1yCygR@f^0vKEfqZboK6BZqn2_6b~tv&OPLj0xQr93|Z~ z{+vpW?69*0;7wJ~z0tf!6`9S`6zDiS=pT>YM9TPC1sqh>-K#m;vi`33e1=@<(f6?V z^#!92-jMP6IjM+Wn%y3_yEmE=3I+aIVu#erjfb=yLGy?Fdq44hoS=anrv^P1S~w9w z5z%}6;Y=uX2jN2mjEMZDwEp$UuzH+<2Jas?;L#&~I0M4MCH2}EH{n{6Mv!~Hb-za? zu@EVBAmol-seyMW;>|ccXt{btYJ^{xJXRKs;y6 zq7jNd`pbrU!d56MSSEM(F!XPy?OMXn*k$YX+|ttD?CO#K?^6D#=L?bl&2Nf=p4yS= z+WbKoRkNF_tNSOZ{LfR3@&7tA>EXS{oU_whS~rWTiy~S-H2gOY)JW>Yxw84+?CV2@ z^v~Fj-j94GL96%tUyMX{mu|axowTM8|ILirk@k+&i!YK@S=Kf-{vUwMFmCVr)**!# ze+ILo&-(T2*J0Ve+w!Xhu$t)nm(aE&+wNbvdE`X~`fq0VVdj59{{Pkd%ECVA?Mt32 zvcKFvN}q89br26b@TZgX%pSPqPk1_8|6RgIqkysa27)DO!I4Z6{}%h@miznrMJ4vX zsGr?xNXIarwV741zxnF$H}-h65HOcPsEQfb*vKEq@fUJsdh$?^h$Rs<_Um8q_*MGK z-u^F{&lxWO|IJ#@k-4q~DbfDLbg=#q`R~Z5g5dwP|NjFrig+veKYNj3*E@!J%$1Sp zEM6g!tRD5u6hk-YM zl3<9^sC9&-wVp*|tf9zg@~?)!XCxKpEbA!KgL2z1Em*C_(W@jd>$uhEb=?=`KYii9 zS9m+TPPjszvTzf5?1vS+AL423)_j%ZPC52nE9U$0y>G8`+4jjjNgd*ca8z#XU>xtq zi*k*HS_Z}Pj>8nIx@K(?fgS$~sLkl?KxX-7I>y-B(k=!)lwvqbWkKPTUpf8nDw{5@yO0!k7NuCflvUKze{Yj6EcGjrQ*YhX*E%I zBh>uboV|j|G5HJ{_o;vnOOG^m~8peuAgYWn5;f(08;aqoMOIIw(#szpE$*O~TP z&p-?WoAG~%JYWd%ftZ|jG~eEe!Ej&ordDv#n7L<59x$=F3n>PRy-&$9SjgB9 zPuPLLi$0Q`WMkEhQ*uWQ)>&TvR+1TMeVD*}P|s5LPuzJsjk^w`tH4L*1-*Xbq`bgS zqr5Rum{zp{4OJO;;ju&noRX7??&R0Q>-ckwR>q$)W1FrRx1=G+N0tJT zc-Vj|wBtnCpp=d$0065;teCdE-zq}MVgA7>AKt7vGnn>04<4YzJBxXUAk#%0{~jAh zu3|tU5Dqr>dW5wuG^}hQB&cou2|QUWzxehm`AI&*k`LJ83E(P13cY#*`htoL>m-eL zs9UZN zNik!_T>v!)he?Lx50f}diL{3&VQb#ROD13vaJJh@6+<%+{F{|(XRoi-Scl33Q(iU0 zJv+-8vd`zszxn>HFglPnuIkYqd`h{8u{SH@*S}?>s~usJ_(|@zKTuX)Iq^^^nMv;O z%f^4_o0c)5l{nB!`yd}xhv%WEu6A|B_l6Qaovy=&uLNOaJagrnI<%6YWxhzEhvNj+ z@@hh8OTlXa#`B4HjAEvbTrdTi#0I??EKY3g!-~ip4p+oIRx88SVI>z3ch~KC#jm%a z!*avCjawN*J?wiZ;<}NJX8!5FM_CqQW&(<+19Z`>s=*(5&$IUJv4MzpSLNe-_rceb zf>`ej=0v=+IaFkCGuOP1nh(qV$JBJecT*Rh`DGIhi9KGknt_yK4*%V*&&7>g66qc- z&|47>*5#b9XAddKe}$^QG&=k5GCgu&ZlsVDJAyMiQBq`AJwq<_j{P0MBA3{SQ*hjo z2l-tHKjm*hctwVE!(!nrnJd=S)r}gG&S`q-Hv8gjKz+HSSN>LH@}eH#5Bg^v1xb5* zQ>|K^|M4c%8~Gja|D{XgK_`>`?_Gcg?*6}VhB7{vF-TzsmNEQmRlghSVUHaJbEGNI z_Bgsh7k6CTq$~bi4BOoRa4=6=8{+l~v2}BPz!=--=b8L(FX?uQC+9QKg3{81hjM>w ztA8TSU-$CS=ogkSi^-`einM?CnMX2lD0|hZfsK#uFa83Fhd1Nj|Nn;rGQpLt(DdJx zv_t=Y*pdGcs7FE=PXEh)jC{U;#lpwx>$Db|Y2{bsI_>7JH#eoR;4X(`p9si1!@k%% zZ2Uv`)?g+c5HXKwpmpjM$-KIuWekPVM3h{$W;V;;@0ugwODqhd>b15ij=j4gpejAp zm9svPj8&<15!TN{LP<`1=Q4*{A#Y8@L#K~`+MXmL&p#l1d(`l32Nv{o4cE207RRb$G9%F} zFG6kWe4G#An`X&AwK$SB-i$SR;8KgRLHXe6`A|v?CiOh3m8w^p)%Fv!<|*{aqJfUCh_2~a?cl=NYt)fmF`#*F z^S9C8%+o^i<5i*P+C|buc51F=K;+r!f--i`e*l`kvDLb2oZ#Bu*(p_=DUsxTlrXbA zgeCIQ`ojjfe^plJl35TJJvq{~u^oEX7O;?xl8bAm!^M7rsrO;rc|xEi&@K)Ax?Q`L znWMO1aFD$cQ?ES@-)wRIH&s)^W$EbwT4IKsdcUl&yw8+LDScnvd=Qu@N>(Cj_eh#IUlBe}f18c^IQ;HGdgft?yP?-tMBg66BB#K-NW z-tVm8x1%?5@F0T@P*G5}*yh@TUVLNVVOcj?C-G~<*{$rN9$HaB;r1bei+ZBQly-`S z$49c+njkk;=lc}#s)KA z-6<`6>5uJ8#)V<0gYdHCO5Ik;sQEOMb^2Os(%B!Z7L&_+Q7S&r(0cFcrK1#&-oG^z zZ>B7fE7uA==KHEtOPXX^hp8T$lz?|oWhmZOt2bT2FYmqj#(6B5ON$0N?Ea(FqT^vS zwOi+6%G{}3&v|a`?}JC);VY!862^9^LCjag^8t(DBF?D#8%Uv#oB4$ ze|UFsmKsN8wWd~kpf-*_JBy(onE)_2o<6BT@qtbFODV+`I?TH*7vYy^wG4S(I2Xrm zxZMrg?hNp-!SCtSS5=q{Dr_ed!c>ZEvxUkvO&OPdT8I&1^WEj**i+*%SKe=wk3`s7 zg&ItvknXI2%BaHM2PM`*^F*qf%j=X!9Qn0P{ zisFrZ868W%_KkpVzrj7H5G?Nn*Bl)6aAhw{9p_}OGf{ny%1vywmaCPg_4bhb&=gW% zIfHLYXO^9zlb%(*hp5eEH4uZ}Q;UkGUj|$Lif?YQ5G!Q54@y)akR3a>wN6BoH{S$Y zkX33kxfOCB?`-G6UlewGJ68J-Y1rvh(~S^eAs2R~8YWcheLH{yvi`n&wH*wuF2qjW z_e~aG^#KpPI1PH4b+!{(4A(1)6)iK{t_%NM=&GX#=im8u5g4emJs6nB;Hu^saPbU! zg~JMEao;;GZ!?|Qm@YjVD2pvSoG2zKVT^Q+|G@D21kGl!xIAy00615>j% zMJwRb=n3=q&SxYgQkMJ3?R>xGkihP9m4+O*C_-%0rEh}3@zKZqn+I=puFSZNW{W&L ziMt%)#kEp7P)n&SJ;P?tadtdycdx5G2mw2s0XxTo>8AKVV;vzyjZJ&j+=8>cSV}Em zy4^D#~Nk^GJ<6w`^{AppObzOv+j$*xqT)m4P60tt?oLy$jp@{R$TiK zTRs#0wzq~&+|CnU8<39M-+S4)>4o~C^u82LD?YSyMqhS}dqLzF0%=4;@Yu;B6+2TW ze80--4W$YA-?J>H!fAPy0}d{NY!wxh;`w(DW}0;NfL|>S!;%}M8%=Qie}1pm7TT(r zf-w9u*Xd+p)E0h#i0srV&u7*>4Zg10qiUWBBNGt+B7x)C!r(%AOm)RO;F7K(UUA|O z#6G%_?&XJ@89|i@Y)2^98UScexAyxAD!2dWKn(d{}oGB@I#RxS4?j7){| zJ!tY8{L8M~9;C8f{wyh3)YI8<^Q3`eou)@V{cbJz_H`r}T!IpouaMF3WAX;_@OPbN zM1jYl6Tcji+Cb6zX(FsDzq{)n;Jkofy_DCfA{=%nu=-@@M031UVn3?Y(1f0$g}BUC zA2(gyg@qbx{~Mo-pdvSxG(p#|8DOvt?iI0qnh@78)S$4&u(f}1)-kLqbbL!OA}qC` z>6c8-GFy!!t@VAFqw@sn3D00AAwE8j>*3mI-g>h0mSRSu^s*|e^WkkD#{^o0tDJ$b zd)|d+7!R&0w??LiwdM3lgUb5an%S^HY;GMF%N7GFH}iSM(c0-+*69N!R3i1cH&>~6 zBk6=Zp8v2N;kpu%U(RMi2n;0AV z15P&(<_Zz1g0u6R@wy#aUCgW}miOamD(iUonmgx{W-8c#=o(Jzq1)L9CYmCwe*&WU z-0x#`B4A8f?4FhOlmI}q-^677MnhNfl0rJ4zgXFiimEf>HJ`@70vGuOSocybp%YKj zZ-=xi>&nv>W<6Vm-=}Lf_PE@*Txt>Z{aL47&iUQ)k*WONj$TT+BkGT$FwUf4LU4$ zIA&mK*2?M(gzRkKnSGu`mqr- z2MciQmH{>;3j37HbWAdW4bXbf}VyFg-p4zubpZOeGS@`j3! zEkJ8$Kx!P8x2fKot|@!6G;Z00rM9THDjFr`&Dz@w7~cJl*^yja%(cnA>QX!TJ!w)3 zpJ-&_=c5)@?$Doo8L#EMj<=Pt$$g8$?d@n*IAzwW$Kh|b@E|M*9voF}U3XRmyRUF} z)faR7sKtG|an_V6U&$%7xs2JjOF^C3fDTT0=ete~1Sq}P6T*?8&fg=kO9a;wH&F#o ze6)f=z=IN~^dI6hXqamJhjfPUprCBT)%b*k$<*`ah6yz*SK@x7#FleyZ4=`=f?z}N zmTya9`5%?lJNu?YTbqRz#dk#wP}m!?u6D#Zpn1edZ{J5ge@>noi4bxFu@0n#T+LXKYYYx=)#OCuTspJ}ui8=W&9(H(n%gL6h zK|G%|PeOl8jK&_03G-0jpr&YORN@W+5V*B8c)I(W&+-+MMoMCG9fjWA@xU097V0$1 zaDi$UZFyvzR+#hn#E_eL2S(=RLd}44eJ`v!QB1RP7g2S>ZL40ioJQ8LZ)>-X(7E!o zXNpj#pR+f&&5>x5wqMC()C5+wX^NXz-)a`}ID|@-2H1!bTN2lM?e-u(1q1WLTQw)z5-MX!(u_55#?&a2YJZbu$tFrRx)bDnxMm;ZY3 z6Jc&stAXe_F}v|01+?52+b zw0VmgR+pl7+*hoE(?lU+jab8Ny0BY4O%9vEQ|e4J?PFN_$-y}BQ65eDb@&Hj0ELQy zmN^X-|7=}o(~rqdi3MD*fvGk9ED4M$HyQDaCaPAISbbp5KDbYba)oO0RHa}7Zfu_s zwG~}_U6PH7xh;2=dRt&*KTj%?g1UO6nO$R>k-JPBK-$u>s@fJr$zNtdX+1XWQ|vW- zeeO4t*j4WDA1Tvz)Foc34(gVK+4T1@spoH-aQm7?e&OunM^PG}DXr9${t?fE{80X} zzi`f|CR!a2E4F~Sx4&O zj%$jm1&gL905sfqEJ?vP?<}en98LtEgBnUpdW}CQXW!S4@Xg# zji2b#v<2n7oJ3m09D?i@Mj1G0sHrBoL*IZ*Mbp zQ_y8is7uy0pz@*d?QjYmOkGU*>}=xd@C>r5Tf~m*%SLOv*)4pl;4`=xW{Q{NF{hvB zw|P&6K%cg-kt$1}T|0Q7&~(u2GKyakM?I|FrL+#hRJV>#FTo>@{*Z{tXi&vbCG{Xt zYY}{WxlQjVWW@hD&r0$b@>R2~^l4OjsTIOzWuh5|jcWP((k|Bam5t+7n5H`OnB*bq z?4){#l5pJlVjK0WU@>Pkbz-ejHNif9m0U(Uv0$|b+sm8OI<*9|jkp-@TuYq6vpa!L!c;8=|5H+gSk(WH8I zs3uXfi)DBUMGPpHrId|nZ~)0CO#RO&QBj(8rj}H)AVWh#<-Xu8TYhulZ9=U9&4CKLh;A9Maw{ zA(ay}wWcll?G+WKbatYh^b|V?hk}O~FM*^4C#$;qR`4k{qMxylKjC4ju-mFVnP(>n zW&lozY$A;n0q`{{aT}=*!@wsuDu!H{LXezMp)!5q%Maqv0}{N#A53?ml5$kW#iOb5 zR=HCq1(-%ICR`-d{q2;KA4*$dG=x(c8TyEeUJ+9gkoo71yNHhVQj$%XnBm0hfg$k# zM^o|V`p~x@<_h;rx53pl?!gdN;?^1;9kDyH;H8USNx5Gaiw*cZAH23s(zIyGoevres zCPs=g?!vp2lxqITWp6G1AiV`t7RQM(enfge8KM32yG!3@#K<8K2=tt6bcHM#`J`4S ztV;poIpH)3Y({HMw58!%NxF?x#$B-j5OC+NWjRa4$&%7I7-f$##XC2h#@39bcYYuw zS2)+wxCwIc(mXyOREmP5;mRbXqA;r#R zVp^ycP3LEo8Jc^;r}Mf5^xB?LPZ9sJ5|7E>YEMy~oFl_W@GID2FaTpY+zJWe=k1A9 zXw5#fE8MZsX2H?+qx9gh@?BH4uHuuS3pNg?RQ zqR&jqn@u(>ad{~tK}eQulmAFGQ`mc>EYFjhIgET+8bk&0+%S-JeTcK(0j=Z|+%|)C zkDKTs!g&q{dkb-}JDdaFO0{RoM6j1FHXR)gfk5L&)b@iDU=-2q!68{t=gdjxrJaP) zE3vd8ikI!luVZ8D8A3Db?&-VD=x!Z)y1UXY57eSbbrIu7^xAz1C2022%2+i{OUg0n zxwOFHu?B`3e`0krA?T-&ASo_Hk8U!r_F2HA7?s{ zaK*$%++W<8lAm%SCJi7foJ*-2ks9L%s;!Olb&Z&p54_!|=85q&+s3vArU&{*rC1LO(mN>Ck_FCa zLRE<`l1&Uzji?BrZ{l-e31#FTwOT51^ z@LKsofuU%G?la}M>4Cho6nlZFjnW^(Pp1GfF!76cpJN_heHKO|t{)~WZsGDI66U)~ z8CIQ;d?8;tZPiJbjEYa(q*o*>H>Ow03B)RP9hzNbI3TceYiN^eM6Z0f3|)FULCR=Q z4HB0G1xs43t8fl6;?Di-9*9)y)qLkjq7=Vmk+_{wPAWl4NONbBxg^)P>r0alq;GO0 ze{g+t@B3PY>t2}m>Q?+T&u5g>#YE_1Tsdkf4oLvZY_<~o% z#e;J3%2s_`j*rJ{QCA?~u|cgGyGVZJ3AT1!+O15z#JX%w$N0IQRh^%#>BI@+3oKi| z<%#UMC#d#itM8PpCo1Y3&jpol&lY$xuGhR9)=WyTaWvg5)={obXC2V3H!=*C2E?Wi zWt-Usr_b_P&oIG+l56n_d2ji5a82Uo)6fvBaM`lomK&7S1doZ)Fvp1}10- z=HEwE8!yUDa3!4G!!yB#m^gL1%%dik+du1h>&+d!^;1fo)8;Y>BB_RdSP^PWo0lty&r_S6TuVnp`h6Q;Xcl1va(c4B6fCm5R7=T z9+)NyawC>hm`Jcxsjb#zs&QOWl~Gfz=HJTCG%<{3HUgBy+1~fEy%BqWN{gU$3m+je z(tIVGHgqWe7?)$M*5I_;QK8bO@@Py^n<+u37}5#ql;XdvLmjvde{W8xz7N|cjj}!pD9XI9OYr$Xr6ftsszu~KW&TlpZ9?5BfjZLq zW++@<0K~FbshrVVmE*9tq-YTaB4jt;8Se~E)U2*|A)4Liu7qn}ZI|K6Qcq2FI+G}3 zP3J#XMw#!9duCgOXRqJTkQAkR(p}Wuw#39pk+J2&A_R?~lyIuq`!Yqlpk*}dvqPi{hpeYB z!3m8G_4&9qlA|Sx^R~83O%uDDyh>ROx$)qoF%o(Dik~Ea#t>Mv@Z(Xo^iZd8nl*WW z*YcH;bxkQ*reLY!F@{U?M4P*hKL~q05U-jA7ugC% z$5zeqs&^;N>2dI3Lsd+TA(?AW%Lw;f9=3rO>0)pRkSdUl%0$T8*GYN_-Pb3pBJ4k^ zn^ATnnJ$<7P8>6kS*O+m+2pb_&e*?08UKySXq8d1#>eklZ!>*kZA>YjNMD@H!#Pxv zREo;^#uMSxu&W*NmUF+|kvk*Dsqk%rjCL7COKl1zq*~GW2@kgH{Pfs zgZC~7GuPh<9zQI7#GH4iUIvUxGEhW9M}72tn0xr7A1<*O_Ub17NL0{)x~8a?euCHp z$)4NUs)fIGSs300v&||r|M9y~G6ksrTU#+}VQ(qDUqgz%w-q;B~ z*d6{ok8WOF=BvW!HaWQj3vgve1#=Le`&}EJ(-NkBg?kA|PcM3N z3wR_`X{H-1p0toIBSX2}NWH=q$!UYq7+kNfdSrmy!$H%9Yn?8$nrJ2)IYa^D4nahK zo0>L?=pl3&GdE49L_K7bAiqYtNQz&A1(yU@jayj()s8IdI|+xxE)$Pd5S?p}tEHNu z%1#S|^zS>v^hRU9gh zV^!5$1yrifNdZ{OOM5GLi<$N^PF@vjO$$|64>vzI_AQV~w!ao}fd z7HoXEhLqZGte{IQ7M_R9DT)hA5OC0*4z*<+RL6bjQA1ps`aGfN5n8L}kQx;_@mtGc zT3@K1Ak7C#7IqvB-ga%j5uGp|VPvq7qxE@HZP|}Jpte5B5Hn-5e_J;|BX~#4bRfgR zs<)e(^ekvdReQKFyIQ8E&R$WKptwt>Mz>&UJ`Ky&0l`brm&C>uudYQB&=|YSrd*>u zs8v$)fnZ(}P^`z4IDqnFI%nD3usthj1GEK|AAZE{n;p`!oxEe5K zLVZ;Y2ymEUZRWNYEB;|Zo*!;`oh%{vp)^O)R#&ZywfR48I!|3ERKdu|D1k|94b8l| zM*oMB>;Vln{S;c9uJrkI`UW=;sMANRb5W^+XI7G)UyEnvV_|hY)n&;v#&eH4P+Q)) zt!mY7k~3}vQ|Q=|w+2rn5~L~DY%*Z0z1ww&_-#~_z^(p)nf+iP<*p$Ydj~qa>+2mM z&)W1`#|X>1y@LQ4YzQeYvP)AKvf@uyG6TQ|GOK~2y=0x9Wi8Pvv-vBnhf&Ay+hx3tmQ<31JDeg2)K$D14$MUnR#@!cF zpzh{fI2HT;M536gik7qsa;a1RG+o|oYAdDcxtWRIdbuWZ&N?)sc_F*HNbY}|dDzNG z&80|a@fAHqJD!Bob$VJZ0bD1_cc(!swWh+4hBQ&imuRpvb82_z>6O3=_ zX-*_5GPqY?xXG}?&QnaFV1r&O-Um`$%2<9rz^XXkY&xJp7=iX<_#6JW_x~4jZygm^ z^R){mAw(dA00DxBCP47uPD5}HE{(er+#v}88V}OA1eeCGaT46!t#OCO9r||O-*}KKdQJ`yR)^F1v^10!vLjFq#S%Gnn>Of(GIGYYr|>km@H@+ulv5;K#UrW zZOcCfqV$=$-D014j4DoXS8kCBCCY3R>jU+BcMWiH=M71EM=Laq=$wjo!O?l)VM3WCIKk%W|bWj!}b$@%dF}j;gV*a9w5`)WI5eSB92G zQm+P*-WtZzDV1w5F0{_Oqc*03A5R2jafyw5|LwefY3O0uHbD2N9DGG@*yeenDDYh; zoQ+9ckM8gX2AJG0!5dAtiHp(LOg=g(-kdaoPfwv9rmTs9zA-`cyxg%l`HL!gPq`8i z5JP$x&rdFqm5t@E*B?ft8lW&$^27-CIT> zSRT*wIgv}$?PpaPx+u%lGChesn$?_li%W`^rv-S#s`1G>?lYF))iBt%r}3|2$ImKu zm~b7zg@w>vPu`ivGy1z|pY`-Sik->B$u3a|-s!M%<EDf7bhFJTWiP7LsyT9i>$gwZNpV(K(Oo&p<)+S#3iHQH@s&gbET|bg3xJuWzx82 zBa&xFM@F14H0HPt&BF;3RV+J~O>*j-Q9jT&%xwkDj<=uKs9c=t+?v_SEtQ{F3` zEJHk`E-5ZL9Mb(im@%WGYdGaTX!VYnP`8^!71+sPXuKmR1Gxt6nbAsLsp|c3zYHmG zPc)W<;eXCi5q!cJqT5{LJ^4787h={fkrNw2@Z3Y}UGfg3Y!A9F0y&}OQC5-}H>A#6 zHoUh{7!Pc-+HPF8EAA|{X3MHcrQejDPxw~#J2LAFRSrn|teqtX28a+1v2n0G0teTxOHe*WJvvjXz=E#4#Fr6Z07LH;#XxxJSV zbj$K`t@(>dMXOzmgXba@y$mwA-H(SL&)%llM9(d&DM3F@7vQyTBz>!z&SyN}|7{mH zUML-aM)O0Q0Ln831% z8^z~y5%-$7#m2WG^0h@K<0o8wdKc+Oq*-mbLd| z%^~y^r_+E{oFf`JmstBb+}2c1Y5Jd|VeR>1XG@CCLm&dh1P&fsMX6`(e&48cg&O(A zS8FYz6W-u*`b~i|yJMM80tN?Bd=(bB!AEw%L5N=sBy)>ML8=l=-i$bTaJF=c(nqOh@~Ll^RSE=LFX+@OY`%f{4jStb>jjj00062B z7+oh&(sV8CZ`8S;IOr3AeIQaKjNUse{Ql$BnAWOzisQloKZ|B3b*AfuM@5{6^h^7h z%Joc zHHX97te)u#dTn0k-`9$P=kM6qC;+uyx5;b$Lo!3)ZiRL3j(v_>b z$LqQx)vTm187%B1p62-OeyhC5`WnHysWuHi=B4<9X*2y+^xuLJ8cK zQuct%=7@-m-jl8fZ>O~_Y(jj&AcZq8*m2lx^A=iSamknFl+4)Gy=n2vKv=mxS%Xv2 z08{A|Cws%_p_&T+2vqbP0hbTffu52N42alr8JxECr!KWp*z3O;f}l3%Q9pfq6}Veq zOKWM9*9wpt9ozHVZ^66!aH`!^&{}r9>tfTlG@m-NaIVFKaPoY63bZz@!B(ZH-?o#v zmptMX;>bBZ)pKo25~H!NMJZOD!SVn45hV3Ma-|9bVbt8 zuGvn_lCnv)7C37z_Ps^zmqkg^c^eDIaBr#e}#SZ+e-2*f40pH3$>2Khw~ zywuJHMJd6TM=8UO<$9O1EaUrZX%$==lU?%CyPmdrjkTpvEgwzklkkg-!c0%rNT-hx zpb3bZZ?DycHx9dJYj?BBxCYzD%fpb{ttk6Q-P`I?XT9MHw;Q5rM&Y|vH#*pLxHWwO zP8g|ONxzi9<-o;esCB?9L2(+3PxZvdvsspwNGW<&mL_-9^XaFR%Rxt1uz3od%T^l0}rsopC9ZiIf-wF+G% zSew&Qaak(ZWBbZD7)~mvQbfk)t2JDdQI9662->z-zGFDf3TlzQUGCx+B+FyOOeyp# zTE8pG*OinNPg{Kot9|2JBpnQ9;vx2$l-gh~Sub*?wH?yvqIikC@nTAkn)gIfW;I8T zhiMQVQPi&R3}glip_$_PZm(Bm1+C((^ImvR<@uc_YTSLzjizH3o!DApUW2a zDalLQE}x<{lg95i?7|sH*|jyjU|X{tCwYUW$n4wd3*nirzDIdr&zdBqKl0Y_j7)o1 zX*&WyNyu0dW13%s+4T1F%6J{A={__%I8=kh9aj8K7b=*yoj+pMC)5y8Bc7fy%1y2( zwn%}YuoN~4 zVKF&ikCez&hjDPQ6WcCp! z?f4PSYXrBS*ABVi=^E#xY^GXEFB~Xz$@m4*s;02&&~k8_nVXG?IwbtNShSlA`i&)< zHn;w)-7VZtBH?-k6IRyN*7z75Uml{R<$^n}ZcevP9s#_CiUiuNjn6VK&+K&X30&1y z^2V|*cccB&HUCO(nvz{>It%{048jTUR~M=(<SyWlHD z9V~)BHwh_&-RX9F=2xogEtE~>dHa4^zvvnZaX7?S_$c$@v+bWMqf%^>wn7n0&E;? z+|71FEGnjv{A7}?hcpA1t>oo{FJOzZnWd7K%a+?p`TrFBpvBmo?Y%RmAO7t`_wd$EsRk!!Qb|OPu8#4+ z&qaR$@QsB!JU8DU$o3t`kzKsR8-)}{>8>vD4LsbJbCeEI5_7oB&6!88w%j^Px^8% zBJnrp|LIjf{-?!LnEZu%2qWMA7nJDwDbZiB>oBnUFQE1RMbU5ECPM7zYe&?ee;ejA zWlDqvApT$=Ojz>ADf3{-ZKE@sgs1C?yU1`$J>~pgjMIbv<(2t=Zs&FyQTQI#0dBB~ z-R(kNO3UT1So5S7U#!tDnFhxmaKPlH$k}pZiF3@L+8WO z8KR^bVfsG2IFGhUL*DrnWMdsr&ah-rEcAdCnS8&lyO}Sr|10cNiyz7Tf#3ll$iiie zo?49nNYsRi#B?s;>%-fZAI#Kixc*55FK}?CK3>E_m_%PpeX|~!YF!gAzS%ja!z1+Z zDA^N-zz2xjT>diz4qLGtV`ju8;k{CQCDGFIVDRzUW-r*r#LmB-kL!V;IRKOIJr$r( z>evm%!NY(R0e@?o@Fs2N3NWRstkWMo6RIN$%e6P=C=7}DXy)FVjT+T zrVuc67Au`|pz`SD23JI00~PkgMVvIXa^l52RzXK3+}x4dxB>avO*d-Mi937IgO2l` zLFyjtoW;R0V|kXv;h2m|OqOcAJI~7_3W$;`XaVM|>$_EAKbCWdm&s-uBBDIfV9;j4 zJd~WgEs2@`tQg$RzP9xBC*6T^!MKlvlvPQ|jg)Dc6U>~$xv}O(6*N_>(Iyz5Y82cX zy(u%#ZGxw^M*d>d#jK#;N#fI8E57tSsv{51FR`2iO_dA!=a9k@b~}OB(9NIOD}`q4 z)V3EFlLJ+EN1`IIEOVl8sQ!~8`9bIwcW;|x}GQHb8j^M$*+m`=H{zEMY#F2% zqr|duz4#L$>(r%5+Z025SMK}?u_ltG3sJh_`)36;rJ#Yp2x2P{u`tLOKY~Y7)$}{d zjQ~?*HN>NqHvzJqW2-}=KVQ4Cx5Z*V<399brUEv&ugT%G%E^ErVw(%Gwj>daMsN&N zDJdwVk-s#94T|5Ch9C}FE_P>hX$dNLzY^>T+bcfB>&3cAA<|Iy&I=)R#;Zd-Mqs-k zWbu#R4w+hRt|%^?Irj-A=zLVyBRPNa!ZhDxsXv&1?dTXotI`liJr$y5RP-bw`l_4M z%-vm|3i`E1SDE+Ec>O~7X9mzSm#pbnG4m#-^1Fq0OYCrgx9lW0rxkyY5xRx1*W2>E z5(vo%QR)-Y^kNp;kJya$WvOL;;Mv#9Z{pvAwjS3#)B(|~MYntYBT1*Y`||y@`q2cN zBfVvVB5#$)f|VHw2NjdY3>|wO@3M%3s!NxX<&rR79vvH~u=kmReyn%i<;$pIyN>?+ z0MIvq0eZ%YvB|Wo3@5Jda@!iF5GH+>5*}DWql<2u1IBdoLIXuqP~wH;$jsB#QqLQ= zO_H%w#;lPkksVda_@Jlz-F<^i>Jnn@6{_&-8zHuHz49L~euPLkMqylt?8G_${=6uCWjLbU6OY3}dzlypvx&4Si z8;w(ynOidB_B1JoD6?_sz!7?K@?wkZ2qKHe!KM5 zzm9;t75iP!SR^Bbwivq0JcazmDf%_A$YAlbh){c`hlfDp;8RB$w%vN!T z0enNghFCt-VG*Y5Q|K!OWER?ZlmvG7PS%iM5C1$6l%Trkt6O|Zf@CA&FrIV9#LNVF z961!InQdmS%7VeB$$EN@wq0R#o1_UVdjeQmwO8HO(TU?-13Nm$1=UV>Ls&+YV->Qs zh*~C_W@))C1#O*=e)VYQ?FRn@FQFHD@9=|8w!mZ;|K*;*XOJX zDvh@KVodysZsb|N;})zoW>Ya-hf{t8WZ4u}wCLPi{br+0cVh&XnCU~|A9b{TJ+Tj6 znYT7ILtA(RL+|Yecgqpi`ZNu_hgR&ZRq43qgO(OwUBq3)Oe@S(_!P_ub8++X@$n{; zZQ;~v!2W^H{z62=-0%~KiK%Dp;1Q26^SYu1@luXSYO$Ku5or(&Ej`d`lXAs-irfd~ zwix$q7IktJ6}uXU6|q6x08v3wG5)(Jb0*&)UGoQx$e` zm7FfRWPSFgMmAfg^;gjsPw#sg%~ykFpVD*L`z&Pz{XsW|m869PYm+U0q(M%BiT0UD~6eQ5AYj2mE#>Z!{cGZ9FY|@E{H3*0{Uo5GLjAc*p;>6lhu%Ha6D=t2;B)GY|_TAL}wLcK#d_ z(XEYCSlaDmLV8cC-^KJyBRplhY6M64YGG3fF+yZP>=k)kZcT9FK9Aj9A@Z*&=2|UZ zCyMm!J!+~ScXd0MQ$(6}4mCw43~`@a4>~VxksrX%h*s?m8$14pO4$4pyuL%b$#7ya z8x%A}wlS!EKgC7+=z+7wb<~J~AKew(SzzPsc3P3)ZOGJ|_Z+zd3s-iMc40xWyCz6n zjI$W&Euo_GyY_D^z=ah?7W6f@pDd56q*zf+eNw*m%?nJ&Fn#2gvCtC|^@Zko8K!bo zU1^u>VxQdnKXv$7skTtB_c3VU{2p{jq(1QnpWBUhu zmUt(H6Qma!qp?795O21xO`KO^HU1GJ`9dgvO10?Bn053t603wrjS$ZG;!4|cd4Fj! z#$pZZ&$(%KJI!x${7b3SaVukEJ~-uP7;$
uthpXiK_5muTl-QGcX){56Yy;%j+ij8#8KmosyY+I$O=c z1WgNxG*R1gVR;w~sowKfoa+^2TyNIzB)c?~OrBxyF|n5N)nxYCL?!k$xOh}DU?Rtz zbihZc>q!l(a%5d^&J3y`b!sytP*0{^Q)#iW0h3ec?f0nFinK}p7vc2Qh<3qlArUc? zQ{t}e(6qa2m5$(`?FMU}v0A%>S^lXP*DpDX8hRG%Eyw+;aPlfP@~W}z?&W-%Jb67x zlG(J+5+$`7pbU27>Wh==^F>V>+8Qo5u!Dar@Iz3(J;Wlv*W~hcII?ahv>XRSSYo)+ z{Kj4>x|TOC_aMbUyms~RgVQiy6AtZ_S!KIVsy$(mftOM@-{3rJ?{UsTj-l^wOViP(oc5xRwUqU8|S^IyuMza+H7jhcbdbOf=qw=9<}th)@U z04Y8YqBkT!vNsX67qcH>U!6#MNB77Y2kv1PH-K#>D++jnQu^0HB$YTCo2V zy+Hvo&&j zkf;4d;Ym^pMxb}q7k003Cb$&i$AQ*rAdtMOyqIcNjdnAQy{qYnAe%=(;envfKaR2` zaVTHYpcng%rOuX@nV6W`lpj4MwB_afNp>wIzFNH&2l6@Snj(2vUM0w&{mFwZbf12U zDubQ3i^vSEbGN!>^p+>y9Nn$cpW=@{c%5-37B7Edh$)P2CCfw9`EZn_)=%m-`63Rw zr@T8cx8j?f=5OM%Q$jV=y_)U#B~>*jT@L-u{O)nMT}kW5_e>W!JE&P`Qk5VYS!Q0- z{X0iX-{TOrHqOaSPBET1IO#Z7m360WaHMKRvQ`5-n5t3YpQt`+x?QllXNR|uk0G; z-gOl6sfl`zUsnxTOnE3oI)F9!0l0c&*m8fd=LGv_g>+_)S+;MHwssAo*6#Q0%UnAx z89*A|A79Ox{YhAgkBe&usRPZ~hdVRdq8~h{$7~w1P$ih>F+hCi@`|NZgDVl(3;3j4 zy}5D;?-%*dbrioe*3x~?w$@z{@D_=f>?MD?G34Efv^GA)*nDxEvssQW5Zz>*v;ttyl^N!C_udU^to}}WE&E<-svdWOi}N}gP%8S!&V^RcM<5s zyU?NPXy#~hWJ_rQCjgIopp=_7wFbAz<(k$~0NdVBIfyhb|*5g3)~bVT5d=NfQEsCY)e&VKSqHs&|bz9u{A-n{!{P+ z!AUX%UsSW9gH1;Q%Il5J=Dj~aesI&k_m>$7m7&Yx-^>r3U;MA|E&nkvP+Un3{}%%z znYqv6#hVX5q3VA<*w-)rf9@RrZ#KyPYQOvco}%-{*oZqjI~n_2)h+*aGuc*En8Z+; zASbcBIpnX?fB5TQkHwoeZ%%h6c7OXnU0+{+Q}A~WgC)r!FV@Jw;MekJJ&$z=>WGbe z)xxBtBn3Kxf6M$SpBjgBWl| zSXfbm|K9vUSUlEgVNtD(_9yu((Ea0#Mcw>~mKrni7O=+lZ^osZ?W(XRc3l6~>PvjC z6B}{8vxJLFwM14yQBhGz$wBPDhsQj4FgyO72uqjG-*l$I4Uv?Xc;wy)*Q5Dcp{FnS zb6yv#gOrpEQR}9O$v7SQzjO3~^G}q>u*n64`Tp+J=tU@aMM%iYW0Al7J8f7bH~$Z7 z**)>U8~40!sas1We_12lc|ZKm_k8|uwwJ~dQN}gj=aY>AOhV?h%|9s~0&#KSd2r}u zjDpYF+XGFyUzwY4PU-tKHO(AUUbLurgFcu3%&cEA4ZM)+=P}FF5c8|Px2DKd)ad%t zs9_%>0beLs9a>ymtoHsj+LIUVNdV$aSXr6-LmXGW^R*=G$#U7|xwS!`xQ=ncb6X)x z*fqNba}>SrvgFMrN#vbzdniVQSN&;6`xnxSyQgL#;Fu9(pM<2O-^l>Ci;D}(d$tE( zBmx5i{g5|s^T8BLn|X;zU*y@{jOh#uuyAw9WSQe6JaGW1)IN|%L_6<(T8ScBv%D8S ze}sRK%zBZ1a_*10=O;=TQp?M}D!wck_hQ#wNZrA4vt;X3w%3tueXim1lIgI@IzNpD zcpnO`Jq2L3!JLkcKuebJ!j&Xh{eG=1E$^xs7%!Bk&R*lRN0uI|tZnz}#P?t&!5*iRIGg(X|s|<3pL^$z!n!Rk^UQ^qlgtZ{{ zea>fsi`iHDa;J*S_JZ*S$BSr{`?GW(eH;f;d~SycZm^+z4J?3-wVX^xHo}O*bI=}p zQ8Uqa{0hLeL;pQJt<&!h8t{%vNf1|i7eTHs4o<3GzFFNQQ=Fx`-Tff1tYQdvb$L47 z3c+CTMzIh9U;Q%PM1W5NK!xQ!$BUuC8K>Dt={UuGH(1F(MfVYwWaCFyG@SNdc-gZ& zppiCK9*sN=W;$i9UMOQ`Z;+ZVSw>RrWZQh+l>)^=Z8FfX;AP~8Gt{UN-;-(jIW906bPVDK zz@bOo1$1%?FoqJ}4rZZH5^UR~)^VQ(lt`gkCF>tFDB#-%@>sL3Qi@xFg_Sobg`S0u z^96jcW6u8T#p;=RW1c`A|AR(>H(5uq(ew+Di9YhGFUQBaS&C)Tv$}A;bxBSZTbG?|maxtd%gG=J zMjZw49q5&U+^tcQpaP)ByM2Dk&K#o2I%lNoz#N^L&(^3jjOb;iQTeI&FGusli30}U zFkTTYTV`K@m*VpqTi>i4EeWOxWzk}^%BP|y26G|2JvWn#s^5LDX27{v#~?0|pjr4B zU;gTeyeC0i$rvFB1tkZ#DBNZ^J0|WiZQ$Ur>fl6z!D{b{wTPp7Fm&ge7WUn<2Z4#P zRd$97_qjuaFTA#CNA~S8Yd*tAbAnV||Fr%P8XBE7#iYhOgR^5dv#hKr&^m66c~Pu` zH&oB8xo@tQMqX)uq8z^c)h}!V7pDLsZWYmOT+^W27WIvxh(#DuWoG%k)$aJ)r-h97 zMwZkU!ubq}+?46G$#;njUt(!jU72+YX3E>z?9b3*5Tpd z(&p=swq*|Ft>Hjh3C6Y-DtrpG;$Xr(4R(^UcoF>Fzi%A=0(TM7qnaF zQw&V2Nq=9K0#&R8{xuDw&deh0YXJewLZ?_Ebp@qwiq4^zxHo>2$*b0p+|rvx=3D z3|RpWF^<->koy{TN)^G`yQ5eMe)--?w+kC$>kJv~Iz_6K#)5%eEp-=4ApqbyI#&%x z^$kpR(EYn#NuO$18TSNZSbvX_2S}TKR0Z=*e04^pq*Jg&SeYu}fV@n7nZdM0AjWFe znDiu%^v^_gZsvoc3!nsd&D-P_d#A$AMl~pk!3kjJloDdCkd(WV)0G{EL5sN4;jS`i zx1b!*tYsK9Z@1Z1n0_37+>%hM%6n&ULRR9Rn(fag1kW}fwHYf+JQdwu+uL8r%z7&lL^(WG?v#E6xIOcHHPlN zuCAMMQbI3EK0dhPafE#3zB9j|O7n+^3%<{3bajN7PGN#h9=zv>qBq9n%8HdggezyZsp3lFgreIJs58vWu&0S<6QZ*Hif+!{>v6QiN zBLQBN)4uUV@jl zkbEJ6a%PB&e)R$```cP6sBMa5pH=W11)t@16v_asrcrIDEPp@KYUKqC_PF!*F6d`A znwq(#=O&aOjI*o+@JvAP>jA1qPQXR^6J*`gT=cB{C_z-(d>u>7W}R))kFm*9O#Z3% z0^QqILD{xP3d$;i7>Z_v0pxlsfdO*5>U(aK0F;8BV;C%#*h#mcW5j6#QSVdI?NxFV zYaYb93H-Fd$P=5CSn5)MOicXqZG!N7l!tozSV7(KkCIn84j5<%(Iu@&a${c?7G5zc z8?J_ZEdRk9>y#nJQ0Cn%KTN~4O+RT9c5jHW&X{pFP*YoC?u{{23)C@? z+eJ+$Z)$y~)1T~PLw|mjTqM1zp=MFdbQ4Rz=x3H>5fDVP{K+#T$=#K?xzFc{x1pvp zo6z#jSNG%N388ZxfLbH6s{xbTnpY{*U%QVg=K#KM7m6EDF{-*YU+a(KR|eKdOZ1Rs zsIm4d$lGg-^;=fX`CL)8Y+*nj70kQK&H|AqIFu^>WL5jfkmQaJX_DKyI0Y@wOK2#* zn+*q7;bW-L1?Nf=V|#A+Q^!GAx39DbH}fzH)VMhy{qW}S>en2TKMcV+W;Yff!JM{( z2gB&68@jX;y=%HX^F7PR>jVAe7x&W+#ceedeCl}D=Zleq$z&p!;zG|{YN z*BcrsjGOUJ4cMPq0p09GTIlLg$8$E_I#qS>q)Gd`QB{KE0&k=7Vh!X zo-l3pF`&e1z4y(%AVUhLWiplWgqT=T|G>kqtm58oEyKeTW8ZUvCLfEf31mFk*T}dV zB&}apagH z(C20!7OB>|2s}UCZvTZdbO{m`Tr8ve-QeFboSu7n%pRv$*AX#l!;r&0EMwb{AJ#kla$!J8s*w%I)Z-VT>OxG)alcn;pve^`jbj z8Kp(Uc4Ba>jY2N<7`SX_t6D?#U$>a~H10gtHxW}!#jlp7)U8G~AkIr(#gl6jy_ves zUz7>bf1J~U!%Dwgg%8z`!z&nbdn&oHC6Sj`f~lFVN}tkbWW0sJ;;UpLEUZI`qJbfh zcv2G9%j$LkBHN)g#slCck&FJ;49QV0KRjA3%?XW6cdOKY`Z_UDCL&czxSd|js^T?9 z(#ATbN4E0z!I#&b0e*Kb`H{6}2#gIK<^ssNd{(7e@955sG9*cRx6E5Ssv-%L*le~D zgg{;m`Q3lsNUBZgl66<64CkPA>v(-Q4?MHm)coyo?~H7zVR7}bxqPy^w>`)rHw{TD zowQI*jAGp``X!#rkfRk-7$O55xSEu={av}1tR7FVD%S3F^uy7~$Td~ff`hoDN0*iN zUGxN;HIpH8IRk&6)3S1Qqg~AG0~_3O-={0s`@*-$tvb&K>hO{{=cD+eG#ZymBz$L% zU8v>$wr6laR&JX$Dv>^1LN}qFA>-HhjBY=5rgBaQWAT+dwBq$)fzC`B-OO!wnd-sX z`eJKWpvC)@HY1LMzPw=V?>(HG(GqI2&WpLq5cO!80vgXj`Qc1k4y#nhni&e>IKFx5 zfAsSmRrEw&2x)m}tqW|O@1{g={v3@KrR@TuiAt-zF6=Mx3P-tahIG=IQ)(QL8p$*# z7@3n@Yj`+EDS1rxZ5-z0oYkokZMbt4lyh(b9rAb;twOu{SHrP|L#E`2TAgO%d_Z#N+*Pm9}6w|4TF6i|8T;66DSr<19k_$!3zT2}R$fs33 zAQX<_<^`xo+jD)-L@6nKARw-6lBCVy=ac7%qSB##c~Te(hu~DNv3HCKo#t0Mu4#eQQcZGF;a8*gb=e1pyD(& zE~#FKL2}<%emh8c0sz0B;Ni!dsXM`Sjw-+i?Hx#MYSQg^5v9p!;BG0#sB}UjCnvXe zYRKOYjx=lLUmLxGt7AYfmv;qW9Pw$+Yo>d^l&kyCXG{P<&Ed_Rm#^}ArJ`o8mj$`U z9LE)Wf2K|C{b`m^i=tu!1#Z;Y4Lgvu#~X&^E-qcWokH^IcWS=y^IJ*4Mnkj59|UH) zA1kMR=f2L0?9@aa-pPUtsfyKa5PnFnt>FyCl(9nO$`WNnN)S3y?fydOth~~5cmH>v zez9)T^0_VmP(jY{idD>Hg=6qId7l3w{ZmSL+l$Q(gd@}*k|te`{HZ}9v3_LqHx?id zcDWkI<1m>n_*tA=)a7@1=3c{@lLoxS?V>zS*i}w9`<)kJouP4`2!I=ko|FQU4Hc(6^6v)7$Yq<(OB16Ub@D`Bwbx zp9)|2+z%t(to2IP_mn5rdCFl(3czv|cw%{RVRa229muRLh^P-J`**6 z_;R|g4kJ2-s}zs<+y&opf|u1$njX;C+Z7p`^s<-MG$W*!bA7Uzq0ysgWi2BiAM&-z z2Vty8zm@-^Q$AaYFZOo`(3TrpI4sA7-yBzG(=v1cbMiaWwbvV*s5+A?SNOG25+@zh zFnF0uF`rc@kfXlw<AA$wcP(xYS>U4x@>)9{nB|j_W!^T{#%43@) z+dymASQ9#&-v^b(C(?QAe4#5C4izH8tKr5Gto=69rBxB8n}zij4#^#WOB9c6$i=hosyeK0mX97haOAz(Bxyr#Hv94WQvx$Mdr4^dx@58UQ z^^?O+!O*4hx9z*b*<(eP%#iUn=)t#1aZ5b)()}^)%d=W}asR9J9_!l)w#0Gq6-L)&&QNW{LyQh~9cG810j!so z1_|Mb(rO*WvXX662Tj@Kq-3wQfRl^8{n>?k*8!z%+-cR=n8cPJIoP(~}^HD#gJICDMh-U-$o@8XF{tl1}&#e>1+_aAty?w(nf69T&UD5g)uT|~Ux{n`)Phh)co8CyzBaDKdH67v4rUq%dNHjD2}pus3uu=Rj;w>D0ChxxSpMr66Y@C5yu~iAT0kAC!$u>V(T^zydu#P(5n9 zHxm`3IFMcrxIJs&T16O?TK}ZGz3;|B9~tr1+pD=K4$@?o+@FG#hetR__G6veTZQhE zT<8t&;}l;2;%d+F(9p~y$_&SWWPN!K2<@7*3Zs5|nQ`(EW*lrM4 zX=5qx2meCC7Z;vZJ8XD7Le7*Y1y3l*I5J09Mx}@zGqu7V=|46p=luPNGPghB+w%Zx zA~v)n?YAz%u3ziLb(k`(aMVI|@C@7@!{P+-J9ydM<(8LPDbdX$<1q|Me>=rBnp&1J znh}=cB_-w@1;!aKP^^$izlt=L%vbYxqAEPMOhzvbFJ#YHP z4uHL?)*Z6k$nz6yA#Bx!(t~>`{8g_i4Y4&UTxn&v`&VDV6TfEns|MDmXHV~Q%xP>q zpT6exj5|Q8sN4y{BEk8a8-Dju_g6$C~CoxUwTj;-UPQNO<31#nI^`jkRmSN=}AfWmKf?c+&|>v zi>mZeK3}R#OK)b1#W#-XC7JZl$?SIg^uUt&tXoo0;lYs;r(E2PxCp(?ls7$#N%yX* zqU>hC4l)Pk-&#A!o{h-wzZd(8Uz$TP-T4iU*escXB+^d5_Ge{R=;imaF-I0|*po0J zY(0+1D`Yg>g_^wmtMIqy+&Z-M?yLfD*JvqN1}v=wir=A*Jm|Q^?kDtH?)Fw(-M$Jv zHiHQ?Wdd|N9980NDel+xdl|!dt;{&Fka4Qz@n6BdS&TU4 z7>RJj47cbBELe4zVMxnBwn4y*C+_Si)X!QYc0Fm~s`6|4?K|c6qP@3X9gO9r^l=wn zQ3-}<+~dBu?#xn`0fsq@Sv*>{lgDE1k<cQ`GJ6*+-xmp@Pj|F6bjQ{M zkNd>lMf5QHXc%ApIJ#+L?AY6SQRxz|xeeL8xVeiq@t3BVBvq3Q20QaHfv)^FLXmS( zS3*qBtGb~wb3171MI#*_bd*j|XYkPRI_ir1(7`{~6jVfl+Y3)e-SCo%1*m3wq*?k6 z-%B&wk6drq>(`{+g;wNr#$BvxQl3UtWVJn9+V<$Qh59*+^?n_DzeA%u|N>-{g3wE`>ToX`}g;yC@RuHq$<7l4goBoH))}Rgx-ZnZ!bXU zMY^;^kRs9v5CR00-ec&!cL=@L8$aK-@AbWF-M`?jJHMWNX4XFE%$}J&=lR%`M4qIP zdYp1)KL$)($hxA$%wmHIe~Yxbc!Ml}jzN64@tNi&_+tuw$-6@X^5(`Fubw2tL3lwtJ zkjKW02R^g%6g&HB_Da>4hg*v?1_APbD7SkccbA_x%BMo-AY*u-%m3ePaP7dQc_( zdvZLeu7XXxcn`R?_tM`~y=)Kzn%e{qMR6BzdTaio4vtv=nt^<4JP9?JFiF(=G9o#> z^C%mVpJ?ywyp0nl3{tJKn-!Ut-eU_?l`)5ujNJ7rDPg;P7+WLi=}Wwl#!6Z$b~Rp} zN`!!JoJaxotvz*~`jsPQ{G%~cPg6xi9)=BaTOgFYEae+S5CuYrvIDbM$D%Im$INmd zD*m+x16dnXgIwy=pOBIIG({CWZ?MR>W;1A()irkdNIi2%sc2pT^m|cn{MyBo`gKTC z-L|gb!uT2W0*mNZHS1{Gtkv=cVURvsO zaHtwK2~rK;c@z9`s~95gnT|n?tixf_A2C z$w~H?AWyM12u;kqMu` z!CA@pJ>V(IEI1_ai@Hy)gk0o?I92v32ZR7VtXw&`zyC8?-4EmKfb; zAToIfu#L^e>N|C(k7c2vp@B|f3T)DFW@a|iZs7B5{qJ$v?KvZT)`i*zl1vyW_pyoa zNPHzSMUVQE;mrN)d#6GN@z!k!4GlIMmL2nYj~r>We<6w(y-Edo`4%d1*T+DX$=+?s za~H1}l=|`^DJeGKdY<++@@6IJ_7sBN856ADpL17_k z`e=aMaJ5lZid)hsVegp?12UmE0RF?vxi|QZdX_@f*}bG=@CoKrjP3#)#`wx0h?a%t zahpmz5ZV-{WoC>4WpbKogcM!9I-F?A8qF%7a6=1Atv_wo%LjCv#XV^y>Gr4}IX!TC zT@|KKCVhR4X_Z|815t)X%cC=Fj~nNi75W|y5NG;+T&hXe zhAu^{N(OgzLg>&cNN52Z?nXKCICx275l8u+N73;UI+335Z`OW$5N`{5N9iCbef-NB z?D4CKyWXKN^WVJ9l^v<=hsE2(mkud6S-t?N9^T2wGAvXWu{R;YORC(ty^Ak}r6{=3`Ml2}_Zra2y_P zNi2&qul^fXl}3Z)po$^W1g2)@WtsZh!Eg=nge(JfjU~kmvoR5LSF#G{oX!S$CD;?6 z9rZq(ky*buQpuEiAxeecoX$u8TO-}3l0|x_B;B2+SLh}jw&SZp*ZT)r^8*g|#nI=$ z!IVsXi_-M1FdhElz)RExVvc!MSak3(r?hw?cd#})@abHMg`r@==sV6C?+W2TY#H`d z^k)Vk(T;v0k^Vy<_oG87C*ted2*ZSUnE9!6zn>%++%m%~SdyG$$dBL?bSd?u_G#Ie z{5ZtoolX8N4Qne+lOg(iq};WmL!fVQk%;!IPr2x*Z#=!~pzVg`iWWtCRmnVR$*@bM z=Z9mQmn2=>&A>mMjcGgkz_yw6tm(bGzUQ}$d=;v3v)n}S^Qe{5L2mI+erNl_vDuaC z$Z%r5X$Si&@7Aq=ft5u(riCWU>2_h-DTx6!PvQoX18KUA^zKiv(MsrtwmcYIkK;Q@4kAvJ5Ia%Er5LiTWcbb0wPohl#~9Tl5GCdi{2vIMo| z#J&&I84^ru6`YuoX>2klAuo@gQ7k(wO?x0g!_ce{@`5Bu3HZTv{wm*sidiB`n5e2m zWSpTcE5f$XZ+yj~!^D4wG1WSKoW+9zDK*7U=x3j$tRh-SN=jZFKc{&)t|LlVn2H!o zSU2!-!dV#e6GB7=)-}HzF*<&Xh%iBAFEPYXLwK#%4LRXY?N&_r%3lJrEC<`QDz18$ zTCpCzM60z|0P(%KPkuAlY`(_~n0Y`Mb&ry;Sm!K6lJ3pS%o2B|`=ME$T9nL72x@H# zzvKP^*U%BGHq&CbaWS&Z@`9O~y3#!?Vg`M{FeU>P%A3x1!>2Nb=a-$p9>HhsH=$~8 zG#nNaVt8tdRNCM?$a+6lln(Su0*9?I*(twTtvSIdXWmz@_MC_a{|3Et*WtiZAKcD@ z)0?mb*o^h8Qh>7Bt=Rh^MdF*|o!sqXaiXT1!*lfW&#oRt0%raz*Uge8kFMr?3diZr zOb!OIM2fdPX-Uufcmn%1*jt7*)%(d4osvY^HF#|!>!3Xt7@ww9a%Yf9&{b!fUm@EA z{wdk&(QLizt846q^Ir58j>CIsGC#P&oO`SfvT&Osc-18ukC+4~UOA*iJ>wgV z^}8cwcMW7fqQs}<n+?^?C)pn|gMw(x-ot0*6V%w)-F+n!9V|x! zz|0EFg|eqSn6ZYmMHkIF`HUZ(kNQ6AvQZ6{RP5X#ReC}ChUGn+j?;ZK;GKL?WWt$C z1L}iGK{953d)vqmH3VnDbmeCqc2v81PU`+fq|R4!mh%@>^FkKOT%kN+7!OAT#qtE) zSk_mw6{%40RIhU>Ihbm5*=;tPic{BO%c3yzMV5e{5bci`V2h>tWJF2@Q%1I#7nKsQ z4QU&|lHl|_Lz}ftxn4n`@N9|JPhyX`f5m1^>ripz5l-xX>5#zIYzuM<_}0zUV#_&Q zO9CL*fmeOR%a?i&UCnag?{=qHNCi_!OSdXnyH}!RwpficKAsM^v*`3j;Gx?hl0;m~ zpZAO_GJ*bhO^(@KwIgd40ET!uwD87B|1cXUna^@h0_gm53K}Hveda!}MQ(PESt<@@ z&sSkQ=pQ(o27hO6pUr3mbS&K2BfJ2`cpQuxyPjX)6crV1Zf<^!JiA`bY!+NJ7B~iIxiNmO#`)?=XOs|o)tG(t&B$DDU|#!NK4h2)FqwA@6gUZA@C%=qt`X}N5wihM@RRHV(b!)(c6{=M986~bz zIHBH0fg<@W-Q0wP$n?gQctXE&@ezak5qf!}`xE8yeOY}^uK3hUn+5qt+fbY4oEW2X z{o6Q<12BM`{FJbF%^ls=R?WZXvCrvPG!alWk;qKd4Nim?e~cs*UNbeAeew?mgAsR~ zeWFw7_+=dI2x*!g3Mxf3!}Oj{Y66#0AjiFef&x~0txksC(jE!V?~PwY?JQ|t&HFHI z^ayh>%lv#$&sV1QH1(FSR)9MNt_7^t3{>J|MH4kUu>T<_iTA0pND#f^qA73PO&A?urlSHukio>%|ru>LZu|!v+F^!7D zQ<8jZb(*i$5jlS5O;rL#tCgij8eZE%jvsML5>B+(+gqFRv;s>n1F-)$W{)h9!J z=Xy^_6Cf=pr74q#A>|#rr@i1@)@X!jV7v0jfpG+9X7@mTvVJdxRB7H;$8k5S^ZZ73 z<}20XsUy)^A(w)H#ET$;5@$S>y{mw;~=)StmlI z3zbvo;qcn&%(7LobhXCyEO*|GRL4v&HH{<@`C3lVN>Cx=)O?qT@4F>PF~ZoCjp=Ld!;UMrtUA}D`c{(ftcwK- zGkIOnq|yG5CDqj>)~c*~92w1o!SwCN^FM1W==99OM9BiF8ZdF3EX ze{A3xJb;&1oPxE(DRxmWy*7u@&!cDYd%u55woH z2OxuPidA@tOLKm$RIr1avN2OvdGsCWWnPy7P@JYJK(1_s;Xw;~o5K&=+w<*2r#%9z z?xi3nnK5uGwT1(Wi1Z{zm{*w(18{jjUHOI47KT#Hg1R4VA%19EQ?Z<;^XiHCNtv>m zeej$Zn0cpRQ`{mO9>#N*R!Ec$`2zKA5;XNve&cZlef8AzLEoSih?Anxr)Jh@>IFq! zaogcH`r2ZX4+)%-+goH}`7hSoN01InVEfwCX?O2)C&2r1$+(2^lnwHipGXc zS$V++Gaci&UuHzqn0Lt+4KsJ!f3h#9_DE#TS9>hyd&4fdmGco#;! zdD3QMtqYoUA1MnO<9>KI;b^aN_$+t-#*_1`s9-b4+6$K4-es1P z6+wO-d!C!Vle55>(T$d=cD3KWdjFlhs=ZsV6Bijl6P%j-Nya+RHm_8QYL~R=>@@&T zraW(n2EuE#t`i-->oF&n1@nDpTAq|bk}$vi9M;_>p#_p03JAxnvF{kyLjuR&s6Y!j zYHoCyI=A3rPmp!H1{N!vHZkhGQa}PrOIle3O?}|9BA6J+`^2Lb=G6vTRUjpOGf3_+ zTIIc)(lIKG+H8@sq@nfxRGIawo3S%Qmb19(%N9(vhhHN}BZI&p^nR(q@u-XduwSZ! zPW6J>FeRwvm_YeF@^W-Exj;_qY9L#Kv9yXScB8aMT5i?gcc(37fs~X_fXkI9N~47g zkEuwAS2W@ILouHQ6t4cO_wMq~y^guF7gG_Dye=*h?HaXWwWlYAdSc7PhGIZKhw$o` zZnO#yk97OA^cRms5)55}Qg+^~B9wd0xcs*?ArGly-vL86N0J7MZ+OVC-ja70cA)Kf z=~}QVuoC=$uZ2;2ajxijnxwKJ?do` z!;TR<1|2Dw_QgO}F|+sh?R1%PSz+;DZDKkW*dckR=on7uyss)Lr){TA{6qFy5sjU@ zS#5iM<+iXTdrT#qOLgoD(2O>pJ_h8@-5jpu=+9~TVXvEgZw@n?177&vP#&xDlRj!1 zA?bkXn+36Y@V4yl4g@3jV5{RwW!g&qxbM)=st+A$^*49!5br7jU+LcW$(He)S_gm4 zHB`@fU16XOV&peeI2GuPHrg zBP5hRA+_%`C)ofg)h`gR7WR7I%{sD?+4OL|8ex)cQ_XI@(^QkA^-SK~CL}AecOZJh zQ8@j}pcC)RPKDV@&5F-dwT{K2`=B)?{i;n7qr8xu10s;ku9qMLrBWcU;aGK89qQcQ zVbk&aazmW5V{GuTqs>$w z^d@L>WA(VyyHzX4#Aq`=hEQ~0%}Z`M+xzN7B}e3O`fSkG)EXtd{Y=lkkR9MTMR?L* zfAq5ThO~HvE4zei=E^j#v~lQmnOxqLehf1-HY~=~Unca?x$zeRZ%;I*jIEGhm{YcQ zE!u_FdbXrjmrTp^l{*YMG2TX%S3G`R#FrUhlwg?ALYBZKtvO~XNINljmwu_q0j{aI zrpsQ@R*`XxqX)PM&ZHKRk%<6-j|OW>+xk5}N5mRJUuPt-fAibsRsWHzmcFZyR?|ys zYBgOzr9Q4sl3b9xv{?Xa*=EVpvaC;x z`0_EUZyiGfPY_++U^x`8-%gu3P*W|)ipbjt>nr2(D~)`@u9v&f1QA03h7y@rbC;94 zYkeCgI#xG;@Bq&(!qxtI03oJ!EonYHN9g*4dz#P6FALw_F1KpGQ>o~zpFM`;0S5l( zyP_5-Qp>Kw$3+r9*(U=iR^qVZkfV>V>k%%F8RA{LC^-&r`rr<&^q+JhdLsA(;4H>l zJ6@lmzV=i(H$qiVEQRf3z3`lwi4F853{`BMs;dY`zygHrqbd!pT;X0KBLkOhiR*Q_ zBTOHk{&;AiZMnpiL1V>RO9a6Jlqnvi7m$U9F@obp#YD#^o#@qT4q3G95_@^W`u6^z zWKysc$ZQ8W>L+ao6)O#J_isBky#fNaF%A$tculKKQ@s*6T}A&LU;o61U{y`c+jjoI z{k?_O7o5A&S$ZDcEAhzLABhC|JHuIA2=o3^4c|sp7P&kMK2y79Hhzs#``#!cqdK|K ziE&w;hl4l4m-|Kb)}OK*rzTC_J=zF1OI$!hYq#h#uwuz4DqCa>+}riw3Ku~=_^%CYkc;X#}r>@hgmud=O)k;&FYOe%4iTD9`y|j@c-B_Y7kUWustl@!D3XmyY2*lI3ve8*rxYB zaZ_iW{3KBfKkZ5HXce_CVollfYh8-_6sm~#>P6tZ+BIXrHn4KyHRLpG(#W{;K6F5Y zp@dUfYba|yJL^LqoyOw^)|UMz$8%Cz-pjhwL9Qjz<4>lr9={$fy{g4MW}S9*Q&&K> z`SQHHeRcJ;#`xY+Pd}V@pLB@%Sl>kc=xM9`r}LWGHw=S|I=G7HMUvtJwqLF|_ zpUco)?S#1hJGmys+8nH8+(;kzx@)Ng6JWv7Br!7R;ncCRH;`adlss6c|7DxL36|ow zH~SLt=7j<@IV0iM%QNp`_s^P&)a!y+UyxLIj1}javQBF z6)1UR%;bC43sV>Aze?NPF!x<@PXFTHd=6eX)Z7W2TneZFIMpjb7WXDYWB1x`X)-d; zX=F6Ne0*$~fSw%D#mIvrnfOq-d<)bVbh4E@^zAl3m;))fO!xj z_soGVWk8jPMQtzmAkidA+gcD0YMd?+ip+AYo$4o#QA=_W*z}DFwk^vxc_@}-o5341 zGHM}ip3PpVmSO_t?0*CRazWH=IvlVS8GcF(X@*+!PMbzW*@ma_?`e+ud z68?cOE^BhGj2N}(ew!OLT+J#q%M^?Ad_d+~%=8QRWVskEnpX%HZnC73ERf{E0@5B! z4>mmAbws866Lb+CK&1RBmiAj!vU8d~R18r8att3CW~_*+`|!<8l6;WD65cgxgUBpi zQ|8gxc(OW}wc^KZUyorj$doRrF%1DGe758gck&v;y{)T7r;24Fjox{AQ&D=7`^Sog zPb;@1@9(oYx3TGpix)Ya6pQ;x-OMU7MZQkb*0%0?B&@L8vFNB_oI#m}HHF1aDwbaW z!RcIlXan8P2ghN=K|(Fg1U~f`Ts2JTz7bCwYkn}ol&kMouVkT%T7<@f#NWwc!-BkP zQhhatF+6O@9+S$eb+p|R8+e<;R#k=+orvdTu^6Ndw=I_ zb$fDB7Pl86=y*h*A)Z}I%9mZaWZ-r&nm_QKBn#7lr{2|Xw##A$%P$to^Tu5+h@c~k zB3g=SPx>8MG_89|mgD!J7|Y{NqR1XM9W&=Mv6}^s+qM&}$Z7;vxf|d$eq2fcXw*sx zX?%?$GJ|S?(ti;F-`fX@t8K%EDH)yuEKzS5*lH`PSVpI`-}-V+v-3hpe4KKn#J-hC zsBczK=$br2bmZpdHa0dMH#Y_?UAs#7X%6Do6GMA^*?1&xhqAp!GYY-A{)hAHEyPq9 z`vCZLs8Exbi0Echzu9+`I=GWnqQm7;Byv~RW?Au1nDI59YOQ8J5BRAb=TjA>L!eHr zRb4Hxv<`sWoC@Pg{)IUD$7uG-X{Dyp)>MnEQ9HO2(wcaKpIL*{O&@Si@47k@UaG%M zQ8^xv-UY<+{E?IYg#R1S@k0v|=Oc2K^eD-x(Mj!w7(Q62$U>HqmKILe{DA3>+__un z!V-IG)|zFj7tF)~jVC^BB-n^eQwx-PSjK%ECa`oAXt_o2+80>t)`0ZfoxNTQx>-ZZ zoK6pQk^K>#0|+#$#<~gk8}|s8_lii*CeBGLW24#!G}9w@??8LM^`sFBg6M_RXx1b?Q~}biA36FbKW_Nx4FH!#;NpaVigL9P$!d) zIX0IAkh$}NVtyF7iDAwQdN}|e_@f*DiJ)Pk*-LK3*_7i9Pn&PAZL>-5oS_PQUO^mh zM&SePadu|j+oYX;hA@yH&>hh+O7utbKgVTQZiAhzxqN<$eSRK3H(Nek1*di?JkGk=6xR}k;5;o6^T-huu*gZ{7Z{*F1V?_YTG%jH+Jf3Xn} zfQY}$ikHhiV3Y>V-%vC5KmBpX-UmRX7!6wP_9{u`uU`K}BRe@}yvA{C7Uc(1(*FMp z@7$U46+&rvVkcQ~fMD=`C_Zu1)j@AKZXOb>{<053e8Hr#Yq+`rTwD$~X2RtOwiRFW zKmF%vkMWpj8a%PM3>S_vFpE3?&P)U8f5`@}ljM%y7EUj{u#>o^GWNt5|LOShnWobb zsO`UGgCZaBH^l$=k23&>{{Ost1^i1PU?OJv3#TUWA^A&;{Bc=6iAQ zBKNNc3HYS{$ASC5>eR0lL~x6M`C-dY-`_d^N%Sek4sd&Q2tdpAm;7hs?`*%_i|8oB z?F-$Rxt#wMdVcOxJ^Y`xe_b<81Dk_}Gz0%+Sj4y*sp u?7T7e@_OlC%Q>U}H~n|+{BMi)_W4h=P3ZpRNzR`^QkK^Mmc4!#^1lGxmT=4f literal 0 HcmV?d00001 diff --git a/docs/_docs/exporters/pushgateway/pushgateway.md b/docs/_docs/exporters/pushgateway/pushgateway.md new file mode 100644 index 0000000..1f15eae --- /dev/null +++ b/docs/_docs/exporters/pushgateway/pushgateway.md @@ -0,0 +1,243 @@ +--- +title: Pushgateway +category: Exporters +permalink: /exporters/pushgateway/ +order: 2 +--- + +The [pushgateway](https://github.com/prometheus/pushgateway) exporter +will allow you to export data for a watcher to a Prometheus gateway. +To install the exporter, you will need to install its dependencies: + +```bash +pip install watchme[exporter-pushgateway] +``` + +The dependencies include the [prometheus client](https://github.com/prometheus/client_python). + +
+ +## Quick Start + +Setup a gateway + +```bash +$ docker run -d -p 9091:9091 prom/pushgateway +``` + +Create and add the exporter to an existing task. The only required parameter is the +url for the gateway (above we use http://localhost:9091). + +```bash +$ watchme add-exporter exporter-pushgateway url@http://localhost:9091 type@pushgateway task- +``` + +
+ +## Detailed Start + +### 1. Setup a Gateway + +To set up a gateway, you can find full instructions (section "Run It") +at [https://github.com/prometheus/pushgateway](https://github.com/prometheus/pushgateway). +A brief summary using Docker is provided here. First, bring up the gateway: + +```bash +$ docker run -d -p 9091:9091 prom/pushgateway +``` + +The image will pull and run, exposing the gateway on port 9091. First use +`docker ps` to ensure that it's running without issue: + +```bash +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +fd6e3d3fb8db prom/pushgateway "/bin/pushgateway" 7 seconds ago Up 6 seconds 0.0.0.0:9091->9091/tcp gifted_merkle +``` + +Then, open your browser to [http://localhost:9091/](http://localhost:9091/). You should +see an empty portal. + +![pushgateway.png]({{ site.url }}{{ site.baseurl }}/exporters/pushgateway/pushgateway.png) + +It's a fairly boring interface, but that's okay - we will fix it up when we push data to it! + + +### 2. Create a Watcher + +Next, let's create a watcher that will collect the data that we want to push to the +exporter. Imagine that we are interested in monitoring the temperate for a region. +Let's call the watcher "weather-watcher" + +```bash +$ watchme create temp-watcher +Adding watcher /home/vanessa/.watchme/temp-watcher... +Generating watcher config /home/vanessa/.watchme/temp-watcher/watchme.cfg +``` + +This is an empty watcher in that there aren't any tasks to run. + +### 3. Create a Task +Let's create a task to get the temperature for Luxembourg. + +```bash +$ watchme add-task temp-watcher task-luxembourg url@https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 selection@.local-temp get_text@true func@get_url_selection type@urls regex@[0-9]+ + +[task-luxembourg] +url = https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 +selection = .local-temp +get_text = true +func = get_url_selection +regex = [0-9]+ +active = true +type = urls +``` + +Above, we create a task called "task-luxembourg" that will go to the URL with the weather, +select the class ".local-temp" on the page, get the text for the class, and filter to the regular expression +for one or more numbers (`[0-9]+`). If you are interested in these details, see the +[urls watcher tasks]({{ site.url }}/watchers/urls/). + +#### Test the task + +We haven't scheduled it to run, but we might try testing it out to make sure +that it runs as we expect. + +```bash +$ watchme run temp-watcher task-luxembourg --test +Found 1 contender tasks. +[1/1] |===================================| 100.0% +{ + "task-luxembourg": [ + "54" + ] +} +``` + +The `--test` flag will ensure that the data is not saved and written to git. +The above worked great, because we see that the temperature is 54. + +### 4. Add an Exporter + +Let's now add an exporter. An exporter is like a task in that it can be activated +(or not), and you can define more than one for any particular watcher. +If you want to see the exporters available: + +```bash +$ watchme list --exporters +watchme: exporters +pushgateway +``` + +Adding an exporter is similar to adding a task! The command asks you to +name the exporter, and the watcher to which you are adding it: + +```bash +$ watchme add-exporter +``` + +And you can follow these labels by any parameters required for the exporter. +For the pushgateway exporter, along with specifying the type, +you are only required one parameter (url) that must start with http. + + +```bash +$ watchme add-exporter temp-watcher exporter-pushgateway url@http://localhost:9091 type@pushgateway +``` + +Along with seeing it on the screen, you can inspect the watcher to see that +the exporter is added: + +```bash +$ watchme inspect temp-watcher +[watcher] +active = false +type = urls + +[task-luxembourg] +url = https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 +selection = .local-temp +get_text = true +func = get_url_selection +regex = [0-9]+ +active = true +type = urls + +[exporter-pushgateway] +url = http://localhost:9091 +active = true +type = pushgateway +``` + +But notice how the exporter isn't added to the task? We can add it manually: + +```bash +# watchme add-exporter +$ watchme add-exporter temp-watcher exporter-pushgateway task-luxembourg +``` + +Or you could have added it when you first created the exporter. Let's show you +the command to remove the exporter: + +```bash +$ watchme remove temp-watcher exporter-pushgateway +Removing exporter-pushgateway from task-luxembourg +exporter-pushgateway removed successfully. +``` + +And then add it again, not only to the watcher but also to the task, with one command! + +```bash +$ watchme add-exporter temp-watcher exporter-pushgateway url@http://localhost:9091 type@pushgateway task-luxembourg +[exporter-pushgateway] +url = http://localhost:9091 +active = true +type = pushgateway +``` + +You can then inspect the watcher to see the full addition (note that "exporter-pushgateway" is added +to exporters for task-luxembourg: + +```bash +vanessa@vanessa-ThinkPad-T460s:~/Documents/Dropbox/Code/Python/watchme$ watchme inspect temp-watcher[watcher] +active = false +type = urls + +[task-luxembourg] +url = https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 +selection = .local-temp +get_text = true +func = get_url_selection +regex = [0-9]+ +active = true +type = urls +exporters = exporter-pushgateway + +[exporter-pushgateway] +url = http://localhost:9091 +active = true +type = pushgateway +``` + +### 5. Run the Exporter + +We can see that pushgateway is active, so when we run the task, it should push +data there. To do it fully, we need to activate the watcher: + +```bash +$ watchme activate temp-watcher +[watcher|temp-watcher] active: true +``` + +And then run the task. + +```bash +$ watchme run temp-watcher task-luxembourg +Exporting list to exporter-pushgateway +``` + +If all goes well, you can return to the web interface and see your data added! + +![pushed.png]({{ site.url }}{{ site.baseurl }}/exporters/pushgateway/pushed.png) + +Great job! You can learn more about the use cases for [push gateway here](https://prometheus.io/docs/practices/pushing/). diff --git a/docs/_docs/exporters/pushgateway/pushgateway.png b/docs/_docs/exporters/pushgateway/pushgateway.png new file mode 100644 index 0000000000000000000000000000000000000000..85dd7677441eefb0b8485998720479038619047e GIT binary patch literal 10724 zcmeIYRahKf5a8QLkia0pbx82w1PyM%gS!*lT`~+V0YV7wZo%DM0>Le~yE_9xXZi29 z``x{7_wByyOZSoP>gwuKb^82Hg)4uS2E8VI4FCX0Rz^}401!f6$}?z)FGt=ci2ln7 z$yHod9Ssd_X+vrKrAp!^rR}EXXyN8*;$jY1Iyl;!v$%e7F*kQ`wQ_VjL+TU;04hLM zQcT?|{ba>YTW#-2V&DOc~CCz2sW1cCE?(*EyZtz5|80~9CbS2}_$PBBaR06ckj6T?;>r|@PEa0e%D=J&yCfH$jH_;15pux&o3A!7lTmPFW3UW`oKj{ z_OOD$MS>FK+}X-5lq|5{+I9cU!`F#~YjJ-pno^fVfve$V>kx z$~J8Fpl)Jv+;i%9m-uNVUS5KsFUjRY@HIp{9|_m@S+bo8q3g7hCMaTqy zILL2VuY}7(JD5>`!4&$8BP$zsg&OV;PSt&TC3pRvN8G8_(~pSNey;t;$eGH5s6g9k zfN)qGIjmuBM#Khv#~Bal=?N>WcViz+rJuJE|al=9Z^`=ghiL__`~ zK+o1|FbDJy+CX8G;`T9}63AZuA&j~~qLW6v81ypB(q6Iv(c zPYl1=cZc0ZIhdP~aNN(iM1RwwX9L4HjoaTpSHO8CM27_Y7CMA~x(<-F!@z#$7vV>1 zmhcV?i@xb%vUdwC^>BWl^Qwx9UdO@Ezly3nuKEG^JMGeuSX?b#M>i=+*{ZA{Jme7+ zOGjyWiBZ1}2kI1IAjmHOu_<=uP(z*wjnC{1RyeI84KlzaL;@J5Mx(UbHzo@<5U~#S zo7^s~2EbcY&s zB)IJ;Uv#DlQYP_Q>Eyad1H>uK9;MC zsv{%cf47lO0Dv!YW4WZ+^5gj7F|7mr_D^yn&l5ezwwB-BKescPny?D!iT47GP55z+ zi~2UY&Sjv%Sqp{C!diV{3O1 z+YgFJw|d#|#`0s5dypqlp$?d%H;+2|Vq%Y8w+#SB6}4KR4R*TK!F~l5!y7kIeBc9my#g#CZqvtdz`wC^HRD(!EjO=s%A znV(aK9BacuC`AtL>qeu)?!7iX$wS!CcEdm#_+`zp$^edxqy!|D#gP1UhSt&p{Svi4 zy#FsKP{w+q{I&JzluTyro$6J4W@n4<<~P|E-wd~XT1x{tpW#WoxeRKiV8QCE=~TM3 zUK87ryX&M^&IWTC+zeox3@x?Z4JGB`J-F8cC_HI;Gf`H z^M^!&#~h10rcfhq60u=kYfj(hr}7h$%3okS7t8EyDNk;1sALQh6xw?wJp#nI&BT@V zbXj*eg=GJl%Kg_1N99)Vw_i{*E7hC&o~&}-G^5_FfI4n9d z@`c&8bTdBPf!ChgjLx={{NgDeBAY?{ENp$xn^tTzNYZ7hDZURFiXSF@gneKWi3o?D z@SD#*Cyv~<(Km8NkVY#b=ScDCtAX6n^Bjv4dX@ktm*;DTi!&uQsQadDiGX6Q@VmF2M>4?}spBJif6%>aI;%%!@&zK@YlqrBaI zlZW(kLKJg1gL5+vB3n<7))qTqa2|*kC$YvZHr$>sBDHkB$)@2VvJ>3WoqqLau_)6! z*Y+$2^?~vacF*?hT+eK0V*B%r)VAVuOr9E_b43Tj8%gIQvm3l9t=9DQ5|Q&Hl*IA! z)TTK*E3iW$Jhh&Pa3u3Lf>W4q-A`%V(s4J?IQ-gZii{xA6xGN}_eY|SQ^ZEofs1#0 zeA4B;|C}PClMXX*n=FhhErr&lspdB-g#~@^uHe?M{#~ZcurZg}>QOw}%&hR{u>EVs zTbwXgdr*-BvjJl&Sf+l>_#0uQ_Dbk38Nhx=>F6^(Uu|lh&$WNFNZId1&dmc^U;Us# zZtpAPy|R2Oa=2cmDm3f5`6|xUN<6&wK0zxn?X{_|#w;QL>s)N`$v90QG6$bnIloy) z?vVwY0N`-gw^h#;@IRjm)y*hfE!!lQ^@Ba-wF99>@OSvCUz-qea`ZP_ChfQk-+f== zngeg8o}N^DS7^YYerd-s$+S}%gB)^H6fB`^SkQ&T8`#B90LVuLlE1IY7rh5rjzZ&; z4v+t}@cXLyi*zYs|MX<^On&TISU)~(nw>;yD3QoANZCmaR|`Eg-F&dc2d>@qNt)NN zX*Tv-X5Ds%>V3z{xdDc=`TK9V3=u9|>J-slyBr7i_y=EW#%EVe%PIw?okn|MFp|ea z7Y~6?_I!NKBl!$#{Lg|;hNlz;M)M!-0E8eTm-wQ!Qv>2B?RXk>9DE!c9QL$Vz{D{5 z<7;w_9E3P9KF}pMU&CTDQq3Ru#5m+&Z$q6Yi7U2hD>)zN-(joRZ~K;sXgq89?(oz3 zJKV;`;^Mg6bqB~XPwGd_3SDv%ZMCH>YF*N9SAA)T-R6~G@m=+{nkM8 zgNS)05kXe%V@Jo$zl)k^_}*5cHNG_a$^D-#Izddms|B{SabI*h-(*R`yl*P-;K%LC z83SQakd~m?94|L-u>9t1a0trlKymS17o8{taKt6%iOXN0)gF#7l{%hYFLJd#YgEyo zqnk=Pmenz_b<@rpy_aj9+$Mb3uI-+`Dm>ZhnJEO^b#ki>iOA7iG zi??wJYu;Ah_?%6QR>|tMKRGOwYqlRtrTI3<4pFz1+h?2<%HQAEBE{w_Mk_3)w*3~SJ z`Sc13tB2R1Rcy(9S;BJ8UALMKjnaj+ZmyBW1kie`wC%2-;t zr^UBv?}2yzG1}nepNJ@VdCWpLePKsxs;aY<&lduyJVkbp$Ajooov6OVGG80#xk0#o z!vo+yyy{TmaAs=dONvXPy2AbSP_=2d0G}kUytui2rzfP6WO!!if{Z+~!R$MEa%=e^ zn-t20gvyi@uF8?&xA4g)bGWwv$`q7m<-xYGTpSUYw%lGtf3>Wm@1GX*C3zTQLp6g! z>rkNbZx&*qU!N|lf1mHJ%@tm~yl`_qBP4|@#QPx16*h3Lx|%b&kwS1R^5|jJL&{ir z+hbWEE!~kVrON96rHT7cbK0zP1)0f@faqV% zbSgRZ$|9MKKVQXTZ!qUn z7T0}k!p=|`c4_iK*3_T)ebCE->Wxa>p=UQ(4ZmID#_e)Yhvv zf?XGR%lQMSZGX#t0))!=QgRHA|Rws z+wo|AX~$cwn*X!DVc{)EDQc8f5Pj_>n7<_T?;99v5AnPD*O%1+v0+l=wYa64D;x4K zKCXgk6-yH2^iYve#Fx%wb$y68t1YqDFk`n z(WWjo2LlY@OJ5RjoxueF=iE)?>Sx&KG9E$R7TUK(Y>^70PF3~m%aZ*+e6ZHvooXRB z4f?1+5uSL#;9%)HBs!~_yvLx|rcVM=l|r}_Bt^b^8S%=~G1K+|DvX_wJqmaI?Z7`8 zhw(eh7SmikR*Qn8mSfu97i?{hWiAW3PR^3*%WBr+%$8hPA6Aip%E;}}C>#t64t<9} zf+i}v#4zKL)XuM4hwOl{DzJX}RXgX6Geg#lbDZ`|R4h1vA;CB2SoH>Y{RP|;ri@xW+>$D`0CeC}FK>xBWwP)#=kMeu~ z-+eqO^~@e{+syl`C9fy25v5h&_q@_-)b<8VXcR`^*jBeSQuk~>=$18pTJa#A;yP23 zTwtrg)U@vAX7qvben2)Z2$5=uRAEJlDO1H6UVUb^bHE?2RA4D2|FLC>^z-c9d)lkO zU&9#`pu(}=9OEVYSp-_Mk4JTIyiy}C^ZLB{oI@tpO;vCF(}N(y3|Yr7{+DoQ^tbc)CI z3Eumr1OWazS6lI=iIEbBP0Ul1_S|z>Wgj6?fIoj}gOw&u-clysZLS4(I>X)e2nfhd zemfj*t0@yAre8B>b1!%Hm4+XyM3}j}6`kGIcNx#tl-AXBX`ICw@(#@@oMCs^rs>AYolUpX{*>)`?<(|QNc09dE8PadUw|za4pMZ%ou&8T(8fT4>o=CJ?Tz(kM3acs=2pd^X zi=L(Tk9>Kp}m#6Fv-J6atEk6L6W>jonwMKELvX}zDUmu`am2X-=4}6yBPfnG`NHPJzp=9a&*yvp zM+`@(gK-RuDc=U~BmO_|@&8`>e^@}1{AEzSPQ@FjX)PU|9KLGNk`s zOcoh_^=GwQ$cc^x{%i9@ZU?Rvr#7>~o#DZvQH1a3P@@1vx)p^LyB@%w3S0(=BG$AD zlIn?33u`+`rnFSGQ=YjI0OK=O$0mLl3wmt@7w-qfg!uS;Ir(vx$&{0=a(6y?T;2%r z;?Ltuyl>XjZqIV%jSTHyR@X`Sj8pZ~A2-%Qs+B2-lH&D*3VLanTYB$V0RUSAT}W}|68OQO~Fn)>8G(IDXpIss_*gTG`#?V1^V#Z(MM5$icQ!~V$V7@!g%%)RFdq*CodWZ6B7yZFWvZ;SAhRF9DcY$#0}ZW5Kh$j!&z{_jcLG) zhyb;rkfoE@Pt$-(i}9Tm_YEnL%ZKON+Y@Iqk%(B(({aY(r`@S^Qx6Lj6F1j`(`%o+ z3>Foen%DB%%?JFnDzv17^DuAD5Pjk6TK%_m`$lDaJ_w%(g{&|MKH93=BxdJV>04^i zU7UIPp`^GI&x#@OL#joa31_hi+Hlvs2XT@gSKCM&7y&@fVyxeK&6u1^aL?a$J+Kx;UhNs7_B0j4>#=Vj(Yze94LSnT zn1x*?{fXQAu1olC4C>qBAG@H**Qi+G-x z(VciIH1yNAy{-VJvvO(tpiboJ<|y6yYUn;Kpkk0qV8@~&?ro82s8hwOWIDH#zn4Tr zU}0ZA5wBjf`<^c@<86mOc5oyxu}8NfRf% zWAo^Wr^8?e(Tj2He79R4pc>KeKA=rNAw%0qO&5$x#|9=|{Ib*1FKTd%aOlm4n%YI>-Rgt`GvQ;JWVr%9>1z+o{gznuZYKy+ zo>BN~p4}sZ?6qIN?s_>uKojQTa_pvDTn3wp!%DRT#cvWpVS6#L7tP%-3ebyn7D&C> z{i~JRN6QpWOD9OZO9I<%hEE6Y=@eE~^6|fYVsk<8|7~0&8(*`Y<4Ub2bF3Rgvi*kr z$VRlilxs3IF1P&zK<~}eH&EBitmA99sKJXeemtrlGsHWf$Gczch^IFqrMX0M^4xQ4cm3RB~M$K9oi>{mvHWx@^(yVf}YivkH@TTANxga!2Zjn~0vf_C( zuef9O5$^W+tGDQj07*w(ou@QzZ(O9`y3@aE$Z2FFZcViXbw-NewKP1O3RngNNrC{d zJ~`(uW|sW;qNMtl2vx>A0}TSg-UM@SvDzo}J3q`9cRn%%E=5r?s<8v5XuL{f|p1-n9Ar=89=U)0>?F(p1)jiAX zn2eI(Qw^_WHLYRQ?ZwJ5M*50?zf`M`>kENuzg_R1vTLsA6Dn%M*ycC))9xiKJQd<5 zMySBhEob6yv~n~l?5m@1oCg47=7q$(knNXXIUK>+M|An&s+7JtxY5e>N^z`IRt9XZJ=6 zJPaUsn5ZU?5%v@H_>i_&3i&+(TzeEaWJ!A8tmNTTPd##O;=pD}JWjg+Q}am~KD9PpivSu3wk(Hk_|>5#|MV64gn-{#jL@h8koV3kUxJDslz+P5m~g z?6&eC6%thLA!7i?+@feU-3Z$Ao_Q(3u`hUX<;93n9n$>A`6Jd)|ZagRX3?u4sIa$ zvpbi5dW!@EomN1}4)Ut6Paa9dY+`hgjcN6}ETXq{amoMy-S)p+fZ&(yPy{DMK{iR^ z$SbB`k}R56LEf=J;=TRy2o-E#au510ucALT6iy+`Ke7Bc`Fp4s-ADz*Ru7#*Fb$dz z*b8t4_Yr}{X4fwFdXudmj-fnM?X0JEOkL%Irq%!SD_2fm8ULsiY3yCzq9(WuHF}U@ zTc0n78sMOCg?#g@O45W1Bpo6W#QDUjL|` zeEzLF$KaJ8q~VZ_OzOf;mG5Y6K}ig?^8U?o~Jdxg_wWvBA@T;Z*B7eZVlkhWlV)cUhl)6-TWY2J>+ zFCFk#D38(F?QCk0WY%JM6E_pC@&!9#B57_uI6Pbh0J_n`Id8@;UGnHY*gT?~^%1@f zgADFn84v~nJ$4N>1)PKD@4F{e16Irq5E%hI8r;wIjzP!$Qzr)y=7_fT6izIlir>zu zS--bh5eFG?TxNOs#)M~OIA%Vv_=1z9R^Kl!8EBM;YM0GPM)63MeN7c` z3WzoytjPr-=ZDiCpd#@bL*PtAtda5d-nYJH#lT2V|o4E zhMvMlmv_BywD*`;aVzdwZ>F^G?6q2^kc+Ex`zsV=wk-*hN-x#LZM}UB3vy)`i;KBF z(SBN+Bv*QuG6UaGEoDlORzueec8t=QT#(bmnHuC|WJ zAMzGUcEhwU&iw+j=+SUZ6Zggr1YjN%l9KXpGSt@IJd`7YD}$WfxDfhbe%2Bxz%JJ) zv%aw3?8Au?8fw(3&707h~ut*XrJaW5xTaqTRnclZ{KQhiqq^ho<8jXd%AG zV-C~j_&6pfJOM^8IW|Ni7M?fXnRy$rp0}A&daFHey2&%Gu#~vMP_@{I)NhxC57v`# z@QfYX^e6dk7NTaM?j8c;(OUv~Z}z!$9zxfMn%zNJL zf6nuD?+`1I0Yjg5hDtkzlCwD=%2t09dk`t~p2C=A%{Apz<$ipQt(O=rW@Tvj)xc1M z!@h+hE!&mP3+k*RyS8Xh=1hQO`GiXv^XiFlAz%M~rTe6wx&5+>YiC=Vz}r?}&}gX? zGAQf?l9~@qrDC>7!vOr*6N%#Y#R}CecL=QIIca8XGQ8a&l8RF9S^t_1vf@EY4;F|^ zx;ho~bT+Q9`QC(2IB?ewJ0JD3ZkO9f{OCN4q)mtMz^`HTgswrK$U|+&P zhhIEbd5$xtU(n(_p`503uBHuc!;q})V)pNJJPy!}(q*cITdpGboxbdoJ5+6Nqt|k( z-k>eHfb^|X;y1izgI9jGQ=VAKX(Iyln&0BfHjwB!#eW~YpGXB|K>NQ+_^x!V1F?*R zGLTf)p+yic$_OZmKkTDjTebDO2!jRy?`4 zECb_2e{Z!Nmj6~cmQ{3pn|vHSa-n{Sgc!1`^k-H7wG-{)t1lwQ8Ut8ay$xw@GoOZ$uO7+0 zOXRE=d|H()>$JazU3r`qH(>Pn_!}2%Pw4BE&5%WXwST6}$N&Ld2(IqV&!;Xy1*OBe zb2HliRB9%{QV>ON_bl~8~4-^)mSyB zd8=w-_Yg-SQkvpJ4s7}5az}vB-Z#MTXL2e~c=#3t1A$7dNbVZqA_udmHB4(Q`V=qN zaH0Z1CzOi%`HioONH$NBuUaS^UD(gH$J5*?0^4AglCN_By-l=(lSBB6ISv@`k#I4L zA+S=^fhe#hX8GLJ@Ohj zR+tVF1)pEJySMdy{&j9t2ps?sKn>UL2NRiBTHpGd2Ov#GZ@!k5=+0Suf>NauwYkgN zE#BT@syiJ|9nW4to^KKohqZ($6FyGx{YFJB;J7jajV|3buaUJ)gK-nPZ0Fbrevq-} z=C{Q%WHgwKHwamzMUKY(Q6Jw)ppk-xBXRjt?&>;XnX-g&f9~$?%Rs$~t`eV!qcB`*2u8bC60*`u%O0tlb zr(t4Bn#m~Bhz~lli@TN#E|%q?@G$@nTck4e-<{A5+{d zs4JIgt`&Ii;ZRDfLkGPwwoa9aO-!;}(fOHBvqT$De>Tqz8)S9!#Ei&2^#oI*A-{ao1 qo___sq`v*T{l7%*|GNdA!2+2zH0(i?lP`HLfUMMK$qI4f!2bb6YlDpd literal 0 HcmV?d00001 diff --git a/setup.py b/setup.py index 8c726bb..5fb898a 100644 --- a/setup.py +++ b/setup.py @@ -10,15 +10,15 @@ from setuptools import setup, find_packages -import codecs import os ################################################################################ # HELPER FUNCTIONS ############################################################# ################################################################################ + def get_lookup(): - '''get version by way of singularity.version, returns a + '''get version by way of singularity.version, returns a lookup dictionary with several global variables without needing to import singularity ''' @@ -28,12 +28,13 @@ def get_lookup(): exec(filey.read(), lookup) return lookup + # Read in requirements def get_requirements(lookup=None, key="INSTALL_REQUIRES"): '''get_requirements reads in requirements and versions from the lookup obtained with get_lookup''' - if lookup == None: + if lookup is None: lookup = get_lookup() install_requires = [] @@ -41,19 +42,19 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): module_name = module[0] module_meta = module[1] if "exact_version" in module_meta: - dependency = "%s==%s" %(module_name,module_meta['exact_version']) + dependency = "%s==%s" % (module_name, module_meta['exact_version']) elif "min_version" in module_meta: - if module_meta['min_version'] == None: + min_version = module_meta['min_version'] + if min_version is None: dependency = module_name else: - dependency = "%s>=%s" %(module_name,module_meta['min_version']) + dependency = "%s>=%s" % (module_name, min_version) install_requires.append(dependency) return install_requires - # Make sure everything is relative to setup.py -install_path = os.path.dirname(os.path.abspath(__file__)) +install_path = os.path.dirname(os.path.abspath(__file__)) os.chdir(install_path) # Get version information from the lookup @@ -95,7 +96,7 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): author_email=AUTHOR_EMAIL, maintainer=AUTHOR, maintainer_email=AUTHOR_EMAIL, - packages=find_packages(), + packages=find_packages(), include_package_data=True, zip_safe=False, url=PACKAGE_URL, @@ -105,7 +106,7 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): keywords=KEYWORDS, setup_requires=["pytest-runner"], tests_require=["pytest"], - install_requires = INSTALL_REQUIRES, + install_requires=INSTALL_REQUIRES, extras_require={ 'all': [INSTALL_ALL], 'watchers': [WATCHERS], @@ -125,4 +126,4 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): 'Programming Language :: Python :: 3', ], - entry_points = {'console_scripts': [ 'watchme=watchme.client:main' ] }) + entry_points={'console_scripts': ['watchme=watchme.client:main']}) diff --git a/watchme/client/__init__.py b/watchme/client/__init__.py index 7279ec2..658c12a 100644 --- a/watchme/client/__init__.py +++ b/watchme/client/__init__.py @@ -171,7 +171,7 @@ def get_parser(): add_exporter.add_argument('--type', dest="exporter_type", choices=WATCHME_EXPORTERS, - default=WATCHME_EXPORTERS[0]) + default=None) add_exporter.add_argument('--active', dest="active", choices=["true", "false"], @@ -199,6 +199,14 @@ def get_parser(): ls = subparsers.add_parser("list", help="list all watchers at a base") + ls.add_argument('--exporters', dest="exporters", + help="list exporters available", + default=False, action='store_true') + + ls.add_argument('--watchers', dest="watchers", + help="list watchers available", + default=False, action='store_true') + # protect and freeze protect = subparsers.add_parser("protect", diff --git a/watchme/client/create.py b/watchme/client/create.py index cf7e73b..3cec159 100644 --- a/watchme/client/create.py +++ b/watchme/client/create.py @@ -20,4 +20,3 @@ def main(args, extra): for watcher in watchers: create_watcher(watcher) - diff --git a/watchme/client/exporter.py b/watchme/client/exporter.py index e62bbb1..fcd3745 100644 --- a/watchme/client/exporter.py +++ b/watchme/client/exporter.py @@ -22,8 +22,8 @@ def main(args, extra): # Are we adding an extra task or exporter? if not exporter.startswith('exporter'): - bot.error('Exporter name must start with exporter-') - bot.exit('watchme add-exporter watcher exporter-pushgateway task1 task2 task3') + bot.error('Exporter name must start with exporter-, found %s' % exporter) + bot.exit('watchme add-exporter task-a task-b task-c') # Extra parameters are parameters for the exporter, or task names if extra == None: @@ -47,7 +47,10 @@ def main(args, extra): watcher = get_watcher(name, base=args.base, create=False) # Double check the exporter type - if exporter_type not in WATCHME_EXPORTERS: + if exporter_type == None: + bot.warning("exporter_type not defined, assumed existing.") + + elif exporter_type not in WATCHME_EXPORTERS: choices = ','.join(WATCHME_EXPORTERS) bot.exit("%s is not a valid exporter, %s" %(exporter_type, choices)) diff --git a/watchme/client/ls.py b/watchme/client/ls.py index 0ba0c01..da58690 100644 --- a/watchme/client/ls.py +++ b/watchme/client/ls.py @@ -10,16 +10,31 @@ from watchme.command import ( get_watchers, - list_watcher + list_watcher, + list_exporters, + list_watcher_types ) from watchme.logger import bot def main(args, extra): '''list installed watchers ''' - if extra == None: - get_watchers(args.base) + # The user wants to list exporters + if args.exporters == True: + list_exporters() + + elif args.watchers == True: + list_watcher_types() + + # Otherwise, we are listing installed watchers and tasks else: - for watcher in extra: - list_watcher(watcher, args.base) + + # If no watchers provided, list the watchers + if extra == None: + get_watchers(args.base) + + # Otherwise, list the tasks of the watcher + else: + for watcher in extra: + list_watcher(watcher, args.base) diff --git a/watchme/client/push.py b/watchme/client/push.py new file mode 100644 index 0000000..caba317 --- /dev/null +++ b/watchme/client/push.py @@ -0,0 +1,43 @@ +''' + +Copyright (C) 2019 Vanessa Sochat. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' + +from watchme import get_watcher +from watchme.utils import ( write_json, generate_temporary_file ) +from watchme.logger import bot +import json +import os + +def main(args, extra): + '''push one or more watchers to an exporter endpoint + ''' + # Required - will print help if not provided + name = args.watcher[0] + task = args.task[0] + exporter = args.exporter[0] + + # Example command to show task- and exporter- + example = 'watchme push watcher task-reddit exporter-pushgateway' + + if not task.startswith('task'): + bot.exit('Task name must start with "task", e.g., %s' % example) + + if not exporter.startswith('exporter'): + bot.exit('Exporter name must start with "exporter", e.g., %s' % example) + + # Get the watcher to interact with, must already exist + watcher = get_watcher(name, base=args.base, create=False) + + bot.info("push is not written yet.") + # Push to the exporter (not the default git) + #result = watcher.push(task=task, + # exporter=exporter, + # name=name, + # export_json=args.json, + # base=args.base) diff --git a/watchme/client/remove.py b/watchme/client/remove.py index c7e0554..1006440 100644 --- a/watchme/client/remove.py +++ b/watchme/client/remove.py @@ -12,7 +12,7 @@ from watchme.logger import bot def main(args, extra): - '''activate one or more watchers + '''remove tasks or exporters from a watcher, or delete the entire watcher ''' # Required - will print help if not provided name = args.watcher[0] @@ -26,10 +26,26 @@ def main(args, extra): # Exit if the user doesn't provide any tasks to remove if extra == None: - bot.exit('Provide tasks to remove, or --delete for entire watcher.') + bot.exit('Provide tasks or exporters to remove, or --delete for entire watcher.') - for task in extra: + # Case 1: one argument indicates removing an entire task or exporter + if len(extra) == 1: - # Remove the task, if it exists - watcher.remove_task(task) + section = extra[0] + # Remove the task or exporter, if it exists + if section.startswith('task'): + watcher.remove_task(section) + + elif section.startswith('exporter'): + watcher.remove_exporter(section) + + else: + bot.error("Task and exporters must begin with (task|exporter)-") + + # Case 2: two arguments indicate removing an exporter from a task + elif len(extra) == 2: + + # Allow the user to specify either order + extra.sort() # task, exporter + watcher.remove_task_exporter(extra[1], extra[0]) diff --git a/watchme/command/__init__.py b/watchme/command/__init__.py index d8a8160..eccc4b4 100644 --- a/watchme/command/__init__.py +++ b/watchme/command/__init__.py @@ -26,5 +26,7 @@ ) from .utils import ( get_watchers, - list_watcher + list_exporters, + list_watcher, + list_watcher_types ) diff --git a/watchme/command/utils.py b/watchme/command/utils.py index f729dde..1f871da 100644 --- a/watchme/command/utils.py +++ b/watchme/command/utils.py @@ -8,7 +8,11 @@ ''' -from watchme.defaults import WATCHME_BASE_DIR +from watchme.defaults import ( + WATCHME_BASE_DIR, + WATCHME_EXPORTERS, + WATCHME_TASK_TYPES +) from watchme.utils import ( get_tmpdir, run_command ) from watchme.logger import bot @@ -55,6 +59,20 @@ def list_watcher(watcher, base=None): else: bot.exit('%s does not exist.' % base) + +def list_exporters(): + '''list the exporter options provided by watchme + ''' + bot.custom(prefix="watchme:", message="exporters", color="CYAN") + bot.info('\n '.join(WATCHME_EXPORTERS)) + +def list_watcher_types(): + '''list the exporter options provided by watchme + ''' + bot.custom(prefix="watchme:", message="watcher task types", color="CYAN") + bot.info('\n '.join(WATCHME_TASK_TYPES)) + + def clone_watcher(repo, base=None, name=None): '''clone a watcher from Github (or other version control with git) meaning that we clone to a temporary folder, and then move diff --git a/watchme/exporters/__init__.py b/watchme/exporters/__init__.py index 6fb4493..d8e6464 100644 --- a/watchme/exporters/__init__.py +++ b/watchme/exporters/__init__.py @@ -9,6 +9,8 @@ ''' +from watchme.logger import bot + class ExporterBase(object): required_params = [] @@ -64,3 +66,7 @@ def validate(self): if param not in self.params: bot.error('Missing required parameter: %s' % param) self.valid = False + + # If the exporter has some custom validation, do it here + if self.valid is True and hasattr(self, "_validate"): + self._validate() diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py index 5bfe2e0..648041b 100644 --- a/watchme/exporters/pushgateway/__init__.py +++ b/watchme/exporters/pushgateway/__init__.py @@ -26,10 +26,16 @@ def __init__(self, name, params={}, **kwargs): bot.error("pip install watchme[exporter-pushgateway]") return - self.type = 'pushgateway' - self.registry = CollectorRegistry() - + self.type = 'pushgateway' super(Exporter, self).__init__(name, params, **kwargs) + self.registry = CollectorRegistry() + + def _validate(self): + '''this function is called after the ExporterBase validate (checking + for required params) to ensure that the url param starts with http. + ''' + if not self.params['url'].startswith('http'): + bot.exit("url must start with http, found %s" % self.params['url']) def _save_text_list(self, name, results): '''for a list of general text results, send them to a pushgateway. diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index 603416c..5dbc10d 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -31,7 +31,11 @@ export_runs, get_exporter, add_exporter, - _add_exporter + remove_exporter, + remove_task_exporter, + _add_exporter, + add_task_exporter, + push ) from .settings import ( @@ -585,7 +589,7 @@ def get_task(self, name, save=False): return task - def _task_selected(self, task, regexp=None): + def _task_selected(self, task, regexp=None, active=True): '''check if a task is active and (if defined) passes user provided task names or regular expressions. @@ -593,6 +597,7 @@ def _task_selected(self, task, regexp=None): ========== task: the task object to check regexp: an optional regular expression (or name) to check + active: a task is selected if it's active (default True) ''' selected = True @@ -601,8 +606,8 @@ def _task_selected(self, task, regexp=None): selected = False # Is the task not active (undefined is active)? - active = task.params.get('active', 'true') - if active == "false": + is_active = task.params.get('active', 'true') + if is_active == "false" and active == True: bot.info('Task %s is not active.' % task) selected = False @@ -614,7 +619,7 @@ def _task_selected(self, task, regexp=None): return selected - def get_tasks(self, regexp=None): + def get_tasks(self, regexp=None, quiet=False, active=True): '''get the tasks for a watcher, possibly matching a regular expression. A list of dictionaries is returned, each holding the parameters for a task. "uri" will hold the task (folder) name, active @@ -623,6 +628,8 @@ def get_tasks(self, regexp=None): ========== regexp: if supplied, the user wants to run only tasks that match a particular pattern + quiet: If quiet, don't print the number of tasks found + active: only return active tasks (default True) ''' self.load_config() @@ -634,10 +641,11 @@ def get_tasks(self, regexp=None): # Check that the task should be run, and is valid if task != None: - if self._task_selected(task, regexp) and task.valid: + if self._task_selected(task, regexp, active) and task.valid: tasks.append(task) - bot.info('Found %s contender tasks.' % len(tasks)) + if quiet == False: + bot.info('Found %s contender tasks.' % len(tasks)) return tasks @@ -863,7 +871,11 @@ def __str__(self): # Exporters Watcher.add_exporter = add_exporter +Watcher.add_task_exporter = add_task_exporter Watcher.get_exporter = get_exporter Watcher._add_exporter = _add_exporter +Watcher.remove_exporter = remove_exporter +Watcher.remove_task_exporter = remove_task_exporter Watcher.export_dict = export_dict Watcher.export_runs = export_runs +Watcher.push = push diff --git a/watchme/watchers/data.py b/watchme/watchers/data.py index 53921ce..e2bb4d2 100644 --- a/watchme/watchers/data.py +++ b/watchme/watchers/data.py @@ -9,6 +9,7 @@ ''' from watchme.logger import bot +from watchme.defaults import WATCHME_EXPORTERS from watchme.utils import ( which, get_user @@ -16,10 +17,12 @@ from watchme.command import ( get_commits, git_show, - git_date + git_date, + git_commit ) -import os import json +import os +import re # Default (git) Data Exports @@ -87,16 +90,15 @@ def export_dict(self, task, return result -# Exporter Functions + # Push to the exporter (not the default git) + result = watcher.push(task=task, + exporter=exporter, + name=name, + export_json=args.json, + base=args.base) - # Add the exporter, optionally with tasks - watcher.add_exporter(exporter=exporter, - exporter_type=exporter_type, - params=params, - tasks=tasks, - force=args.force, - active=args.active) +# Exporter Functions def add_exporter(self, name, exporter_type, params, tasks, force=False, active="true"): '''add an exporter, meaning an extra plugin to push data to a remote. @@ -114,30 +116,51 @@ def add_exporter(self, name, exporter_type, params, tasks, force=False, active=" if not name.startswith('exporter'): bot.exit('Exporter name must start with "exporter" (e.g., exporter-grafana)') - # Ensure it's a valid type - if exporter_type not in WATCHME_EXPORTERS: - bot.exit('%s is not a valid type: %s' % WATCHME_EXPORTERS) + self.load_config() - # Validate variables provided for task - if task_type.startswith('pushgateway'): - from watchme.exporters.pushgateway import Exporter + # If it already exists and the user isn't recreating, don't add it again. + if name in self.config.sections() and exporter_type == None: + exporter = self.get_exporter(name) + # Otherwise, this is creation of a new exporter else: - bot.exit('exporter_type %s not installed' % exporter_type) - # Convert list to dictionary - params = self._get_params_dict(params) + # Ensure it's a valid type + if exporter_type not in WATCHME_EXPORTERS: + bot.exit('%s is not a valid type: %s' % WATCHME_EXPORTERS) + + # Validate variables provided for task + if exporter_type.startswith('pushgateway'): + from watchme.exporters.pushgateway import Exporter + + else: + bot.exit('exporter_type %s not installed' % exporter_type) + + # Convert list to dictionary + params = self._get_params_dict(params) + + # Creating the exporter will validate parameters + exporter = Exporter(name, params=params) - # Creating the exporter will validate parameters - exporter = Exporter(name, params=params) # Exit if the exporter is not valid if not exporter.valid: - bot.exit('%s is not valid, will not be added.' % exporter) + bot.exit('%s is not valid, will not be added.' % exporter.name) - # Write to file (all tasks get active = True added, and type) - self._add_exporter(exporter, force, active, tasks) + # Add the exporter to the config + if exporter_type != None: + self._add_exporter(exporter, force, active) + # Add tasks to it + for task in tasks: + self.add_task_exporter(task, exporter.name) + + # Save all changes + self.save() + + # Commit changes + git_commit(repo=self.repo, task=self.name, + message="ADD exporter %s" % exporter.name) def _add_exporter(self, exporter, force=False, active='true', tasks=[]): @@ -150,7 +173,7 @@ def _add_exporter(self, exporter, force=False, active='true', tasks=[]): Parameters ========== - task: the Task object to add, should have a name and params and + exporter: the Task object to add, should have a name and params and be child of watchme.tasks.TaskBase force: if task already exists, overwrite active: add the task as active (default "true") @@ -163,29 +186,88 @@ def _add_exporter(self, exporter, force=False, active='true', tasks=[]): # Don't overwrite a section that already exists if exporter.name in self.config.sections(): if not force: - bot.exit('%s exists, use --force to overwrite.' % task.name) + bot.exit('%s exists, use --force to overwrite.' % exporter.name) self.remove_section(exporter.name, save=False) # Add the new section self.config[exporter.name] = exporter.export_params(active=active) self.print_section(exporter.name) - # Add the exporter to any tasks - for task in tasks: - if task in self.config.sections(): - exporters = self.get_setting(task, "exporters", default=[]) - if exporter.name not in exporters: - exporters.append(exporter.name) - self.set_setting(task, "exporters", exporters) - else: - bot.warning("Task %s not found installed, skipping adding exporter to it." % task) - # Save all changes self.save() - # Commit changes - git_commit(repo=self.repo, task=self.name, - message="ADD exporter %s" % exporter.name) + +def add_task_exporter(self, task, name): + '''append an exporter to a task exporters list, if it isn't already + included. Since the configparser supports strings, the list + of exporters is a list of comma separated values. + + Parameters + ========== + task: the name of the task to add the exporter to + name: the name of the exporter to add, if not already there + ''' + if task in self.config.sections(): + exporters = self.get_setting(task, "exporters", default="") + exporters = [e for e in exporters.split(',') if e] + if name not in exporters: + exporters.append(name) + self.set_setting(task, "exporters", ','.join(exporters)) + else: + bot.warning("Task %s not found installed, skipping adding exporter to it." % task) + + +def remove_task_exporter(self, task, name): + '''remove an exporter from a task exporters list + + Parameters + ========== + task: the name of the task to add the exporter to + name: the name of the exporter to add, if not already there + ''' + if task in self.config.sections(): + exporters = self.get_setting(task, "exporters", default="") + + if name in exporters: + bot.info("Removing %s from %s" %(name, task)) + exporters = re.sub(exporters, name, "").strip(",") + + # If no more exporters, remove the param entirely + if len(exporters) == 0: + self.remove_setting(task, 'exporters') + + # Otherwise, update the shorter list + else: + self.set_setting(task, 'exporters', exporters) + self.save() + + else: + bot.warning("Task %s not found installed, skipping removing exporter from it." % task) + + +def remove_exporter(self, name): + '''remove a an exporter from the watcher repo, if it exists, along with + any tasks that it is added to. + + Parameters + ========== + name: the name of the exporter to remove + ''' + if self.get_section(name) != None: + if self.is_frozen(): + bot.exit('watcher is frozen, unfreeze first.') + self.remove_section(name) + + # Remove the exporter from any tasks + for task in self.get_tasks(quiet=True, active=False): + self.remove_task_exporter(task.name, name) + + bot.info('%s removed successfully.' % name) + git_commit(self.repo, self.name, "REMOVE exporter %s" % name) + + else: + bot.warning('Exporter %s does not exist.' % name) + # Get Exporters @@ -245,8 +327,9 @@ def export_runs(self, results): task = self.get_task(name, save=True) # Get exporters added to task - exporters = self.get_setting(task.name, 'exporters', []) - + exporters = self.get_setting(task.name, 'exporters', "") + exporters = exporters.split(",") + # Validate each exporter exists and is active, then run. for exporter in exporters: @@ -284,7 +367,7 @@ def export_runs(self, results): # Only save if the export type is not json, and the result is a text string elif not task.params.get('save_as') == 'json' and not os.path.exists(result[0]): - bot.debug('Exporting list to ' + client.name) + bot.info('Exporting list to ' + client.name) client._save_text_list(name, result) # Case 2. The result is a string @@ -292,6 +375,7 @@ def export_runs(self, results): # Only export if it's not a file path (so it's a string) if not(os.path.exists(result)): + bot.info('Exporting text to ' + client.name) client._save_text(result) # Case 3. The result is a dictionary or a file, ignore for now. diff --git a/watchme/watchers/urls/helpers.py b/watchme/watchers/urls/helpers.py index c4194f2..5ad4f20 100644 --- a/watchme/watchers/urls/helpers.py +++ b/watchme/watchers/urls/helpers.py @@ -9,6 +9,7 @@ ''' import os +import re import tempfile import requests From c180c0c29a8942029501260af121c878930ef48e Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Tue, 7 May 2019 15:05:14 -0400 Subject: [PATCH 17/19] add dummy push function, need to think about it Signed-off-by: Vanessa Sochat --- watchme/client/__init__.py | 5 ++- watchme/client/push.py | 16 +++++--- watchme/watchers/__init__.py | 2 + watchme/watchers/data.py | 79 ++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 7 deletions(-) diff --git a/watchme/client/__init__.py b/watchme/client/__init__.py index 658c12a..0b0447d 100644 --- a/watchme/client/__init__.py +++ b/watchme/client/__init__.py @@ -108,7 +108,7 @@ def get_parser(): help="push data to an extra exporter") push.add_argument('watcher', nargs=1, - help='the watcher export data from') + help='the watcher to push data for') push.add_argument('task', nargs=1, help='the name of the task to push data for') @@ -116,6 +116,9 @@ def get_parser(): push.add_argument('exporter', nargs=1, help='the name of the exporter to push to') + push.add_argument('filename', nargs=1, + help='the name of the file to export data from.') + push.add_argument('--all', dest="all", help="instead of last timepoint, push all temporal data.", default=False, action='store_true') diff --git a/watchme/client/push.py b/watchme/client/push.py index caba317..389728d 100644 --- a/watchme/client/push.py +++ b/watchme/client/push.py @@ -21,9 +21,10 @@ def main(args, extra): name = args.watcher[0] task = args.task[0] exporter = args.exporter[0] + filename = args.filename[0] # Example command to show task- and exporter- - example = 'watchme push watcher task-reddit exporter-pushgateway' + example = 'watchme push ' if not task.startswith('task'): bot.exit('Task name must start with "task", e.g., %s' % example) @@ -35,9 +36,12 @@ def main(args, extra): watcher = get_watcher(name, base=args.base, create=False) bot.info("push is not written yet.") + # Push to the exporter (not the default git) - #result = watcher.push(task=task, - # exporter=exporter, - # name=name, - # export_json=args.json, - # base=args.base) + result = watcher.push(task=task, + exporter=exporter, + filename=filename, + name=name, + push_all=args.all, + push_json=args.json, + base=args.base) diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index 5dbc10d..284c75e 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -31,6 +31,7 @@ export_runs, get_exporter, add_exporter, + has_exporter, remove_exporter, remove_task_exporter, _add_exporter, @@ -873,6 +874,7 @@ def __str__(self): Watcher.add_exporter = add_exporter Watcher.add_task_exporter = add_task_exporter Watcher.get_exporter = get_exporter +Watcher.has_exporter = has_exporter Watcher._add_exporter = _add_exporter Watcher.remove_exporter = remove_exporter Watcher.remove_task_exporter = remove_task_exporter diff --git a/watchme/watchers/data.py b/watchme/watchers/data.py index e2bb4d2..5a068e4 100644 --- a/watchme/watchers/data.py +++ b/watchme/watchers/data.py @@ -98,6 +98,77 @@ def export_dict(self, task, base=args.base) +def push(self, task, exporter, + name=None, + filename=None, + push_json=False, + push_all=True, + from_commit=None, + to_commit=None, + base=None): + '''Manually push a watcher task data file to an exporter endpoint. + + Parameters + ========== + task: the task folder for the watcher to look in + exporter: the exporter to use + name: the name of the watcher, defaults to the client's + push_json: provide json to the exporter to push. + push_all: export all data (and not just the last timepoint) + from_commit: the commit to start at + to_commit: the commit to go to + filename: the filename to filter to. Includes all files if not specified. + ''' + if name == None: + name = self.name + + if base == None: + base = self.base + + # Quit early if the task isn't there + if not self.has_task(task): + bot.exit('%s is not a valid task for %s' % (task, name)) + + # Also quit if the exporter isn't there + if not self.has_exporter(exporter): + bot.exit('%s is not a valid exporter for task %s' % (exporter, task)) + + repo = os.path.join(base, self.name) + if not os.path.exists(repo): + bot.exit('%s does not exist.' % repo) + + filepath = os.path.join(base, self.name, task, filename) + + # Ensure that the filename exists in the repository + if not os.path.exists(filepath): + bot.exit('%s does not exist for watcher %s' %(filepath, name)) + + # Now filepath must be relative to the repo + filepath = os.path.join(task, filename) + + commits = get_commits(repo=repo, + from_commit=from_commit, + to_commit=to_commit, + grep="EXPORT results %s" % task, + filename=filepath) + + # Keep lists of commits, dates, content + result = {'commits': [], 'dates': [], 'content': []} + + bot.warning("Function not yet written") + # Empty content (or other) returns None +# for commit in commits: +# content = git_show(repo=repo, commit=commit, filename=filepath) + +# if export_json is True: +# content = json.loads(content) + +# result['content'].append(content) +# result['dates'].append(git_date(repo=repo, commit=commit)) +# result['commits'].append(commit) + return result + + # Exporter Functions def add_exporter(self, name, exporter_type, params, tasks, force=False, active="true"): @@ -308,6 +379,14 @@ def get_exporter(self, name): return exporter +def has_exporter(self, name): + '''returns True or False to indicate if the watcher has a specific exporter + ''' + self.load_config() + if self.has_section(name) and name.startswith('exporter'): + return True + return False + def export_runs(self, results): ''' export data retrieved to the set of exporters defined and active, From dbd68bd5cce61fc5fefd90f19e52e6a486650c69 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Tue, 7 May 2019 17:38:18 -0400 Subject: [PATCH 18/19] adding push Signed-off-by: Vanessa Sochat --- docs/_docs/exporters/index.md | 14 +++ watchme/client/__init__.py | 4 - watchme/client/push.py | 5 +- watchme/exporters/__init__.py | 21 +++++ watchme/exporters/pushgateway/__init__.py | 65 ++++++++++++-- watchme/tasks/__init__.py | 71 ++++++++++++++- watchme/watchers/__init__.py | 65 ++------------ watchme/watchers/data.py | 100 +++++++--------------- 8 files changed, 199 insertions(+), 146 deletions(-) diff --git a/docs/_docs/exporters/index.md b/docs/_docs/exporters/index.md index ff56aff..e18b766 100644 --- a/docs/_docs/exporters/index.md +++ b/docs/_docs/exporters/index.md @@ -99,3 +99,17 @@ $ watchme run If an exporter doesn't run successfully, it will be disabled for the task. This is in case you forget about it. + + +## Push Manually + +Optionally, you can choose to push data to an exporter manually. +This will ignore if the exporter is active, and only check that it's +defined for the task and is valid. + +```bash +$ watchme push +``` + +If you want to customize how the export works (e.g., what commits) +then you can use the python functions directly. diff --git a/watchme/client/__init__.py b/watchme/client/__init__.py index 0b0447d..abf62fa 100644 --- a/watchme/client/__init__.py +++ b/watchme/client/__init__.py @@ -119,10 +119,6 @@ def get_parser(): push.add_argument('filename', nargs=1, help='the name of the file to export data from.') - push.add_argument('--all', dest="all", - help="instead of last timepoint, push all temporal data.", - default=False, action='store_true') - push.add_argument('--json', dest="json", help="signal to load the file as json", default=False, action='store_true') diff --git a/watchme/client/push.py b/watchme/client/push.py index 389728d..ede8380 100644 --- a/watchme/client/push.py +++ b/watchme/client/push.py @@ -35,13 +35,10 @@ def main(args, extra): # Get the watcher to interact with, must already exist watcher = get_watcher(name, base=args.base, create=False) - bot.info("push is not written yet.") - # Push to the exporter (not the default git) result = watcher.push(task=task, exporter=exporter, filename=filename, name=name, - push_all=args.all, - push_json=args.json, + export_json=args.json, base=args.base) diff --git a/watchme/exporters/__init__.py b/watchme/exporters/__init__.py index d8e6464..2ad069c 100644 --- a/watchme/exporters/__init__.py +++ b/watchme/exporters/__init__.py @@ -43,6 +43,27 @@ def set_params(self, params): self.params[key] = value + def push(self, result): + '''push dummy function, optional for subclass to implement. + ''' + bot.warning('push is not implemented for %s' % self.type) + + + def export(self, result, task): + '''the export function is the entrypoint to export data for an + exporter. Based on the data type, we call any number of supporting + functions. If True is returned, the data is exported. If False is + returned, there was an error. If None is returned, there is no + exporter defined for the data type. + + Parameters + ========== + result: the result object to export, a string, list, dict, or file + task: the task object associated. + ''' + bot.warning('export is not implemented for %s' % self.type) + + def export_params(self, active="true"): '''export parameters, meaning returning a dictionary of the task parameters plus the addition of the task type and active status. diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py index 648041b..5e90980 100644 --- a/watchme/exporters/pushgateway/__init__.py +++ b/watchme/exporters/pushgateway/__init__.py @@ -37,16 +37,64 @@ def _validate(self): if not self.params['url'].startswith('http'): bot.exit("url must start with http, found %s" % self.params['url']) +# Push Functions + + def push(self, result, task): + '''push dummy function, expects the dictionary with commits, dates, + and results. Since pushgateway only takes numbers, we parse the + list of content. + ''' + if "content" in result: + return self._save_text_list(task.name, result['content']) + + +# Export Functions + + def export(self, result, task): + '''the export function is the entrypoint to export data for an + exporter. Based on the data type, we call any number of supporting + functions. If True is returned, the data is exported. If False is + returned, there was an error. If None is returned, there is no + exporter defined for the data type. + ''' + # Case 1. The result is a list + if isinstance(result, list): + + # Get rid of Nones, if the user accidentally added + result = [r for r in result if r] + + if len(result) == 0: + bot.error('%s returned empty list of results.' % name) + + # Only save if the export type is not json, and the result is a text string + elif not task.params.get('save_as') == 'json' and not os.path.exists(result[0]): + bot.info('Exporting list to ' + client.name) + return self._save_text_list(task.name, result) + + # Case 2. The result is a string + elif isinstance(result, str): + + # Only export if it's not a file path (so it's a string) + if not(os.path.exists(result)): + bot.info('Exporting text to ' + client.name) + return self._save_text(result) + + # Case 3. The result is a dictionary or a file, ignore for now. + else: + bot.warning('Files/dictionary are not currently supported for export') + def _save_text_list(self, name, results): '''for a list of general text results, send them to a pushgateway. - + for any error, the calling function should return False immediately. + Parameters ========== results: list of string results to write to the pushgateway ''' for r in range(len(results)): - self._write_to_pushgateway(results[r]) - + if not self._write_to_pushgateway(results[r]): + return False + return True def _save_text(self, result): '''exports the text to the exporter @@ -55,8 +103,7 @@ def _save_text(self, result): ========== result: the result object to save, not a path to a file in this case ''' - self._write_to_pushgateway(result) - + return self._write_to_pushgateway(result) def _write_to_pushgateway(self, result): ''' writes data to the pushgateway @@ -66,15 +113,15 @@ def _write_to_pushgateway(self, result): result: the result object to save ''' from prometheus_client import Gauge, push_to_gateway - g = Gauge(self.name.replace('-', ':'), '', registry=self.registry) - g.set(result) try: + g = Gauge(self.name.replace('-', ':'), '', registry=self.registry) + g.set(result) push_to_gateway(self.params['url'], job='watchme', registry=self.registry) except: bot.error('An exception occurred while trying to export data using %s' % self.name) + return False - #TODO: disable task, and add a --test command - #TODO: need commands to add/remove exporters from a task + return True diff --git a/watchme/tasks/__init__.py b/watchme/tasks/__init__.py index e44635e..39654b0 100644 --- a/watchme/tasks/__init__.py +++ b/watchme/tasks/__init__.py @@ -107,6 +107,75 @@ def run(self): bot.error('Cannot find function.') +# Global Save function to write results + + def write_results(self, result, repo): + '''an entrypoint function for a general task. By default, we parse + results based on the result type. Any particular subclass of the + TaskBase can modify or extend these functions. + + Parameters + ========== + result: the result object to parse + repo: the repo base (watcher.repo) + ''' + files = [] + + # Case 1. The result is a list + if isinstance(result, list): + # Get rid of Nones, if the user accidentally added + result = [r for r in result if r] + + if len(result) == 0: + bot.error('%s returned empty list of results.' % name) + + # json output is specified + elif self.params.get('save_as') == 'json': + bot.debug('Saving single list as one json...') + files.append(self._save_json(result, repo)) + + elif self.params.get('save_as') == 'json': + bot.debug('Saving single list as multiple json...') + files += self._save_json_list(result, repo) + + # Otherwise, sniff for list of paths + elif os.path.exists(result[0]): + bot.debug('Found list of paths...') + files += self._save_files_list(result, repo) + + # Finally, assume just writing text to file + else: + bot.debug('Saving content from list to file...') + files += self._save_text_list(result, repo) + + # Case 2. The result is a string + elif isinstance(result, str): + + # if it's a path to a file, just save to repository + if os.path.exists(result): + files.append(self._save_file(result, repo)) + + # Otherwise, it's a string that needs to be saved to file + else: + files.append(self._save_text(result, repo)) + + # Case 3. The result is a dictionary + elif isinstance(result, dict): + files.append(self._save_json(result,repo)) + + elif result == None: + bot.error('Result for task %s is None' % self.name) + + elif hasattr(self, '_write_results'): + return self._write_results(result) + + else: + bot.error('Unsupported result format %s' % type(result)) + + # Get rid of None results (don't check excessively for None above) + files = [f for f in files if f] + return files + # Saving @@ -217,5 +286,3 @@ def _save_text_list(self, results, repo): ''' file_name = self.params.get('file_name', 'result.txt') return self._save_list(results, repo, self._save_text, file_name) - - diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index 284c75e..5926e12 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -552,7 +552,7 @@ def has_task(self, name): return True return False - def get_task(self, name, save=False): + def get_task(self, name): '''get a particular task, based on the name. This is where each type of class should check the "type" parameter from the config, and import the correct Task class. @@ -560,7 +560,6 @@ def get_task(self, name, save=False): Parameters ========== name: the name of the task to load - save: if saving, will be True ''' self.load_config() @@ -586,7 +585,7 @@ def get_task(self, name, save=False): bot.exit('Type %s not properly set up in get_task' % task_type) # if not valid, will return None - task = Task(name, params, _save=save) + task = Task(name, params) return task @@ -756,8 +755,7 @@ def run(self, regexp=None, parallel=True, test=False, show_progress=True): else: # or print results to the screen print(json.dumps(results, indent=4)) - - + def finish_runs(self, results): '''finish runs should take a dictionary of results, with keys as the folder name, and for each, depending on the result type, @@ -771,65 +769,15 @@ def finish_runs(self, results): ''' for name, result in results.items(): task_folder = os.path.join(self.repo, name) - task = self.get_task(name, save=True) - - # Files to be added via Git after - files = [] + task = self.get_task(name) # Ensure that the task folder exists if not os.path.exists(task_folder): mkdir_p(task_folder) git_add(self.repo, task_folder) - # Case 1. The result is a list - if isinstance(result, list): - # Get rid of Nones, if the user accidentally added - result = [r for r in result if r] - - if len(result) == 0: - bot.error('%s returned empty list of results.' % name) - - # json output is specified - elif task.params.get('save_as') == 'json': - bot.debug('Saving single list as one json...') - files.append(task._save_json(result, self.repo)) - - elif task.params.get('save_as') == 'json': - bot.debug('Saving single list as multiple json...') - files += task._save_json_list(result, self.repo) - - # Otherwise, sniff for list of paths - elif os.path.exists(result[0]): - bot.debug('Found list of paths...') - files += task._save_files_list(result, self.repo) - - # Finally, assume just writing text to file - else: - bot.debug('Saving content from list to file...') - files += task._save_text_list(result, self.repo) - - # Case 2. The result is a string - elif isinstance(result, str): - # if it's a path to a file, just save to repository - if os.path.exists(result): - files.append(task._save_file(result, self.repo)) - - # Otherwise, it's a string that needs to be saved to file - else: - files.append(task._save_text(result, self.repo)) - - # Case 3. The result is a dictionary - elif isinstance(result, dict): - files.append(task._save_json(result, self.repo)) - - elif result == None: - bot.error('Result for task %s is None' % name) - - else: - bot.error('Unsupported result format %s' % type(result)) - - # Get rid of None results (don't check excessively for None above) - files = [f for f in files if f] + # Files to be added to the repo via git after + files = task.write_results(result, self.repo) # Add files to git, and commit files.append(write_timestamp(repo=self.repo, task=name)) @@ -871,6 +819,7 @@ def __str__(self): Watcher.schedule = schedule # Exporters + Watcher.add_exporter = add_exporter Watcher.add_task_exporter = add_task_exporter Watcher.get_exporter = get_exporter diff --git a/watchme/watchers/data.py b/watchme/watchers/data.py index 5a068e4..8fde598 100644 --- a/watchme/watchers/data.py +++ b/watchme/watchers/data.py @@ -101,8 +101,7 @@ def export_dict(self, task, def push(self, task, exporter, name=None, filename=None, - push_json=False, - push_all=True, + export_json=False, from_commit=None, to_commit=None, base=None): @@ -113,60 +112,39 @@ def push(self, task, exporter, task: the task folder for the watcher to look in exporter: the exporter to use name: the name of the watcher, defaults to the client's - push_json: provide json to the exporter to push. + export_json: read the filename data as json (not text) push_all: export all data (and not just the last timepoint) from_commit: the commit to start at to_commit: the commit to go to filename: the filename to filter to. Includes all files if not specified. ''' - if name == None: - name = self.name - - if base == None: - base = self.base - - # Quit early if the task isn't there - if not self.has_task(task): - bot.exit('%s is not a valid task for %s' % (task, name)) - - # Also quit if the exporter isn't there + # Quit if the exporter isn't there if not self.has_exporter(exporter): bot.exit('%s is not a valid exporter for task %s' % (exporter, task)) - repo = os.path.join(base, self.name) - if not os.path.exists(repo): - bot.exit('%s does not exist.' % repo) + # Get the exporter, ensure still valid. + exporter = self.get_exporter(exporter) + task_instance = self.get_task(task) - filepath = os.path.join(base, self.name, task, filename) + if task_instance == None: + bot.exit("Task %s does not exist." % task) - # Ensure that the filename exists in the repository - if not os.path.exists(filepath): - bot.exit('%s does not exist for watcher %s' %(filepath, name)) + if exporter.valid is False: + bot.exit("Exporter %s is not valid." % exporter.name) - # Now filepath must be relative to the repo - filepath = os.path.join(task, filename) + # Make sure the exporter is added to the task + if exporter.name not in task_instance.params.get('exporters', ''): + bot.exit("Exporter %s is not added to task %s" %(exporter.name, task)) - commits = get_commits(repo=repo, - from_commit=from_commit, - to_commit=to_commit, - grep="EXPORT results %s" % task, - filename=filepath) + result = self.export_dict(task=task_instance.name, + filename=filename, + name=name, + export_json=export_json, + from_commit=from_commit, + to_commit=to_commit, + base=base) - # Keep lists of commits, dates, content - result = {'commits': [], 'dates': [], 'content': []} - - bot.warning("Function not yet written") - # Empty content (or other) returns None -# for commit in commits: -# content = git_show(repo=repo, commit=commit, filename=filepath) - -# if export_json is True: -# content = json.loads(content) - -# result['content'].append(content) -# result['dates'].append(git_date(repo=repo, commit=commit)) -# result['commits'].append(commit) - return result + exporter.push(result, task_instance) # Exporter Functions @@ -403,11 +381,11 @@ def export_runs(self, results): ''' for name, result in results.items(): - task = self.get_task(name, save=True) + task = self.get_task(name) # Get exporters added to task exporters = self.get_setting(task.name, 'exporters', "") - exporters = exporters.split(",") + exporters = [e for e in exporters.split(",") if e] # Validate each exporter exists and is active, then run. for exporter in exporters: @@ -432,29 +410,13 @@ def export_runs(self, results): # A client with "None" indicates a dependency is likely missing if client == None: - bot.warning("Check dependencies for %s.." % exporter) + bot.warning("Exporter %s does not exist, or check dependencies for it." % exporter) continue - # Case 1. The result is a list - if isinstance(result, list): - - # Get rid of Nones, if the user accidentally added - result = [r for r in result if r] - - if len(result) == 0: - bot.error('%s returned empty list of results.' % name) - - # Only save if the export type is not json, and the result is a text string - elif not task.params.get('save_as') == 'json' and not os.path.exists(result[0]): - bot.info('Exporting list to ' + client.name) - client._save_text_list(name, result) - - # Case 2. The result is a string - elif isinstance(result, str): - - # Only export if it's not a file path (so it's a string) - if not(os.path.exists(result)): - bot.info('Exporting text to ' + client.name) - client._save_text(result) - - # Case 3. The result is a dictionary or a file, ignore for now. + # If we get here, save the result. True = success, False = Fail, None = NA + result = client.export(result, task) + + # If exporter fails, remove it for the task. + if result == False: + bot.warning("Issue with export of %s, removing exporter %s." % (task.name, exporter)) + self.remove_task_exporter(task.name, exporter) From 3494d12373d867d461f8e544d4ad4d93cfb4dca1 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Wed, 8 May 2019 11:56:58 -0400 Subject: [PATCH 19/19] seems to be a circle blip Signed-off-by: Vanessa Sochat --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 40b83f6..72d6612 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,7 @@ install_python_3: &install_python_3 fi install_python_2: &install_python_2 - name: install Python 3.5 dependencies + name: install Python 2 dependencies command: | ls $HOME if [ ! -d "/home/circleci/conda" ]; then