diff --git a/.circleci/config.yml b/.circleci/config.yml index 40b83f6..72d6612 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,7 @@ install_python_3: &install_python_3 fi install_python_2: &install_python_2 - name: install Python 3.5 dependencies + name: install Python 2 dependencies command: | ls $HOME if [ ! -d "/home/circleci/conda" ]; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f1cd83..aa7fabd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Critical items to know are: - changed behaviour ## [master](https://github.com/vsoch/watchme/tree/master) + - Addition of exporters (0.0.17) - Adding option for regular expression for URL wachers, user agent header (0.0.16) - requests is missing from install dependencies (0.0.15) - small bug fixes (0.0.14) diff --git a/docs/_data/links.yml b/docs/_data/links.yml index 83b30a5..2b618d4 100644 --- a/docs/_data/links.yml +++ b/docs/_data/links.yml @@ -12,5 +12,7 @@ navigation: url: watchers/ - name: Examples url: examples/ +- name: Exporters + url: exporters/ - name: Contributing url: contributing/ diff --git a/docs/_docs/exporters/index.md b/docs/_docs/exporters/index.md new file mode 100644 index 0000000..e18b766 --- /dev/null +++ b/docs/_docs/exporters/index.md @@ -0,0 +1,115 @@ +--- +title: WatchMe Exporters +category: Exporters +permalink: /exporters/index.html +order: 1 +--- + +Once you've followed the [getting started]({{ site.baseurl }}/getting-started/) +guide to [install]({{ site.baseurl }}/install/) watchme and have created your first +set of watchers, you might want to do more! Specifically, you can add +extra exporters to push your data to custom locations. The following exporters +are available: + + - [pushgateway]({{ site.baseurl }}/exporters/pushgateway/) to push data to a Prometheus gateway. + +And instructions for using a general exporter, either in sync with running a task +or separately, are provided below. + + +## List Exporters + +To see exporters available, you can list them: + +```bash +$ watchme list --exporters +watchme: exporters +pushgateway +``` + +An exporter is like a task in that it can be activated +(or not), and you can define more than one for any particular watcher. + +## Add an Exporter + +The most basic thing to do is to add an exporter to a watcher. You should +read the documentation for your exporter of interest (linked above) to +know the required parameters for the exporter (it varies). +Adding an exporter is similar to adding a task! The command asks you to +name the exporter, and the watcher to which you are adding it: + +```bash +$ watchme add-exporter +``` + +Most likely the exporter requires parameters, so you add these in the same way +you added parameters to a task. Notice that "type" is essential to designate the +type that you want (the list at the top of the page): + +```bash +$ watchme add-exporter exporter- param@value type@ +``` + +If you already have tasks defined that you want the exporter added to, just add them to the command: + +```bash +$ watchme add-exporter exporter- param@value type@ +``` + +If you've already added the exporter to the watcher and want to add it to a task: + +```bash +$ watchme add-exporter +``` + +Watchme will know that you aren't adding a new exporter (but instead using +an existing one) because you haven't provided a `type@` +variable. If you want to re-create an exporter that already exists, you +can issue the same command but you'll need `--force`. + +```bash +$ watchme add-exporter exporter- param@value type@ --force +``` + +## Remove an Exporter + +If you change your mind, you can remove an exporter from a watcher entirely: + +```bash +$ watchme remove +``` + +You can also remove it from a specific task (but not from the watcher entirely) + +```bash +$ watchme remove +``` + + +## Run the Exporter + +By default, an exporter will run with a task. This means that you should +run the task to run the exporter. If the watcher is active, and the task +is active, and the exporter is defined for the task, it will run. + +```bash +$ watchme activate +$ watchme run +``` + +If an exporter doesn't run successfully, it will be disabled for the task. +This is in case you forget about it. + + +## Push Manually + +Optionally, you can choose to push data to an exporter manually. +This will ignore if the exporter is active, and only check that it's +defined for the task and is valid. + +```bash +$ watchme push +``` + +If you want to customize how the export works (e.g., what commits) +then you can use the python functions directly. diff --git a/docs/_docs/exporters/pushgateway/pushed.png b/docs/_docs/exporters/pushgateway/pushed.png new file mode 100644 index 0000000..5692063 Binary files /dev/null and b/docs/_docs/exporters/pushgateway/pushed.png differ diff --git a/docs/_docs/exporters/pushgateway/pushgateway.md b/docs/_docs/exporters/pushgateway/pushgateway.md new file mode 100644 index 0000000..1f15eae --- /dev/null +++ b/docs/_docs/exporters/pushgateway/pushgateway.md @@ -0,0 +1,243 @@ +--- +title: Pushgateway +category: Exporters +permalink: /exporters/pushgateway/ +order: 2 +--- + +The [pushgateway](https://github.com/prometheus/pushgateway) exporter +will allow you to export data for a watcher to a Prometheus gateway. +To install the exporter, you will need to install its dependencies: + +```bash +pip install watchme[exporter-pushgateway] +``` + +The dependencies include the [prometheus client](https://github.com/prometheus/client_python). + +
+ +## Quick Start + +Setup a gateway + +```bash +$ docker run -d -p 9091:9091 prom/pushgateway +``` + +Create and add the exporter to an existing task. The only required parameter is the +url for the gateway (above we use http://localhost:9091). + +```bash +$ watchme add-exporter exporter-pushgateway url@http://localhost:9091 type@pushgateway task- +``` + +
+ +## Detailed Start + +### 1. Setup a Gateway + +To set up a gateway, you can find full instructions (section "Run It") +at [https://github.com/prometheus/pushgateway](https://github.com/prometheus/pushgateway). +A brief summary using Docker is provided here. First, bring up the gateway: + +```bash +$ docker run -d -p 9091:9091 prom/pushgateway +``` + +The image will pull and run, exposing the gateway on port 9091. First use +`docker ps` to ensure that it's running without issue: + +```bash +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +fd6e3d3fb8db prom/pushgateway "/bin/pushgateway" 7 seconds ago Up 6 seconds 0.0.0.0:9091->9091/tcp gifted_merkle +``` + +Then, open your browser to [http://localhost:9091/](http://localhost:9091/). You should +see an empty portal. + +![pushgateway.png]({{ site.url }}{{ site.baseurl }}/exporters/pushgateway/pushgateway.png) + +It's a fairly boring interface, but that's okay - we will fix it up when we push data to it! + + +### 2. Create a Watcher + +Next, let's create a watcher that will collect the data that we want to push to the +exporter. Imagine that we are interested in monitoring the temperate for a region. +Let's call the watcher "weather-watcher" + +```bash +$ watchme create temp-watcher +Adding watcher /home/vanessa/.watchme/temp-watcher... +Generating watcher config /home/vanessa/.watchme/temp-watcher/watchme.cfg +``` + +This is an empty watcher in that there aren't any tasks to run. + +### 3. Create a Task +Let's create a task to get the temperature for Luxembourg. + +```bash +$ watchme add-task temp-watcher task-luxembourg url@https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 selection@.local-temp get_text@true func@get_url_selection type@urls regex@[0-9]+ + +[task-luxembourg] +url = https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 +selection = .local-temp +get_text = true +func = get_url_selection +regex = [0-9]+ +active = true +type = urls +``` + +Above, we create a task called "task-luxembourg" that will go to the URL with the weather, +select the class ".local-temp" on the page, get the text for the class, and filter to the regular expression +for one or more numbers (`[0-9]+`). If you are interested in these details, see the +[urls watcher tasks]({{ site.url }}/watchers/urls/). + +#### Test the task + +We haven't scheduled it to run, but we might try testing it out to make sure +that it runs as we expect. + +```bash +$ watchme run temp-watcher task-luxembourg --test +Found 1 contender tasks. +[1/1] |===================================| 100.0% +{ + "task-luxembourg": [ + "54" + ] +} +``` + +The `--test` flag will ensure that the data is not saved and written to git. +The above worked great, because we see that the temperature is 54. + +### 4. Add an Exporter + +Let's now add an exporter. An exporter is like a task in that it can be activated +(or not), and you can define more than one for any particular watcher. +If you want to see the exporters available: + +```bash +$ watchme list --exporters +watchme: exporters +pushgateway +``` + +Adding an exporter is similar to adding a task! The command asks you to +name the exporter, and the watcher to which you are adding it: + +```bash +$ watchme add-exporter +``` + +And you can follow these labels by any parameters required for the exporter. +For the pushgateway exporter, along with specifying the type, +you are only required one parameter (url) that must start with http. + + +```bash +$ watchme add-exporter temp-watcher exporter-pushgateway url@http://localhost:9091 type@pushgateway +``` + +Along with seeing it on the screen, you can inspect the watcher to see that +the exporter is added: + +```bash +$ watchme inspect temp-watcher +[watcher] +active = false +type = urls + +[task-luxembourg] +url = https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 +selection = .local-temp +get_text = true +func = get_url_selection +regex = [0-9]+ +active = true +type = urls + +[exporter-pushgateway] +url = http://localhost:9091 +active = true +type = pushgateway +``` + +But notice how the exporter isn't added to the task? We can add it manually: + +```bash +# watchme add-exporter +$ watchme add-exporter temp-watcher exporter-pushgateway task-luxembourg +``` + +Or you could have added it when you first created the exporter. Let's show you +the command to remove the exporter: + +```bash +$ watchme remove temp-watcher exporter-pushgateway +Removing exporter-pushgateway from task-luxembourg +exporter-pushgateway removed successfully. +``` + +And then add it again, not only to the watcher but also to the task, with one command! + +```bash +$ watchme add-exporter temp-watcher exporter-pushgateway url@http://localhost:9091 type@pushgateway task-luxembourg +[exporter-pushgateway] +url = http://localhost:9091 +active = true +type = pushgateway +``` + +You can then inspect the watcher to see the full addition (note that "exporter-pushgateway" is added +to exporters for task-luxembourg: + +```bash +vanessa@vanessa-ThinkPad-T460s:~/Documents/Dropbox/Code/Python/watchme$ watchme inspect temp-watcher[watcher] +active = false +type = urls + +[task-luxembourg] +url = https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 +selection = .local-temp +get_text = true +func = get_url_selection +regex = [0-9]+ +active = true +type = urls +exporters = exporter-pushgateway + +[exporter-pushgateway] +url = http://localhost:9091 +active = true +type = pushgateway +``` + +### 5. Run the Exporter + +We can see that pushgateway is active, so when we run the task, it should push +data there. To do it fully, we need to activate the watcher: + +```bash +$ watchme activate temp-watcher +[watcher|temp-watcher] active: true +``` + +And then run the task. + +```bash +$ watchme run temp-watcher task-luxembourg +Exporting list to exporter-pushgateway +``` + +If all goes well, you can return to the web interface and see your data added! + +![pushed.png]({{ site.url }}{{ site.baseurl }}/exporters/pushgateway/pushed.png) + +Great job! You can learn more about the use cases for [push gateway here](https://prometheus.io/docs/practices/pushing/). diff --git a/docs/_docs/exporters/pushgateway/pushgateway.png b/docs/_docs/exporters/pushgateway/pushgateway.png new file mode 100644 index 0000000..85dd767 Binary files /dev/null and b/docs/_docs/exporters/pushgateway/pushgateway.png differ diff --git a/docs/_docs/getting-started/index.md b/docs/_docs/getting-started/index.md index 3f2fce9..cf4ce1f 100644 --- a/docs/_docs/getting-started/index.md +++ b/docs/_docs/getting-started/index.md @@ -637,6 +637,15 @@ as a single (loadable) json file: $ watchme export system task-cpu vanessa-thinkpad-t460s_vanessa.json --json ``` +### Advanced Export + +What if .git doesn't work for you, or you want to send data elsewhere? We had +a fantastic idea and contribution in [this thread](https://github.com/vsoch/watchme/issues/30) +that resulted in the creation of exporters, or additional exports to make +other than using GitHub. Each exporter is considered to be a plugin (akin to +a watcher) and can be optionally installed and used. For more information, +see the [exporters]({{ site.url }}/exporters) page. + ## Licenses This code is licensed under the Mozilla, version 2.0 or later [LICENSE](LICENSE). diff --git a/docs/_docs/install/index.md b/docs/_docs/install/index.md index 2c44741..a6d4a9a 100644 --- a/docs/_docs/install/index.md +++ b/docs/_docs/install/index.md @@ -10,8 +10,8 @@ order: 1 The only dependency for watchme is to have [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [crontab](https://www.digitalocean.com/community/tutorials/how-to-use-cron-to-automate-tasks-on-a-vps) on your system. Git is used for version control of the pages you are watching, and crontab is -used for scheduling your watches. If you want to install a custom watcher type, -see [installing extras](#installing-extras) below. +used for scheduling your watches. If you want to install a custom exporter or +watcher, see [installing extras](#installing-extras) below. ## Install @@ -78,6 +78,18 @@ To install all watchers only: $ pip install watchme[watchers] ``` +To install all exporters only: + +```bash +$ pip install watchme[exporters] +``` + +To install a specific exporter: + +```bash +$ pip install watchme[exporter-pushgateway] +``` + or a specific watcher task group: ```bash diff --git a/setup.py b/setup.py index 40f514b..5fb898a 100644 --- a/setup.py +++ b/setup.py @@ -81,11 +81,15 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): INSTALL_REQUIRES = get_requirements(lookup) INSTALL_ALL = get_requirements(lookup, 'INSTALL_ALL') WATCHERS = get_requirements(lookup, 'INSTALL_WATCHERS') + EXPORTERS = get_requirements(lookup, 'INSTALL_EXPORTERS') # Watchers URLS_DYNAMIC = get_requirements(lookup, 'INSTALL_URLS_DYNAMIC') PSUTILS = get_requirements(lookup, 'INSTALL_PSUTILS') + # Exporters + PUSHGATEWAY = get_requirements(lookup, 'INSTALL_PUSHGATEWAY') + setup(name=NAME, version=VERSION, author=AUTHOR, @@ -106,8 +110,10 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): extras_require={ 'all': [INSTALL_ALL], 'watchers': [WATCHERS], + 'exporters': [EXPORTERS], 'watcher-urls-dynamic': [URLS_DYNAMIC], - 'watcher-psutils': [PSUTILS] + 'watcher-psutils': [PSUTILS], + 'exporter-pushgateway': [PUSHGATEWAY], }, classifiers=[ 'Intended Audience :: Science/Research', diff --git a/watchme/client/__init__.py b/watchme/client/__init__.py index c570206..abf62fa 100644 --- a/watchme/client/__init__.py +++ b/watchme/client/__init__.py @@ -14,6 +14,7 @@ from watchme.defaults import ( WATCHME_WATCHER, WATCHME_TASK_TYPES, + WATCHME_EXPORTERS, WATCHME_DEFAULT_TYPE ) import watchme @@ -101,6 +102,27 @@ def get_parser(): help="the output file (json) to export the data", default=None) + # push (intended for exporters) + + push = subparsers.add_parser("push", + help="push data to an extra exporter") + + push.add_argument('watcher', nargs=1, + help='the watcher to push data for') + + push.add_argument('task', nargs=1, + help='the name of the task to push data for') + + push.add_argument('exporter', nargs=1, + help='the name of the exporter to push to') + + push.add_argument('filename', nargs=1, + help='the name of the file to export data from.') + + push.add_argument('--json', dest="json", + help="signal to load the file as json", + default=False, action='store_true') + # create @@ -110,28 +132,53 @@ def get_parser(): create.add_argument('watchers', nargs="*", help='watchers to create (default: single watcher)') - # add - add = subparsers.add_parser("add-task", - help="add a task to a watcher.") + # add task - add.add_argument('watcher', nargs=1, - help='the watcher to add to') + add_task = subparsers.add_parser("add-task", + help="add a task to a watcher.") - add.add_argument('task', nargs=1, - help='the name of the task to add. Must start with task') + add_task.add_argument('watcher', nargs=1, + help='the watcher to add to') - add.add_argument('--type', dest="watcher_type", - choices=WATCHME_TASK_TYPES, - default=WATCHME_DEFAULT_TYPE) + add_task.add_argument('task', nargs=1, + help='the name of the task to add.') - add.add_argument('--active', dest="active", - choices=["true", "false"], - default="true") + add_task.add_argument('--type', dest="watcher_type", + choices=WATCHME_TASK_TYPES, + default=WATCHME_DEFAULT_TYPE) - add.add_argument('--force', dest="force", - help="force overwrite a task, if already exists.", - default=False, action='store_true') + add_task.add_argument('--active', dest="active", + choices=["true", "false"], + default="true") + + add_task.add_argument('--force', dest="force", + help="force overwrite a task, if already exists.", + default=False, action='store_true') + + + # add exporter + + add_exporter = subparsers.add_parser("add-exporter", + help="add an exporter to a watcher.") + + add_exporter.add_argument('watcher', nargs=1, + help='the watcher to add to') + + add_exporter.add_argument('name', nargs=1, + help='the name of the exporter to add.') + + add_exporter.add_argument('--type', dest="exporter_type", + choices=WATCHME_EXPORTERS, + default=None) + + add_exporter.add_argument('--active', dest="active", + choices=["true", "false"], + default="true") + + add_exporter.add_argument('--force', dest="force", + help="force overwrite an exporter, if already exists.", + default=False, action='store_true') # inspect @@ -151,6 +198,10 @@ def get_parser(): ls = subparsers.add_parser("list", help="list all watchers at a base") + ls.add_argument('--exporters', dest="exporters", + help="list exporters available", + default=False, action='store_true') + ls.add_argument('--watchers', dest="watchers", help="list watchers available", default=False, action='store_true') @@ -309,7 +360,8 @@ def help(return_code=0): sys.exit(0) if args.command == "activate": from .activate import main - elif args.command == "add-task": from .add import main + elif args.command == "add-task": from .addtask import main + elif args.command == "add-exporter": from .exporter import main elif args.command == "edit": from .edit import main elif args.command == "export": from .export import main elif args.command == "create": from .create import main @@ -319,6 +371,7 @@ def help(return_code=0): elif args.command == "inspect": from .inspect import main elif args.command == "list": from .ls import main elif args.command == "protect": from .protect import main + elif args.command == "push": from .push import main elif args.command == "remove": from .remove import main elif args.command == "run": from .run import main elif args.command == "schedule": from .schedule import main diff --git a/watchme/client/activate.py b/watchme/client/activate.py index 67cab12..ed19687 100644 --- a/watchme/client/activate.py +++ b/watchme/client/activate.py @@ -12,7 +12,7 @@ from watchme.logger import bot def main(args, extra): - '''activate one or more watchers + '''activate one or more watchers, tasks, or exporters ''' # Doesn't work if watcher not provided watcher = args.watcher[0] @@ -22,6 +22,7 @@ def main(args, extra): if extra == None: watcher.activate() else: + # This can be used for tasks and exporters for name in extra: watcher.activate(name) diff --git a/watchme/client/add.py b/watchme/client/addtask.py similarity index 100% rename from watchme/client/add.py rename to watchme/client/addtask.py diff --git a/watchme/client/deactivate.py b/watchme/client/deactivate.py index e683bb1..221c4dc 100644 --- a/watchme/client/deactivate.py +++ b/watchme/client/deactivate.py @@ -22,6 +22,7 @@ def main(args, extra): if extra == None: watcher.deactivate() else: + # This can be used for a task or an exporter for name in extra: watcher.deactivate(name) diff --git a/watchme/client/exporter.py b/watchme/client/exporter.py new file mode 100644 index 0000000..fcd3745 --- /dev/null +++ b/watchme/client/exporter.py @@ -0,0 +1,63 @@ +''' + +Copyright (C) 2019 Vanessa Sochat. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' + +from watchme import get_watcher +from watchme.logger import bot +from watchme.defaults import WATCHME_EXPORTERS + +def main(args, extra): + '''add an exporter to a watcher, optionally with params and tasks + ''' + + # Required - will print help if not provided + name = args.watcher[0] + exporter = args.name[0] + + # Are we adding an extra task or exporter? + if not exporter.startswith('exporter'): + bot.error('Exporter name must start with exporter-, found %s' % exporter) + bot.exit('watchme add-exporter task-a task-b task-c') + + # Extra parameters are parameters for the exporter, or task names + if extra == None: + extra = [] + + # Type can also be an argument (required for exporter) + exporter_type = args.exporter_type + + # Go through extras and determine params and tasks + params = [] + tasks = [] + for param in extra: + if param.startswith('type@'): + exporter_type = param.replace('type@', '') + elif param.startswith('task'): + tasks.append(param) + else: + params.append(param) + + # Get the watcher to interact with, must already exist + watcher = get_watcher(name, base=args.base, create=False) + + # Double check the exporter type + if exporter_type == None: + bot.warning("exporter_type not defined, assumed existing.") + + elif exporter_type not in WATCHME_EXPORTERS: + choices = ','.join(WATCHME_EXPORTERS) + bot.exit("%s is not a valid exporter, %s" %(exporter_type, choices)) + + # Add the exporter, optionally with tasks + watcher.add_exporter(name=exporter, + exporter_type=exporter_type, + params=params, + tasks=tasks, + force=args.force, + active=args.active) diff --git a/watchme/client/ls.py b/watchme/client/ls.py index 75494c1..7ff1b37 100644 --- a/watchme/client/ls.py +++ b/watchme/client/ls.py @@ -11,14 +11,19 @@ from watchme.command import ( get_watchers, list_watcher, + list_exporters, list_watcher_types ) from watchme.logger import bot def main(args, extra): '''list installed watchers - ''' - if args.watchers == True: + ''' + # The user wants to list exporters + if args.exporters == True: + list_exporters() + + elif args.watchers == True: list_watcher_types() # Otherwise, we are listing installed watchers and tasks diff --git a/watchme/client/push.py b/watchme/client/push.py new file mode 100644 index 0000000..ede8380 --- /dev/null +++ b/watchme/client/push.py @@ -0,0 +1,44 @@ +''' + +Copyright (C) 2019 Vanessa Sochat. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' + +from watchme import get_watcher +from watchme.utils import ( write_json, generate_temporary_file ) +from watchme.logger import bot +import json +import os + +def main(args, extra): + '''push one or more watchers to an exporter endpoint + ''' + # Required - will print help if not provided + name = args.watcher[0] + task = args.task[0] + exporter = args.exporter[0] + filename = args.filename[0] + + # Example command to show task- and exporter- + example = 'watchme push ' + + if not task.startswith('task'): + bot.exit('Task name must start with "task", e.g., %s' % example) + + if not exporter.startswith('exporter'): + bot.exit('Exporter name must start with "exporter", e.g., %s' % example) + + # Get the watcher to interact with, must already exist + watcher = get_watcher(name, base=args.base, create=False) + + # Push to the exporter (not the default git) + result = watcher.push(task=task, + exporter=exporter, + filename=filename, + name=name, + export_json=args.json, + base=args.base) diff --git a/watchme/client/remove.py b/watchme/client/remove.py index c7e0554..1006440 100644 --- a/watchme/client/remove.py +++ b/watchme/client/remove.py @@ -12,7 +12,7 @@ from watchme.logger import bot def main(args, extra): - '''activate one or more watchers + '''remove tasks or exporters from a watcher, or delete the entire watcher ''' # Required - will print help if not provided name = args.watcher[0] @@ -26,10 +26,26 @@ def main(args, extra): # Exit if the user doesn't provide any tasks to remove if extra == None: - bot.exit('Provide tasks to remove, or --delete for entire watcher.') + bot.exit('Provide tasks or exporters to remove, or --delete for entire watcher.') - for task in extra: + # Case 1: one argument indicates removing an entire task or exporter + if len(extra) == 1: - # Remove the task, if it exists - watcher.remove_task(task) + section = extra[0] + # Remove the task or exporter, if it exists + if section.startswith('task'): + watcher.remove_task(section) + + elif section.startswith('exporter'): + watcher.remove_exporter(section) + + else: + bot.error("Task and exporters must begin with (task|exporter)-") + + # Case 2: two arguments indicate removing an exporter from a task + elif len(extra) == 2: + + # Allow the user to specify either order + extra.sort() # task, exporter + watcher.remove_task_exporter(extra[1], extra[0]) diff --git a/watchme/command/__init__.py b/watchme/command/__init__.py index 78853e3..eccc4b4 100644 --- a/watchme/command/__init__.py +++ b/watchme/command/__init__.py @@ -26,6 +26,7 @@ ) from .utils import ( get_watchers, + list_exporters, list_watcher, list_watcher_types ) diff --git a/watchme/command/create.py b/watchme/command/create.py index 101f6d8..6c08d4e 100644 --- a/watchme/command/create.py +++ b/watchme/command/create.py @@ -27,7 +27,7 @@ def create_watcher(name=None, watcher_type=None, base=None): name: the watcher to create, uses default or WATCHME_WATCHER watcher_type: the type of watcher to create. defaults to WATCHER_DEFAULT_TYPE - base: The watcher base to use (defaults to $HOME/.watchme) + base: The watcher base to use (defaults to $HOME/.watchme) ''' if name == None: name = WATCHME_WATCHER diff --git a/watchme/command/utils.py b/watchme/command/utils.py index 022440f..86394f3 100644 --- a/watchme/command/utils.py +++ b/watchme/command/utils.py @@ -9,7 +9,8 @@ ''' from watchme.defaults import ( - WATCHME_BASE_DIR, + WATCHME_BASE_DIR, + WATCHME_EXPORTERS, WATCHME_TASK_TYPES ) @@ -59,6 +60,14 @@ def list_watcher(watcher, base=None): else: bot.exit('%s does not exist.' % base) + +def list_exporters(): + '''list the exporter options provided by watchme + ''' + bot.custom(prefix="watchme:", message="exporters", color="CYAN") + bot.info('\n '.join(WATCHME_EXPORTERS)) + + def list_watcher_types(): '''list the exporter options provided by watchme ''' diff --git a/watchme/defaults.py b/watchme/defaults.py index a99ad3c..10fa210 100644 --- a/watchme/defaults.py +++ b/watchme/defaults.py @@ -57,6 +57,7 @@ def getenv(variable_key, default=None, required=False, silent=True): # The types of valid watchers (currently only urls). Corresponds with # a folder under "main/watchers" +WATCHME_EXPORTERS = ['pushgateway'] WATCHME_TASK_TYPES = ['urls', 'url', 'psutils', 'results'] WATCHME_DEFAULT_TYPE = "urls" diff --git a/watchme/exporters/__init__.py b/watchme/exporters/__init__.py new file mode 100644 index 0000000..2ad069c --- /dev/null +++ b/watchme/exporters/__init__.py @@ -0,0 +1,93 @@ +''' + +Copyright (C) 2019 Vanessa Sochat +Copyright (C) 2019 Antoine Solnichkin + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' + +from watchme.logger import bot + +class ExporterBase(object): + + required_params = [] + + def __init__(self, name, params={}, **kwargs): + + self.name = name + self.valid = False + self.params = {} + self.set_params(params) + self.validate() + + + def get_type(self): + '''get the exporter type. + ''' + return self.type + + # Parameters + + def set_params(self, params): + '''iterate through parameters, set into dictionary. + + Parameters + ========== + params: a list of key@value pairs to set. + ''' + for key,value in params.items(): + key = key.lower() + self.params[key] = value + + + def push(self, result): + '''push dummy function, optional for subclass to implement. + ''' + bot.warning('push is not implemented for %s' % self.type) + + + def export(self, result, task): + '''the export function is the entrypoint to export data for an + exporter. Based on the data type, we call any number of supporting + functions. If True is returned, the data is exported. If False is + returned, there was an error. If None is returned, there is no + exporter defined for the data type. + + Parameters + ========== + result: the result object to export, a string, list, dict, or file + task: the task object associated. + ''' + bot.warning('export is not implemented for %s' % self.type) + + + def export_params(self, active="true"): + '''export parameters, meaning returning a dictionary of the task + parameters plus the addition of the task type and active status. + ''' + params = self.params.copy() + params['active'] = active + params['type'] = self.type + return params + + # Validation + + # For now, it should always be valid as no required parameters are defined. + def validate(self): + '''validate the parameters set for the Exporter. Exit if there are any + errors. Ensure required parameters are defined, and have correct + values. + ''' + self.valid = True + + for param in self.required_params: + if param not in self.params: + bot.error('Missing required parameter: %s' % param) + self.valid = False + + # If the exporter has some custom validation, do it here + if self.valid is True and hasattr(self, "_validate"): + self._validate() diff --git a/watchme/exporters/pushgateway/__init__.py b/watchme/exporters/pushgateway/__init__.py new file mode 100644 index 0000000..5e90980 --- /dev/null +++ b/watchme/exporters/pushgateway/__init__.py @@ -0,0 +1,127 @@ +''' + +Copyright (C) 2019 Vanessa Sochat +Copyright (C) 2019 Antoine Solnichkin + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' + +from watchme.logger import bot +from watchme.exporters import ExporterBase + +class Exporter(ExporterBase): + + required_params = ['url'] + + def __init__(self, name, params={}, **kwargs): + + # Ensure that the user has installed PushGateway + try: + from prometheus_client import CollectorRegistry + except: + bot.error("prometheus_client module not found.") + bot.error("pip install watchme[exporter-pushgateway]") + return + + self.type = 'pushgateway' + super(Exporter, self).__init__(name, params, **kwargs) + self.registry = CollectorRegistry() + + def _validate(self): + '''this function is called after the ExporterBase validate (checking + for required params) to ensure that the url param starts with http. + ''' + if not self.params['url'].startswith('http'): + bot.exit("url must start with http, found %s" % self.params['url']) + +# Push Functions + + def push(self, result, task): + '''push dummy function, expects the dictionary with commits, dates, + and results. Since pushgateway only takes numbers, we parse the + list of content. + ''' + if "content" in result: + return self._save_text_list(task.name, result['content']) + + +# Export Functions + + def export(self, result, task): + '''the export function is the entrypoint to export data for an + exporter. Based on the data type, we call any number of supporting + functions. If True is returned, the data is exported. If False is + returned, there was an error. If None is returned, there is no + exporter defined for the data type. + ''' + # Case 1. The result is a list + if isinstance(result, list): + + # Get rid of Nones, if the user accidentally added + result = [r for r in result if r] + + if len(result) == 0: + bot.error('%s returned empty list of results.' % name) + + # Only save if the export type is not json, and the result is a text string + elif not task.params.get('save_as') == 'json' and not os.path.exists(result[0]): + bot.info('Exporting list to ' + client.name) + return self._save_text_list(task.name, result) + + # Case 2. The result is a string + elif isinstance(result, str): + + # Only export if it's not a file path (so it's a string) + if not(os.path.exists(result)): + bot.info('Exporting text to ' + client.name) + return self._save_text(result) + + # Case 3. The result is a dictionary or a file, ignore for now. + else: + bot.warning('Files/dictionary are not currently supported for export') + + def _save_text_list(self, name, results): + '''for a list of general text results, send them to a pushgateway. + for any error, the calling function should return False immediately. + + Parameters + ========== + results: list of string results to write to the pushgateway + ''' + for r in range(len(results)): + if not self._write_to_pushgateway(results[r]): + return False + return True + + def _save_text(self, result): + '''exports the text to the exporter + + Parameters + ========== + result: the result object to save, not a path to a file in this case + ''' + return self._write_to_pushgateway(result) + + def _write_to_pushgateway(self, result): + ''' writes data to the pushgateway + + Parameters + ========== + result: the result object to save + ''' + from prometheus_client import Gauge, push_to_gateway + + try: + g = Gauge(self.name.replace('-', ':'), '', registry=self.registry) + g.set(result) + push_to_gateway(self.params['url'], + job='watchme', + registry=self.registry) + except: + bot.error('An exception occurred while trying to export data using %s' % self.name) + return False + + return True diff --git a/watchme/version.py b/watchme/version.py index 65a1ee1..4772fb1 100644 --- a/watchme/version.py +++ b/watchme/version.py @@ -31,12 +31,21 @@ ('psutil', {'min_version': '5.4.3'}), ) +INSTALL_PUSHGATEWAY = ( + ('prometheus_client', {'min_version': '0.6.0'}), +) + # Install all watchers and exporters INSTALL_ALL = (INSTALL_REQUIRES + INSTALL_PSUTILS + - INSTALL_URLS_DYNAMIC) + INSTALL_URLS_DYNAMIC + + INSTALL_PUSHGATEWAY) # Install all watchers INSTALL_WATCHERS = (INSTALL_REQUIRES + INSTALL_PSUTILS + INSTALL_URLS_DYNAMIC) + +# Install all exporters +INSTALL_EXPORTERS = (INSTALL_REQUIRES + + INSTALL_PUSHGATEWAY) diff --git a/watchme/watchers/README.md b/watchme/watchers/README.md index 5ecd717..d413a07 100644 --- a/watchme/watchers/README.md +++ b/watchme/watchers/README.md @@ -5,6 +5,7 @@ Each of these is a Watcher that the user can request. - [urls](urls) to watch for changes in websites (default) - [psutils](psutils) to get basic system statistics + ## Watcher Base The watcher base is defined in the [init](__init__.py) file here. diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index e5045c4..0700115 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -27,7 +27,16 @@ from configparser import NoOptionError from .data import ( - export_dict + export_dict, + export_runs, + get_exporter, + add_exporter, + has_exporter, + remove_exporter, + remove_task_exporter, + _add_exporter, + add_task_exporter, + push ) from .settings import ( @@ -72,12 +81,12 @@ class Watcher(object): - repo=None - configfile=None + repo = None + configfile = None - def __init__(self, name=None, - base=None, - create=False, **kwargs): + def __init__(self, name=None, + base=None, + create=False, **kwargs): '''the watcher base loads configuration files for the user (in $HOME) and module, and then stores any arguments given from the caller @@ -96,7 +105,6 @@ def __init__(self, name=None, # Load the configuration self.load_config() - def _set_base(self, base=None, create=False): ''' set the base for the watcher, ensuring that it exists. @@ -118,7 +126,8 @@ def _set_base(self, base=None, create=False): if create is True: create_watcher(self.name) else: - bot.exit('Watcher %s does not exist. Use watchme create.' % self.name) + bot.exit( + 'Watcher %s does not exist. Use watchme create.' % self.name) # Config @@ -127,7 +136,6 @@ def save(self): '''save the configuration to file.''' write_config(self.configfile, self.config) - def edit_task(self, name, action, key, value=None): '''edit a task, meaning doing an addition (add), update (update), or "remove", All actions require a value other than remove. @@ -177,7 +185,6 @@ def edit_task(self, name, action, key, value=None): bot.exit('%s is not found in config, cannot be removed.' % key) self.save() - def has_section(self, name): '''returns True or False to indicate if the watcher has a specified section. To get a task, use self.has_task. @@ -189,10 +196,9 @@ def has_section(self, name): self.load_config() if name in self.config._sections: return True - bot.warning('%s not found for watcher %s' %(name, self.name)) + bot.warning('%s not found for watcher %s' % (name, self.name)) return False - def load_config(self): '''load a configuration file, and set the active setting for the watcher if the file doesn't exist, the function will exit and prompt the user @@ -213,7 +219,6 @@ def load_config(self): # Only update the config if we've changed it self.save() - def _get_params_dict(self, pairs): '''iterate through parameters, make keys lowercase, and ensure valid format. @@ -245,8 +250,7 @@ def add_task(self, task, task_type, params, force=False, active="true"): Parameters ========== - task: the Task object to add, should have a name and params and - be child of watchme.tasks.TaskBase + task: the task name to add, must start with task- task_type: must be in WATCHME_TASK_TYPES, meaning a client exists params: list of parameters to be validated (key@value) force: if task already exists, overwrite @@ -256,7 +260,7 @@ def add_task(self, task, task_type, params, force=False, active="true"): # Check again, in case user calling from client if not task.startswith('task'): bot.exit('Task name must start with "task" (e.g., task-reddit)') - + # Ensure it's a valid type if task_type not in WATCHME_TASK_TYPES: bot.exit('%s is not a valid type: %s' % WATCHME_TASK_TYPES) @@ -276,7 +280,7 @@ def add_task(self, task, task_type, params, force=False, active="true"): # Convert list to dictionary params = self._get_params_dict(params) - + # Creating the task will validate parameters newtask = Task(task, params=params) @@ -340,7 +344,8 @@ def delete(self): if self.is_frozen(): bot.exit('watcher %s is frozen, unfreeze to delete.' % self.name) elif self.is_protected(): - bot.exit('watcher %s is protected, turn off protection to delete.' % self.name) + bot.exit( + 'watcher %s is protected, turn off protection to delete.' % self.name) repo = os.path.dirname(self.configfile) @@ -378,12 +383,12 @@ def remove_task(self, task): # Inspect - + def inspect(self, tasks=None, create_command=False): '''inspect a watcher, or one or more tasks belonging to it. This means printing the configuration for the entire watcher (if tasks is None) or just for one or more tasks. - + Parameters ========== tasks: one or more tasks to inspect (None will show entire file) @@ -408,7 +413,6 @@ def inspect(self, tasks=None, create_command=False): else: self.print_add_task(task) - def list(self, quiet=False): '''list the watchers. If quiet is True, don't print to the screen.''' watchers = get_watchers(base=self.base, quiet=quiet) @@ -420,12 +424,11 @@ def protect(self, status="on"): '''protect a watcher, meaning that it cannot be deleted. This does not influence removing a task. To freeze the entire watcher, use the freeze() function. - ''' + ''' self._set_status('watcher', 'protected', status) git_commit(self.repo, self.name, "PROTECT %s" % status) self.print_section('watcher') - def freeze(self): '''freeze a watcher, meaning that it along with its tasks cannot be deleted. This does not prevent the user from manual editing. @@ -455,7 +458,6 @@ def _set_status(self, section, setting, value): bot.exit('Status must be "on" or "off"') self.set_setting(section, setting, value) self.save() - def is_protected(self): '''return a boolean to indicate if the watcher is protected or frozen. @@ -467,7 +469,6 @@ def is_protected(self): if self.get_setting('watcher', status) == "on": protected = True return protected - def is_frozen(self): '''return a boolean to indicate if the watcher is frozen. @@ -483,7 +484,7 @@ def is_frozen(self): def _active_status(self, status='true', name=None): '''a general function to change the status, used by activate and deactivate. - + Parameters ========== status: must be one of true, false @@ -497,7 +498,7 @@ def _active_status(self, status='true', name=None): # Cut out early if section not in config if name not in self.config._sections: - bot.exit('%s is not a valid task or section' % name) + bot.exit('%s is not a valid task or section' % name) if status not in ['true', 'false']: bot.exit('status must be true or false.') @@ -513,9 +514,9 @@ def _active_status(self, status='true', name=None): # Add the task name if name != None: - message = "%s task %s" %(message, name) + message = "%s task %s" % (message, name) - bot.info('[%s|%s] active: %s' % (name, self.name, status)) + bot.info('[%s|%s] active: %s' % (name, self.name, status)) return message def activate(self, task=None): @@ -523,7 +524,6 @@ def activate(self, task=None): ''' message = self._active_status('true', task) git_commit(self.repo, self.name, message) - def deactivate(self, task=None): '''turn the active status of a watcher to false. If a task is provided, @@ -533,7 +533,6 @@ def deactivate(self, task=None): message = self._active_status('false', task) git_commit(self.repo, self.name, message) - def is_active(self, task=None): '''determine if the watcher is active by reading from the config directly if a task name is provided, check the active status of the task @@ -608,7 +607,7 @@ def _task_selected(self, task, regexp=None, active=True): regexp: an optional regular expression (or name) to check active: a task is selected if it's active (default True) ''' - selected = True + selected = True # A task can be None if it wasn't found if task == None: @@ -619,7 +618,7 @@ def _task_selected(self, task, regexp=None, active=True): if is_active == "false" and active == True: bot.info('Task %s is not active.' % task) selected = False - + # The user wants to search for a custom task name if regexp != None: if not re.search(regexp, task): @@ -628,7 +627,6 @@ def _task_selected(self, task, regexp=None, active=True): return selected - def get_tasks(self, regexp=None, quiet=False, active=True): '''get the tasks for a watcher, possibly matching a regular expression. A list of dictionaries is returned, each holding the parameters for @@ -679,10 +677,10 @@ def run_tasks(self, queue, parallel=True, show_progress=True): {'task-reddit-hpc': [('url', 'https://www.reddit.com/r/hpc'), ('active', 'true'), ('type', 'urls')]} - ''' + ''' if parallel is True: return self._run_parallel(queue, show_progress) - + # Otherwise, run in serial results = {} @@ -701,7 +699,6 @@ def run_tasks(self, queue, parallel=True, show_progress=True): return results - def _run_parallel(self, queue, show_progress=True): ''' run tasks in parallel using the Workers class. Returns a dictionary (lookup) wit results, with the key being the task name @@ -718,14 +715,13 @@ def _run_parallel(self, queue, show_progress=True): for task in queue: - # Export parameters and functions + # Export parameters and functions funcs[task.name] = task.export_func() tasks[task.name] = task.export_params() workers = Workers(show_progress=show_progress) return workers.run(funcs, tasks) - def run(self, regexp=None, parallel=True, test=False, show_progress=True): '''run the watcher, which should be done via the crontab, including: @@ -759,15 +755,15 @@ def run(self, regexp=None, parallel=True, test=False, show_progress=True): # Step 3: Run the tasks. This means preparing a list of funcs/params, # and then submitting with multiprocessing results = self.run_tasks(tasks, parallel, show_progress) - + # Finally, finish the runs. if test is False: self.finish_runs(results) + self.export_runs(results) else: # or print results to the screen print(json.dumps(results, indent=4)) - - + def finish_runs(self, results): '''finish runs should take a dictionary of results, with keys as the folder name, and for each, depending on the result type, @@ -819,7 +815,7 @@ def __str__(self): Watcher.print_section = print_section Watcher.print_add_task = print_add_task -# Schedule +# Schedule Watcher.remove_schedule = remove_schedule Watcher.get_crontab = get_crontab @@ -829,6 +825,15 @@ def __str__(self): Watcher.clear_schedule = clear_schedule Watcher.schedule = schedule -# Data +# Exporters +Watcher.add_exporter = add_exporter +Watcher.add_task_exporter = add_task_exporter +Watcher.get_exporter = get_exporter +Watcher.has_exporter = has_exporter +Watcher._add_exporter = _add_exporter +Watcher.remove_exporter = remove_exporter +Watcher.remove_task_exporter = remove_task_exporter Watcher.export_dict = export_dict +Watcher.export_runs = export_runs +Watcher.push = push diff --git a/watchme/watchers/data.py b/watchme/watchers/data.py index fb7b262..8fde598 100644 --- a/watchme/watchers/data.py +++ b/watchme/watchers/data.py @@ -9,6 +9,7 @@ ''' from watchme.logger import bot +from watchme.defaults import WATCHME_EXPORTERS from watchme.utils import ( which, get_user @@ -16,13 +17,14 @@ from watchme.command import ( get_commits, git_show, - git_date + git_date, + git_commit ) -import os import json +import os +import re -# Data Exports - +# Default (git) Data Exports def export_dict(self, task, filename, @@ -86,3 +88,335 @@ def export_dict(self, task, result['dates'].append(git_date(repo=repo, commit=commit)) result['commits'].append(commit) return result + + + # Push to the exporter (not the default git) + result = watcher.push(task=task, + exporter=exporter, + name=name, + export_json=args.json, + base=args.base) + + +def push(self, task, exporter, + name=None, + filename=None, + export_json=False, + from_commit=None, + to_commit=None, + base=None): + '''Manually push a watcher task data file to an exporter endpoint. + + Parameters + ========== + task: the task folder for the watcher to look in + exporter: the exporter to use + name: the name of the watcher, defaults to the client's + export_json: read the filename data as json (not text) + push_all: export all data (and not just the last timepoint) + from_commit: the commit to start at + to_commit: the commit to go to + filename: the filename to filter to. Includes all files if not specified. + ''' + # Quit if the exporter isn't there + if not self.has_exporter(exporter): + bot.exit('%s is not a valid exporter for task %s' % (exporter, task)) + + # Get the exporter, ensure still valid. + exporter = self.get_exporter(exporter) + task_instance = self.get_task(task) + + if task_instance == None: + bot.exit("Task %s does not exist." % task) + + if exporter.valid is False: + bot.exit("Exporter %s is not valid." % exporter.name) + + # Make sure the exporter is added to the task + if exporter.name not in task_instance.params.get('exporters', ''): + bot.exit("Exporter %s is not added to task %s" %(exporter.name, task)) + + result = self.export_dict(task=task_instance.name, + filename=filename, + name=name, + export_json=export_json, + from_commit=from_commit, + to_commit=to_commit, + base=base) + + exporter.push(result, task_instance) + + +# Exporter Functions + +def add_exporter(self, name, exporter_type, params, tasks, force=False, active="true"): + '''add an exporter, meaning an extra plugin to push data to a remote. + + Parameters + ========== + name: the name of the exporter to add, should start with exporter- + exporter_type: must be in WATCHME_EXPORTERS, meaning a client exists + params: list of parameters to be validated (key@value) + tasks: list of task names to add the exporter to + force: if task already exists, overwrite + active: add the task as active (default "true") + ''' + # Check again, in case user calling from client + if not name.startswith('exporter'): + bot.exit('Exporter name must start with "exporter" (e.g., exporter-grafana)') + + self.load_config() + + # If it already exists and the user isn't recreating, don't add it again. + if name in self.config.sections() and exporter_type == None: + exporter = self.get_exporter(name) + + # Otherwise, this is creation of a new exporter + else: + + # Ensure it's a valid type + if exporter_type not in WATCHME_EXPORTERS: + bot.exit('%s is not a valid type: %s' % WATCHME_EXPORTERS) + + # Validate variables provided for task + if exporter_type.startswith('pushgateway'): + from watchme.exporters.pushgateway import Exporter + + else: + bot.exit('exporter_type %s not installed' % exporter_type) + + # Convert list to dictionary + params = self._get_params_dict(params) + + # Creating the exporter will validate parameters + exporter = Exporter(name, params=params) + + + # Exit if the exporter is not valid + if not exporter.valid: + bot.exit('%s is not valid, will not be added.' % exporter.name) + + # Add the exporter to the config + if exporter_type != None: + self._add_exporter(exporter, force, active) + + # Add tasks to it + for task in tasks: + self.add_task_exporter(task, exporter.name) + + # Save all changes + self.save() + + # Commit changes + git_commit(repo=self.repo, task=self.name, + message="ADD exporter %s" % exporter.name) + + +def _add_exporter(self, exporter, force=False, active='true', tasks=[]): + '''add a new exporter to the watcher, meaning we: + + 1. Check first that the task doesn't already exist (if the task + exists, we only add if force is set to true) + 2. Validate the task (depends on the task) + 3. write the task to the helper config file, if valid. + + Parameters + ========== + exporter: the Task object to add, should have a name and params and + be child of watchme.tasks.TaskBase + force: if task already exists, overwrite + active: add the task as active (default "true") + ''' + self.load_config() + + if active not in ["true", "false"]: + bot.exit('Active must be "true" or "false"') + + # Don't overwrite a section that already exists + if exporter.name in self.config.sections(): + if not force: + bot.exit('%s exists, use --force to overwrite.' % exporter.name) + self.remove_section(exporter.name, save=False) + + # Add the new section + self.config[exporter.name] = exporter.export_params(active=active) + self.print_section(exporter.name) + + # Save all changes + self.save() + + +def add_task_exporter(self, task, name): + '''append an exporter to a task exporters list, if it isn't already + included. Since the configparser supports strings, the list + of exporters is a list of comma separated values. + + Parameters + ========== + task: the name of the task to add the exporter to + name: the name of the exporter to add, if not already there + ''' + if task in self.config.sections(): + exporters = self.get_setting(task, "exporters", default="") + exporters = [e for e in exporters.split(',') if e] + if name not in exporters: + exporters.append(name) + self.set_setting(task, "exporters", ','.join(exporters)) + else: + bot.warning("Task %s not found installed, skipping adding exporter to it." % task) + + +def remove_task_exporter(self, task, name): + '''remove an exporter from a task exporters list + + Parameters + ========== + task: the name of the task to add the exporter to + name: the name of the exporter to add, if not already there + ''' + if task in self.config.sections(): + exporters = self.get_setting(task, "exporters", default="") + + if name in exporters: + bot.info("Removing %s from %s" %(name, task)) + exporters = re.sub(exporters, name, "").strip(",") + + # If no more exporters, remove the param entirely + if len(exporters) == 0: + self.remove_setting(task, 'exporters') + + # Otherwise, update the shorter list + else: + self.set_setting(task, 'exporters', exporters) + self.save() + + else: + bot.warning("Task %s not found installed, skipping removing exporter from it." % task) + + +def remove_exporter(self, name): + '''remove a an exporter from the watcher repo, if it exists, along with + any tasks that it is added to. + + Parameters + ========== + name: the name of the exporter to remove + ''' + if self.get_section(name) != None: + if self.is_frozen(): + bot.exit('watcher is frozen, unfreeze first.') + self.remove_section(name) + + # Remove the exporter from any tasks + for task in self.get_tasks(quiet=True, active=False): + self.remove_task_exporter(task.name, name) + + bot.info('%s removed successfully.' % name) + git_commit(self.repo, self.name, "REMOVE exporter %s" % name) + + else: + bot.warning('Exporter %s does not exist.' % name) + + +# Get Exporters + + +def get_exporter(self, name): + '''get a particular task, based on the name. This is where each type + of class should check the "type" parameter from the config, and + import the correct Task class. + + Parameters + ========== + name: the name of the task to load + save: if saving, will be True + ''' + self.load_config() + + exporter = None + + # Only sections that start with task- are considered tasks + if name in self.config._sections and name.startswith('exporter'): + + # Task is an ordered dict, key value pairs are entries + params = self.config._sections[name] + + # Get the task type (if removed, consider disabled) + exporter_type = params.get('type', '') + + # If we get here, validate and prepare the task + if exporter_type.startswith("pushgateway"): + from watchme.exporters.pushgateway import Exporter + + else: + bot.exit('%s is not a valid exporter type.' % exporter_type) + + # if not valid, will return None (or exit) + exporter = Exporter(name, params) + + return exporter + + +def has_exporter(self, name): + '''returns True or False to indicate if the watcher has a specific exporter + ''' + self.load_config() + if self.has_section(name) and name.startswith('exporter'): + return True + return False + + +def export_runs(self, results): + ''' export data retrieved to the set of exporters defined and active, + which can vary based on the task. The export will be run if: + + 1. The task has a list of one or more exporters to use. + 2. Any given exporter is defined also in the watchme.cfg + 3. The exporter is active. + + If the user does not wish to export data for a specific task, + the exporter can be turned off (active set to false), the exporter + name can be removed from "exporters" or the exporter- configuration + can be removed from the file entirely. + ''' + for name, result in results.items(): + + task = self.get_task(name) + + # Get exporters added to task + exporters = self.get_setting(task.name, 'exporters', "") + exporters = [e for e in exporters.split(",") if e] + + # Validate each exporter exists and is active, then run. + for exporter in exporters: + + # Check 1: The exporter must be defined + if not self.has_section(exporter): + bot.warning("%s not defined in watchme.cfg for %s" %(exporter, task)) + continue + + # Check 2: It must be active + # Be more conservative to run an export, default to false + if self.get_setting(exporter, "active", default="false") == "false": + bot.warning("%s is defined but not active for %s" %(exporter, task)) + continue + + # If we get here, safe to instantiate the exporter. + try: + client = self.get_exporter(exporter) + except: + bot.warning("Check parameters for %s, client did not validate." % exporter) + continue + + # A client with "None" indicates a dependency is likely missing + if client == None: + bot.warning("Exporter %s does not exist, or check dependencies for it." % exporter) + continue + + # If we get here, save the result. True = success, False = Fail, None = NA + result = client.export(result, task) + + # If exporter fails, remove it for the task. + if result == False: + bot.warning("Issue with export of %s, removing exporter %s." % (task.name, exporter)) + self.remove_task_exporter(task.name, exporter) diff --git a/watchme/watchers/urls/helpers.py b/watchme/watchers/urls/helpers.py index 2a55ae3..6e9dfa3 100644 --- a/watchme/watchers/urls/helpers.py +++ b/watchme/watchers/urls/helpers.py @@ -9,6 +9,7 @@ ''' import os +import re import tempfile import requests import re