From 197cfc63060b426cf21ace76f373bf3ee05f1cfd Mon Sep 17 00:00:00 2001 From: Chris Sandulow Date: Fri, 28 Aug 2015 18:03:10 -0400 Subject: [PATCH] support running outside of its own directory all files were read/write from the current dir and assumed the script was run in that manner. when run in a different context it failed. this patch adds the base file path to all open() methods. --- baler.py | 18 ++++++++++++++---- combine.py | 21 ++++++++++++--------- inbound_urls.txt | 2 +- reaper.py | 15 ++++++++++++--- thresher.py | 11 +++++++++-- winnower.py | 13 ++++++++++--- 6 files changed, 58 insertions(+), 22 deletions(-) diff --git a/baler.py b/baler.py index 5dde618..903185e 100755 --- a/baler.py +++ b/baler.py @@ -1,3 +1,5 @@ +#! /usr/bin/env python + import ConfigParser import datetime as dt import gzip @@ -18,7 +20,9 @@ def tiq_output(reg_file, enr_file): config = ConfigParser.SafeConfigParser() - cfg_success = config.read('combine.cfg') + base_path = os.path.dirname(__file__) + full_path = base_path + '/combine.cfg' + cfg_success = config.read(full_path) if not cfg_success: logger.error('tiq_output: Could not read combine.cfg.') logger.error('HINT: edit combine-example.cfg and save as combine.cfg.') @@ -148,7 +152,9 @@ def bale_CRITs(harvest, filename): data = {'confidence': 'medium'} start_time = time.time() config = ConfigParser.SafeConfigParser() - cfg_success = config.read('combine.cfg') + base_path = os.path.dirname(__file__) + full_path = base_path + '/combine.cfg' + cfg_success = config.read(full_path) if not cfg_success: logger.error('tiq_output: Could not read combine.cfg.\n') logger.error('HINT: edit combine-example.cfg and save as combine.cfg.\n') @@ -200,7 +206,9 @@ def bale_CRITs(harvest, filename): def bale(input_file, output_file, output_format, is_regular): config = ConfigParser.SafeConfigParser() - cfg_success = config.read('combine.cfg') + base_path = os.path.dirname(__file__) + full_path = base_path + '/combine.cfg' + cfg_success = config.read(full_path) if not cfg_success: logger.error('Baler: Could not read combine.cfg.') logger.error('HINT: edit combine-example.cfg and save as combine.cfg.') @@ -218,4 +226,6 @@ def bale(input_file, output_file, output_format, is_regular): format_funcs[output_format](harvest, output_file) if __name__ == "__main__": - bale('crop.json', 'harvest.csv', 'csv', True) + base_path = os.path.dirname(__file__) + '/' + bp = base_path + bale(bp + 'crop.json', bp + 'harvest.csv', 'csv', True) diff --git a/combine.py b/combine.py index c246b49..5ce8e2d 100755 --- a/combine.py +++ b/combine.py @@ -36,19 +36,22 @@ else: out_file = 'harvest.'+out_type -reap('harvest.json') -thresh('harvest.json', 'crop.json') -bale('crop.json', out_file, out_type, True) +base_path = os.path.dirname(__file__) + '/' +bp = base_path + +reap(bp + 'harvest.json') +thresh(bp + 'harvest.json', bp + 'crop.json') +bale(bp + 'crop.json', out_file, out_type, True) if args.enrich or args.tiq_test: - winnow('crop.json', 'crop.json', 'enrich.json') - bale('enrich.json', 'enriched.'+out_type, out_type, False) + winnow(bp + 'crop.json', bp + 'crop.json', bp +'enrich.json') + bale(bp + 'enrich.json', bp + 'enriched.'+out_type, out_type, False) if args.tiq_test: - tiq_output('crop.json', 'enrich.json') + tiq_output(bp + 'crop.json', bp + 'enrich.json') if args.delete: # be careful with this when we support a JSON output type - os.remove('harvest.json') - os.remove('crop.json') - os.remove('enrich.json') + os.remove(bp + 'harvest.json') + os.remove(bp + 'crop.json') + os.remove(bp + 'enrich.json') diff --git a/inbound_urls.txt b/inbound_urls.txt index 5e7fa61..be0ab46 100644 --- a/inbound_urls.txt +++ b/inbound_urls.txt @@ -25,4 +25,4 @@ https://www.packetmail.net/iprep.txt http://www.autoshun.org/files/shunlist.csv http://charles.the-haleys.org/ssh_dico_attack_hdeny_format.php/hostsdeny.txt http://virbl.org/download/virbl.dnsbl.bit.nl.txt -http://botscout.com/last_caught_cache.htm +http://botscout.com/last_caught_cache.htm \ No newline at end of file diff --git a/reaper.py b/reaper.py index a142586..50ad39b 100755 --- a/reaper.py +++ b/reaper.py @@ -1,9 +1,12 @@ +#! /usr/bin/env python + import ConfigParser import grequests import json import sys from logger import get_logger import logging +import os logger = get_logger('reaper') @@ -13,18 +16,22 @@ def exception_handler(request, exception): def reap(file_name): config = ConfigParser.SafeConfigParser(allow_no_value=False) - cfg_success = config.read('combine.cfg') + base_path = os.path.dirname(__file__) + full_path = base_path + '/combine.cfg' + cfg_success = config.read(full_path) if not cfg_success: logger.error('Reaper: Could not read combine.cfg.') logger.error('HINT: edit combine-example.cfg and save as combine.cfg.') return inbound_url_file = config.get('Reaper', 'inbound_urls') + inbound_url_file = base_path + '/' + inbound_url_file outbound_url_file = config.get('Reaper', 'outbound_urls') + outbound_url_file = base_path + '/' + outbound_url_file try: with open(inbound_url_file, 'rb') as f: - inbound_urls = [url.rstrip('\n') for url in f.readlines()] + inbound_urls = [url.rstrip('\n') for url in f.readlines()] except EnvironmentError as e: logger.error('Reaper: Error while opening "%s" - %s' % (inbound_url_file, e.strerror)) return @@ -81,4 +88,6 @@ def reap(file_name): if __name__ == "__main__": - reap('harvest.json') + base_path = os.path.dirname(__file__) + '/' + bp = base_path + reap(bp + 'harvest.json') diff --git a/thresher.py b/thresher.py index 2efc499..ab1d15f 100755 --- a/thresher.py +++ b/thresher.py @@ -1,3 +1,5 @@ +#! /usr/bin/env python + import ConfigParser import bs4 import datetime @@ -7,6 +9,7 @@ from logger import get_logger from csv import reader from itertools import ifilter +import os logger = get_logger('thresher') @@ -149,7 +152,9 @@ def process_malwaregroup(response, source, direction): def thresh(input_file, output_file): config = ConfigParser.SafeConfigParser(allow_no_value=False) - cfg_success = config.read('combine.cfg') + base_path = os.path.dirname(__file__) + full_path = base_path + '/combine.cfg' + cfg_success = config.read(full_path) if not cfg_success: logger.error('Thresher: Could not read combine.cfg.') logger.error('HINT: edit combine-example.cfg and save as combine.cfg.') @@ -211,4 +216,6 @@ def thresh(input_file, output_file): if __name__ == "__main__": - thresh('harvest.json', 'crop.json') + base_path = os.path.dirname(__file__) + '/' + bp = base_path + thresh(bp + 'harvest.json', bp + 'crop.json') diff --git a/winnower.py b/winnower.py index 774f145..f6c324e 100755 --- a/winnower.py +++ b/winnower.py @@ -7,6 +7,7 @@ import pygeoip import re import sys +import os from netaddr import IPAddress, IPRange, IPSet from sortedcontainers import SortedDict @@ -19,7 +20,9 @@ reserved_ranges = IPSet(['0.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', '192.88.99.0/24', '198.18.0.0/15', '198.51.100.0/24', '203.0.113.0/24', '233.252.0.0/24']) gi_org = SortedDict() -geo_data = pygeoip.GeoIP('data/GeoIP.dat', pygeoip.MEMORY_CACHE) +base_path = os.path.dirname(__file__) +full_path = base_path + '/data/GeoIP.dat' +geo_data = pygeoip.GeoIP(full_path, pygeoip.MEMORY_CACHE) def load_gi_org(filename): @@ -122,7 +125,9 @@ def is_fqdn(address): def winnow(in_file, out_file, enr_file): config = ConfigParser.SafeConfigParser(allow_no_value=True) - cfg_success = config.read('combine.cfg') + base_path = os.path.dirname(__file__) + full_path = base_path + '/combine.cfg' + cfg_success = config.read(full_path) if not cfg_success: logger.error('Winnower: Could not read combine.cfg.') logger.error('HINT: edit combine-example.cfg and save as combine.cfg.') @@ -205,4 +210,6 @@ def winnow(in_file, out_file, enr_file): if __name__ == "__main__": - winnow('crop.json', 'crop.json', 'enriched.json') + base_path = os.path.dirname(__file__) + '/' + bp = base_path + winnow(bp + 'crop.json', bp + 'crop.json', bp + 'enriched.json')