support running outside of its own directory

baler.py

@@ -1,3 +1,5 @@
+#! /usr/bin/env python
+
 import ConfigParser
 import datetime as dt
 import gzip
@@ -18,7 +20,9 @@
 
 def tiq_output(reg_file, enr_file):
     config = ConfigParser.SafeConfigParser()
-    cfg_success = config.read('combine.cfg')
+    base_path = os.path.dirname(__file__)
+    full_path = base_path + '/combine.cfg'
+    cfg_success = config.read(full_path)
     if not cfg_success:
         logger.error('tiq_output: Could not read combine.cfg.')
         logger.error('HINT: edit combine-example.cfg and save as combine.cfg.')
@@ -148,7 +152,9 @@ def bale_CRITs(harvest, filename):
     data = {'confidence': 'medium'}
     start_time = time.time()
     config = ConfigParser.SafeConfigParser()
-    cfg_success = config.read('combine.cfg')
+    base_path = os.path.dirname(__file__)
+    full_path = base_path + '/combine.cfg'
+    cfg_success = config.read(full_path)
     if not cfg_success:
         logger.error('tiq_output: Could not read combine.cfg.\n')
         logger.error('HINT: edit combine-example.cfg and save as combine.cfg.\n')
@@ -200,7 +206,9 @@ def bale_CRITs(harvest, filename):
 
 def bale(input_file, output_file, output_format, is_regular):
     config = ConfigParser.SafeConfigParser()
-    cfg_success = config.read('combine.cfg')
+    base_path = os.path.dirname(__file__)
+    full_path = base_path + '/combine.cfg'
+    cfg_success = config.read(full_path)
     if not cfg_success:
         logger.error('Baler: Could not read combine.cfg.')
         logger.error('HINT: edit combine-example.cfg and save as combine.cfg.')
@@ -218,4 +226,6 @@ def bale(input_file, output_file, output_format, is_regular):
     format_funcs[output_format](harvest, output_file)
 
 if __name__ == "__main__":
-    bale('crop.json', 'harvest.csv', 'csv', True)
+    base_path = os.path.dirname(__file__) + '/'
+    bp = base_path
+    bale(bp + 'crop.json', bp + 'harvest.csv', 'csv', True)

combine.py

@@ -36,19 +36,22 @@
 else:
     out_file = 'harvest.'+out_type
 
-reap('harvest.json')
-thresh('harvest.json', 'crop.json')
-bale('crop.json', out_file, out_type, True)
+base_path = os.path.dirname(__file__) + '/'
+bp = base_path
+
+reap(bp + 'harvest.json')
+thresh(bp + 'harvest.json', bp + 'crop.json')
+bale(bp + 'crop.json', out_file, out_type, True)
 
 if args.enrich or args.tiq_test:
-    winnow('crop.json', 'crop.json', 'enrich.json')
-    bale('enrich.json', 'enriched.'+out_type, out_type, False)
+    winnow(bp + 'crop.json', bp + 'crop.json', bp +'enrich.json')
+    bale(bp + 'enrich.json', bp + 'enriched.'+out_type, out_type, False)
 
 if args.tiq_test:
-    tiq_output('crop.json', 'enrich.json')
+    tiq_output(bp + 'crop.json', bp + 'enrich.json')
 
 if args.delete:
     # be careful with this when we support a JSON output type
-    os.remove('harvest.json')
-    os.remove('crop.json')
-    os.remove('enrich.json')
+    os.remove(bp + 'harvest.json')
+    os.remove(bp + 'crop.json')
+    os.remove(bp + 'enrich.json')

inbound_urls.txt

@@ -25,4 +25,4 @@ https://www.packetmail.net/iprep.txt
 http://www.autoshun.org/files/shunlist.csv
 http://charles.the-haleys.org/ssh_dico_attack_hdeny_format.php/hostsdeny.txt
 http://virbl.org/download/virbl.dnsbl.bit.nl.txt
-http://botscout.com/last_caught_cache.htm
+http://botscout.com/last_caught_cache.htm

reaper.py

@@ -1,9 +1,12 @@
+#! /usr/bin/env python
+
 import ConfigParser
 import grequests
 import json
 import sys
 from logger import get_logger
 import logging
+import os
 
 
 logger = get_logger('reaper')
@@ -13,18 +16,22 @@ def exception_handler(request, exception):
 
 def reap(file_name):
     config = ConfigParser.SafeConfigParser(allow_no_value=False)
-    cfg_success = config.read('combine.cfg')
+    base_path = os.path.dirname(__file__)
+    full_path = base_path + '/combine.cfg'
+    cfg_success = config.read(full_path)
     if not cfg_success:
         logger.error('Reaper: Could not read combine.cfg.')
         logger.error('HINT: edit combine-example.cfg and save as combine.cfg.')
         return
 
     inbound_url_file = config.get('Reaper', 'inbound_urls')
+    inbound_url_file = base_path + '/' + inbound_url_file
     outbound_url_file = config.get('Reaper', 'outbound_urls')
+    outbound_url_file = base_path + '/' + outbound_url_file
 
     try:
         with open(inbound_url_file, 'rb') as f:
-    	    inbound_urls = [url.rstrip('\n') for url in f.readlines()]
+            inbound_urls = [url.rstrip('\n') for url in f.readlines()]
     except EnvironmentError as e:
         logger.error('Reaper: Error while opening "%s" - %s' % (inbound_url_file, e.strerror))
         return
@@ -81,4 +88,6 @@ def reap(file_name):
 
 
 if __name__ == "__main__":
-    reap('harvest.json')
+    base_path = os.path.dirname(__file__) + '/'
+    bp = base_path
+    reap(bp + 'harvest.json')

thresher.py

@@ -1,3 +1,5 @@
+#! /usr/bin/env python
+
 import ConfigParser
 import bs4
 import datetime
@@ -7,6 +9,7 @@
 from logger import get_logger
 from csv import reader
 from itertools import ifilter
+import os
 
 logger = get_logger('thresher')
 
@@ -149,7 +152,9 @@ def process_malwaregroup(response, source, direction):
 def thresh(input_file, output_file):
 
     config = ConfigParser.SafeConfigParser(allow_no_value=False)
-    cfg_success = config.read('combine.cfg')
+    base_path = os.path.dirname(__file__)
+    full_path = base_path + '/combine.cfg'
+    cfg_success = config.read(full_path)
     if not cfg_success:
         logger.error('Thresher: Could not read combine.cfg.')
         logger.error('HINT: edit combine-example.cfg and save as combine.cfg.')
@@ -211,4 +216,6 @@ def thresh(input_file, output_file):
 
 
 if __name__ == "__main__":
-    thresh('harvest.json', 'crop.json')
+    base_path = os.path.dirname(__file__) + '/'
+    bp = base_path
+    thresh(bp + 'harvest.json', bp + 'crop.json')

winnower.py

@@ -7,6 +7,7 @@
 import pygeoip
 import re
 import sys
+import os
 
 from netaddr import IPAddress, IPRange, IPSet
 from sortedcontainers import SortedDict
@@ -19,7 +20,9 @@
 reserved_ranges = IPSet(['0.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', '192.88.99.0/24',
                          '198.18.0.0/15', '198.51.100.0/24', '203.0.113.0/24', '233.252.0.0/24'])
 gi_org = SortedDict()
-geo_data = pygeoip.GeoIP('data/GeoIP.dat', pygeoip.MEMORY_CACHE)
+base_path = os.path.dirname(__file__)
+full_path = base_path + '/data/GeoIP.dat'
+geo_data = pygeoip.GeoIP(full_path, pygeoip.MEMORY_CACHE)
 
 
 def load_gi_org(filename):
@@ -122,7 +125,9 @@ def is_fqdn(address):
 
 def winnow(in_file, out_file, enr_file):
     config = ConfigParser.SafeConfigParser(allow_no_value=True)
-    cfg_success = config.read('combine.cfg')
+    base_path = os.path.dirname(__file__)
+    full_path = base_path + '/combine.cfg'
+    cfg_success = config.read(full_path)
     if not cfg_success:
         logger.error('Winnower: Could not read combine.cfg.')
         logger.error('HINT: edit combine-example.cfg and save as combine.cfg.')
@@ -205,4 +210,6 @@ def winnow(in_file, out_file, enr_file):
 
 
 if __name__ == "__main__":
-    winnow('crop.json', 'crop.json', 'enriched.json')
+    base_path = os.path.dirname(__file__) + '/'
+    bp = base_path
+    winnow(bp + 'crop.json', bp + 'crop.json', bp + 'enriched.json')