Initial support for python3

.gitignore

@@ -61,3 +61,8 @@ test*.txt
 todo.txt
 extra_code.py
 crits_stats
+
+# virtualenv
+.Python
+include/
+pip-selfcheck.json

baler.py

@@ -1,4 +1,4 @@
-import ConfigParser
+import configparser as ConfigParser
 import datetime as dt
 import gzip
 import json
@@ -11,7 +11,7 @@
 import unicodecsv
 import threading
 from logger import get_logger
-from Queue import Queue
+import queue as Queue
 
 logger = get_logger('baler')
 

dnsdb_query.py

@@ -21,8 +21,8 @@
 import os
 import sys
 import time
-import urllib2
-from cStringIO import StringIO
+import urllib
+import io
 
 try:
     import json
@@ -70,25 +70,25 @@ def _query(self, path):
         url = '%s/lookup/%s' % (self.server, path)
         if self.limit:
             url += '?limit=%d' % self.limit
-        req = urllib2.Request(url)
+        req = urllib.request(url)
         req.add_header('Accept', 'application/json')
         req.add_header('X-Api-Key', self.apikey)
         try:
-            http = urllib2.urlopen(req)
+            http = urllib.request.urlopen(req)
             while True:
                 line = http.readline()
                 if not line:
                     break
                 res.append(json.loads(line))
-        except (urllib2.HTTPError, urllib2.URLError), e:
+        except (urllib.HTTPError, urllib.URLError) as e:
             sys.stderr.write(str(e) + '\n')
         return res
 
 def sec_to_text(ts):
     return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts))
 
 def rrset_to_text(m):
-    s = StringIO()
+    s = io()
 
     if 'bailiwick' in m:
         s.write(';;  bailiwick: %s\n' % m['bailiwick'])
@@ -216,7 +216,7 @@ def main():
 
     try:
         cfg = parse_config(options.config)
-    except IOError, e:
+    except IOError as e:
         sys.stderr.write(e.message)
         sys.exit(1)
 

reaper.py

@@ -1,4 +1,4 @@
-import ConfigParser
+import configparser as ConfigParser
 import grequests
 import json
 import sys
@@ -23,14 +23,14 @@ def reap(file_name):
     outbound_url_file = config.get('Reaper', 'outbound_urls')
 
     try:
-        with open(inbound_url_file, 'rb') as f:
+        with open(inbound_url_file, 'r') as f:
     	    inbound_urls = [url.rstrip('\n') for url in f.readlines()]
     except EnvironmentError as e:
         logger.error('Reaper: Error while opening "%s" - %s' % (inbound_url_file, e.strerror))
         return
 
     try:
-        with open(outbound_url_file, 'rb') as f:
+        with open(outbound_url_file, 'r') as f:
             outbound_urls = [url.rstrip('\n') for url in f.readlines()]
     except EnvironmentError as e:
         logger.error('Reaper: Error while opening "%s" - %s' % (outbound_url_file, e.strerror))
@@ -76,7 +76,7 @@ def reap(file_name):
     logger.info('Storing raw feeds in %s' % file_name)
     harvest = {'inbound': inbound_harvest, 'outbound': outbound_harvest}
 
-    with open(file_name, 'wb') as f:
+    with open(file_name, 'w') as f:
         json.dump(harvest, f, indent=2)
 
 

requirements.txt

@@ -9,5 +9,6 @@ netaddr>=0.7.12,<0.8.0
 pygeoip>=0.3.1,<0.4.0
 requests>=2.10.0,<3.0.0
 sortedcontainers>=0.9.4,<1.6.0
-wsgiref==0.1.2
 unicodecsv>=0.9.4,<0.15.0
+urllib3
+configparser

thresher.py

@@ -1,16 +1,14 @@
-import ConfigParser
+import configparser as ConfigParser
 import bs4
 import datetime
 import feedparser
 import json
 import re
 from logger import get_logger
 from csv import reader
-from itertools import ifilter
 
 logger = get_logger('thresher')
 
-
 def indicator_type(indicator):
     ip_regex = r'^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'
     domain_regex = r'(www\.)?(?P<address>([\d\w.][-\d\w.]{0,253}[\d\w.]+\.)+(xn--vermgensberatung-pwb|xn--vermgensberater-ctb|xn--clchc0ea0b2g2a9gcd|xn--xkc2dl3a5ee0h|xn--mgberp4a5d4ar|xn--xkc2al3hye2a|xn--nqv7fs00ema|xn--mgbc0a9azcg|xn--mgba3a4f16a|xn--lgbbat1ad8j|xn--i1b6b1a6a2e|xn--mgbx4cd0ab|xn--mgbbh1a71e|xn--mgbayh7gpa|xn--mgbaam7a8h|xn--fiq228c5hs|xn--b4w605ferd|xn--6qq986b3xl|cancerresearch|xn--ygbi2ammx|xn--yfro4i67o|xn--fzc2c9e2c|xn--fpcrj9c3d|spreadbetting|international|xn--qcka1pmc|xn--ogbpf8fl|xn--ngbc5azd|xn--mgbab2bd|xn--mgb9awbf|xn--80asehdb|xn--80adxhks|xn--3e0b707e|versicherung|construction|xn--zfr164b|xn--xhq521b|xn--vuq861b|xn--ses554g|xn--s9brj9c|xn--rhqv96g|xn--q9jyb4c|xn--pgbs0dh|xn--kpry57d|xn--kprw13d|xn--j6w193g|xn--hxt814e|xn--h2brj9c|xn--gecrj9c|xn--flw351e|xn--d1acj3b|xn--czr694b|xn--80ao21a|xn--6frz82g|xn--55qw42g|xn--45brj9c|xn--3ds443g|xn--3bst00m|xn--1qqw23a|williamhill|productions|photography|motorcycles|investments|enterprises|engineering|contractors|blackfriday|barclaycard|accountants|xn--wgbl6a|xn--wgbh1c|xn--unup4y|xn--o3cw4h|xn--mxtq1m|xn--kput3i|xn--io0a7i|xn--fiqz9s|xn--fiqs8s|xn--fiq64b|xn--czru2d|xn--czrs0t|xn--cg4bki|xn--9et52u|xn--90a3ac|xn--80aswg|xn--55qx5d|xn--4gbrim|xn--45q11c|xn--30rr7y|vlaanderen|university|technology|restaurant|republican|properties|management|industries|immobilien|healthcare|foundation|eurovision|cuisinella|creditcard|consulting|bnpparibas|associates|apartments|accountant|yodobashi|xn--vhquv|xn--p1acf|xn--nqv7f|xn--l1acc|xn--j1amh|xn--d1alf|xn--c1avg|xn--90ais|vacations|solutions|melbourne|marketing|institute|goldpoint|furniture|financial|equipment|education|directory|community|christmas|bloomberg|aquarelle|amsterdam|allfinanz|yokohama|xn--p1ai|xn--node|ventures|training|supplies|software|services|saarland|redstone|property|plumbing|pictures|pharmacy|partners|mortgage|memorial|marriott|lighting|infiniti|holdings|graphics|football|flsmidth|firmdale|feedback|exchange|everbank|engineer|download|discount|diamonds|democrat|delivery|computer|clothing|cleaning|catering|capetown|business|builders|budapest|brussels|boutique|bargains|barclays|attorney|airforce|zuerich|youtube|whoswho|wedding|website|trading|toshiba|tickets|temasek|systems|surgery|support|spiegel|singles|shriram|shiksha|science|schwarz|schmidt|samsung|reviews|rentals|recipes|realtor|panerai|organic|okinawa|neustar|network|markets|limited|leclerc|latrobe|lacaixa|komatsu|kitchen|hosting|holiday|hangout|hamburg|guitars|gallery|frogans|forsale|flowers|florist|flights|fitness|fishing|finance|fashion|exposed|domains|digital|dentist|cruises|cricket|courses|country|cooking|company|cologne|college|channel|cartier|careers|caravan|capital|auction|android|academy|abogado|yandex|yachts|webcam|voyage|voting|vision|villas|viajes|travel|tienda|tennis|tattoo|taipei|sydney|suzuki|supply|social|schule|school|ryukyu|review|report|repair|reisen|quebec|pictet|piaget|physio|photos|otsuka|oracle|online|nissan|nagoya|museum|moscow|mormon|monash|market|maison|madrid|luxury|london|lawyer|kaufen|juegos|joburg|insure|hiphop|hermes|gratis|google|global|garden|futbol|expert|events|estate|energy|emerck|durban|doosan|direct|design|dental|degree|datsun|dating|credit|condos|coffee|clinic|claims|church|chrome|center|casino|career|camera|berlin|bayern|alsace|agency|active|abbott|world|works|watch|wales|vodka|video|vegas|trust|trade|tours|tools|tokyo|today|tirol|tires|tatar|sucks|style|study|space|solar|shoes|rodeo|rocks|reise|rehab|press|praxi|poker|place|pizza|photo|party|parts|paris|osaka|ninja|nexus|movie|money|miami|media|mango|lotto|lotte|loans|legal|lease|kyoto|koeln|jetzt|irish|house|horse|homes|guide|gripe|green|gmail|globo|glass|gives|gifts|forex|faith|epson|email|deals|dance|dabur|cymru|codes|coach|click|citic|chloe|cheap|cards|canon|build|boats|black|bingo|autos|audio|archi|adult|actor|zone|yoga|work|wiki|wien|wang|voto|vote|toys|town|tips|tech|surf|sohu|site|sexy|scot|saxo|sarl|sale|ruhr|rsvp|rich|rest|reit|qpon|prof|prod|post|porn|pohl|plus|pink|pics|page|nico|news|navy|name|mtpc|moda|mobi|mini|menu|meme|meet|maif|luxe|ltda|loan|link|limo|life|lidl|lgbt|land|kred|kiwi|kddi|jobs|java|info|immo|host|here|help|haus|guru|guge|goog|golf|gold|gift|ggee|gent|gbiz|fund|fish|film|farm|fans|fail|erni|dvag|doha|docs|diet|desi|dclk|date|coop|cool|club|city|chat|cern|cash|casa|care|camp|buzz|bond|blue|bike|best|beer|bank|band|asia|arpa|army|aero|zip|xyz|xxx|xin|wtf|wtc|wme|win|wed|vet|uol|uno|tui|top|tel|tax|soy|sky|sew|scb|sca|sap|rip|rio|ren|red|pub|pro|ovh|org|ooo|onl|ong|one|nyc|ntt|nrw|nra|nhk|ngo|new|net|mtn|mov|moe|mma|mil|lds|lat|krd|kim|jcb|iwc|int|ink|ing|ifm|ibm|how|hiv|gov|gop|goo|gmx|gmo|gle|gdn|gal|frl|foo|fly|fit|fan|eus|esq|edu|eat|dnp|dev|day|dad|crs|com|cfd|ceo|cbn|cat|cal|cab|bzh|boo|bmw|biz|bio|bid|bbc|bar|axa|afl|ads|zw|zm|za|yt|ye|ws|wf|vu|vn|vi|vg|ve|vc|va|uz|uy|us|uk|ug|ua|tz|tw|tv|tt|tr|to|tn|tm|tl|tk|tj|th|tg|tf|td|tc|sz|sy|sx|sv|su|st|sr|so|sn|sm|sl|sk|sj|si|sh|sg|se|sd|sc|sb|sa|rw|ru|rs|ro|re|qa|py|pw|pt|ps|pr|pn|pm|pl|pk|ph|pg|pf|pe|pa|om|nz|nu|nr|np|no|nl|ni|ng|nf|ne|nc|na|mz|my|mx|mw|mv|mu|mt|ms|mr|mq|mp|mo|mn|mm|ml|mk|mh|mg|me|md|mc|ma|ly|lv|lu|lt|ls|lr|lk|li|lc|lb|la|kz|ky|kw|kr|kp|kn|km|ki|kh|kg|ke|jp|jo|jm|je|it|is|ir|iq|io|in|im|il|ie|id|hu|ht|hr|hn|hm|hk|gy|gw|gu|gt|gs|gr|gq|gp|gn|gm|gl|gi|gh|gg|gf|ge|gd|gb|ga|fr|fo|fm|fk|fj|fi|eu|et|es|er|eg|ee|ec|dz|do|dm|dk|dj|de|cz|cy|cx|cw|cv|cu|cr|co|cn|cm|cl|ck|ci|ch|cg|cf|cd|cc|ca|bz|by|bw|bv|bt|bs|br|bo|bn|bm|bj|bi|bh|bg|bf|be|bd|bb|ba|az|ax|aw|au|at|as|ar|aq|ao|an|am|al|ai|ag|af|ae|ad|ac))'
@@ -102,7 +100,7 @@ def process_packetmail(response, source, direction):
     data = []
     filter_comments = lambda x: not x[0].startswith('#')
     try:
-        for line in ifilter(filter_comments,
+        for line in filter(filter_comments,
                             reader(response.splitlines(), delimiter=';')):
             i = line[0]
             date = line[1].split(' ')[1]
@@ -210,7 +208,7 @@ def thresh(input_file, output_file):
             pass
 
     logger.info('Storing parsed data in %s', output_file)
-    with open(output_file, 'wb') as f:
+    with open(output_file, 'w') as f:
         json.dump(harvest, f, indent=2)
 
 

winnower.py

@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-import ConfigParser
+import configparser as ConfigParser
 import csv
 import datetime as dt
 import dnsdb_query