Skip to content

Commit 33b5950

Browse files
committed
usage of oxl-utils
1 parent 2e940ec commit 33b5950

File tree

2 files changed

+8
-34
lines changed

2 files changed

+8
-34
lines changed

src/builder/enrich_data.py

+6-33
Original file line numberDiff line numberDiff line change
@@ -4,56 +4,29 @@
44
from os import system as os_shell
55
from ipaddress import ip_address
66

7-
from dns.resolver import Resolver, NoAnswer, NXDOMAIN, LifetimeTimeout, NoNameservers
8-
from dns.exception import SyntaxError as DNSSyntaxError
7+
from oxl_utils.ps import process_list_in_threads
8+
from oxl_utils.net import resolve_dns
99
from maxminddb import open_database as mmdb_database
1010

1111
from config import *
1212

13-
dns_resolver = Resolver(configure=False)
14-
dns_resolver.lifetime = 0.1
15-
dns_resolver.timeout = 0.1
16-
dns_resolver.nameservers = NAMESERVERS
13+
1714
ptr_cache_lock = Lock()
1815

1916

2017
def lookup_ptrs(reports: list[dict]) -> dict:
21-
batch = 0
2218
ptrs = {}
23-
reports_lst = list(reports)
2419

2520
def _ptr_lookup(ip: str):
2621
try:
27-
ptr = [p.to_text() for p in dns_resolver.resolve_address(ip)][0]
22+
ptr = resolve_dns(ip, t='PTR')[0]
2823
with ptr_cache_lock:
2924
ptrs[ip] = ptr
3025

31-
except (IndexError, NoAnswer, NXDOMAIN, DNSSyntaxError, NoNameservers, LifetimeTimeout):
26+
except IndexError:
3227
pass
3328

34-
while batch * PTR_LOOKUP_THREADS < len(reports_lst):
35-
threads = []
36-
for i in range(PTR_LOOKUP_THREADS):
37-
idx = (batch * PTR_LOOKUP_THREADS) + i
38-
if idx > len(reports_lst) - 1:
39-
break
40-
41-
threads.append(Thread(
42-
target=_ptr_lookup,
43-
kwargs={'ip': list(reports_lst)[idx]},
44-
))
45-
46-
for t in threads:
47-
t.start()
48-
49-
threads_done = False
50-
while not threads_done:
51-
threads_done = all(not t.is_alive() for t in threads)
52-
sleep(0.05)
53-
54-
batch += 1
55-
56-
del reports_lst
29+
process_list_in_threads(callback=_ptr_lookup, to_process=list(reports), key='ip', parallel=PTR_LOOKUP_THREADS)
5730
return ptrs
5831

5932

src/builder/requirements.txt

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
netaddr
22
mmdb_writer
33
maxminddb
4-
dnspython
4+
dnspython
5+
oxl-utils

0 commit comments

Comments
 (0)