Skip to content

Commit d443950

Browse files
committed
builder updates
1 parent 54e8326 commit d443950

File tree

6 files changed

+58
-16
lines changed

6 files changed

+58
-16
lines changed

src/build.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from config import *
88
from util import log
9-
from enrich_data import ip_asn_info
9+
from enrich_data import ip_asn_info, net_asn_info
1010
from write import write_ip_asn, write_nets
1111

1212

@@ -36,7 +36,9 @@ def build_dbs_ip_asn(reports: dict, ptrs: dict, lookup_lists: dict, networks: di
3636
lookup_lists=lookup_lists,
3737
)
3838
net = {'network': networks[key][networks['ip_to_net'][ip]]}
39-
net['network'].pop('ipv')
39+
if 'ipv' in net['network']:
40+
net['network'].pop('ipv')
41+
4042
net_sm = {'network': {
4143
'reported_ips': net['network']['reported_ips'],
4244
'reputation': net['network']['reputation'],
@@ -140,7 +142,9 @@ def build_dbs_net(networks: dict):
140142

141143
for n, nv in net_list.items():
142144
ipv = nv.pop('ipv')
145+
nv = {**nv, **net_asn_info(n)}
143146
n = f"{n}/{BGP_NET_SIZE[ipv]}"
147+
144148
if ipv =='4':
145149
json4[n] = nv
146150

src/config.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33
INFO_CATEGORIES = ['hosting', 'vpn', 'proxy']
44
CATEGORIES = ['bot', 'probe', 'rate', 'attack', 'crawler']
55

6-
BASE_PATH = '/tmp/risk-db'
6+
BASE_PATH = Path('/tmp/risk-db')
77
MMDB_DESCRIPTION = 'OXL RISK-Database - risk.oxl.app (CC BY-SA 4.0)'
88
REPORT_COOLDOWN = 60
99
ASN_JSON_FILE = Path('/tmp/asn_full.json') # source: https://github.com/O-X-L/geoip-asn
10+
ASN_MMDB_FILE = BASE_PATH / 'oxl_geoip_asn.mmdb' # source: https://github.com/O-X-L/geoip-asn
1011
SRC_PATH = Path(__file__).resolve().parent
1112
ASN_FILE_HOSTING = SRC_PATH / 'kind' / 'hosting.txt'
1213
ASN_FILE_PROXY = SRC_PATH / 'kind' / 'proxy.txt'

src/enrich_data.py

+20-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ def load_lookup_lists() -> dict:
8686

8787

8888
def ip_asn_info(ip: str, reports: dict, lookup_lists: dict, ptrs: dict) -> dict:
89-
with mmdb_database(f'{BASE_PATH}/oxl_geoip_asn.mmdb') as m:
89+
with mmdb_database(ASN_MMDB_FILE) as m:
9090
ip_md = m.get(ip)
9191

9292
try:
@@ -106,6 +106,7 @@ def ip_asn_info(ip: str, reports: dict, lookup_lists: dict, ptrs: dict) -> dict:
106106
},
107107
'url': {
108108
'asn': f'https://risk.oxl.app/api/asn/{asn}',
109+
'net': f'https://risk.oxl.app/api/net/{ip}',
109110
'ipinfo': f'https://ipinfo.io/{ip}',
110111
'shodan': f'https://www.shodan.io/host/{ip}',
111112
},
@@ -130,3 +131,21 @@ def ip_asn_info(ip: str, reports: dict, lookup_lists: dict, ptrs: dict) -> dict:
130131
'full': d,
131132
'small': d_small,
132133
}
134+
135+
136+
def net_asn_info(ip: str) -> dict:
137+
with mmdb_database(ASN_MMDB_FILE) as m:
138+
ip_md = m.get(ip)
139+
140+
try:
141+
asn = int(ip_md['asn'][2:])
142+
143+
except ValueError:
144+
return {}
145+
146+
return {
147+
'asn': asn,
148+
'url': {
149+
'asn': f'https://risk.oxl.app/api/asn/{asn}',
150+
}
151+
}

src/main.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ def main():
2727
lookup_lists = load_lookup_lists()
2828

2929
log('Building and writing DBs')
30-
build_dbs_ip_asn(reports=reports, ptrs=ptrs, lookup_lists=lookup_lists, networks=networks)
3130
build_dbs_net(networks=networks)
31+
build_dbs_ip_asn(reports=reports, ptrs=ptrs, lookup_lists=lookup_lists, networks=networks)
3232

3333
log('Done')
3434

src/reputation.py

+4-11
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
from ipaddress import IPv4Address, AddressValueError, IPv4Interface, IPv6Interface
2-
31
from config import *
2+
from util import get_ip_version, get_network_address
3+
44

55
# pylint: disable=W0613
66
def _reporter_reputation(r: dict) -> int:
@@ -72,15 +72,8 @@ def reports_by_network_reputation(reports: list[dict]) -> dict:
7272
ip_to_net = {}
7373

7474
for r in reports:
75-
try:
76-
IPv4Address(r['ip'])
77-
ipv = '4'
78-
n = IPv4Interface(f"{r['ip']}/{BGP_NET_SIZE[ipv]}").network.network_address.compressed
79-
80-
except AddressValueError:
81-
ipv = '6'
82-
n = IPv6Interface(f"{r['ip']}/{BGP_NET_SIZE[ipv]}").network.network_address.compressed
83-
75+
ipv = get_ip_version(r['ip'])
76+
n = get_network_address(r['ip'])
8477
reputation = _reporter_reputation(r)
8578

8679
_save_net_report(dst=reported_nets_all, r=r, n=n)

src/util.py

+25
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,32 @@
11
from time import time
2+
from ipaddress import IPv4Address, AddressValueError, IPv4Interface, IPv6Interface
3+
4+
from config import BGP_NET_SIZE
25

36
start_time = time()
47

58

69
def log(msg: str):
710
print(f'{msg} ({int(time() - start_time)}s)')
11+
12+
13+
def get_ip_version(ip: str) -> str:
14+
try:
15+
IPv4Address(ip)
16+
return '4'
17+
18+
except AddressValueError:
19+
return '6'
20+
21+
22+
def get_network_address(ip: str) -> str:
23+
try:
24+
IPv4Address(ip)
25+
return IPv4Interface(f"{ip}/{BGP_NET_SIZE['4']}").network.network_address.compressed
26+
27+
except AddressValueError:
28+
return IPv6Interface(f"{ip}/{BGP_NET_SIZE['6']}").network.network_address.compressed
29+
30+
31+
# def get_network_cidr(ip: str) -> str:
32+
# return f"{get_network_address(ip)}/{BGP_NET_SIZE[get_ip_version(ip)]}"

0 commit comments

Comments
 (0)