Skip to content

Commit 6119f64

Browse files
tarcisiozftarcisiozf
authored
Merge pull request #30 from bilalfastnu/update-db
add support of isp and connection type
2 parents a4eb1bf + 248c5e6 commit 6119f64

File tree

6 files changed

+164
-9
lines changed

6 files changed

+164
-9
lines changed

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@
1010
"format": "standard --fix",
1111
"lint": "standard",
1212
"update-geo-data": "./update-geo-db.sh",
13-
"update-asn-data": "./update-asn-db.sh"
13+
"update-asn-data": "./update-asn-db.sh",
14+
"update-isp-data": "./update-isp-db.sh",
15+
"update-connection-type-data": "./update-connection-type-db.sh"
1416
},
1517
"dependencies": {
1618
"async": "^3.2.1",

‎test/integration.js‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,11 +118,12 @@ describe('RPC integration', () => {
118118
client.request(query, (err, data) => {
119119
try {
120120
if (err) throw err
121-
122121
assert.strictEqual(data[0], '8.8.8.8')
123122
assert.ok(data[1].geo)
124123
assert.ok(data[1].dns)
125124
assert.ok(data[1].asn)
125+
assert.ok(data[1].isp)
126+
assert.ok(data[1].connectionType)
126127

127128
done()
128129
} catch (err) {
@@ -145,6 +146,8 @@ describe('RPC integration', () => {
145146
assert.ok(data[1].geo)
146147
assert.ok(data[1].dns)
147148
assert.ok(data[1].asn)
149+
assert.ok(data[1].isp)
150+
assert.ok(data[1].connectionType)
148151

149152
done()
150153
} catch (err) {
@@ -167,6 +170,8 @@ describe('RPC integration', () => {
167170
assert.ok(data[1].geo)
168171
assert.ok(data[1].dns)
169172
assert.ok(data[1].asn)
173+
assert.ok(data[1].isp)
174+
assert.ok(data[1].connectionType)
170175

171176
done()
172177
} catch (err) {
@@ -198,4 +203,50 @@ describe('RPC integration', () => {
198203
}
199204
})
200205
}).timeout(7000)
206+
207+
it('geo-isp: retrieves ips ISP', (done) => {
208+
const query = {
209+
action: 'getIpIsp',
210+
args: ['53.1.34.21']
211+
}
212+
213+
client.request(query, (err, data) => {
214+
try {
215+
if (err) throw err
216+
217+
assert.strictEqual(
218+
data[0], '53.1.34.21', 'result contains queried ip'
219+
)
220+
221+
const res = data[1]
222+
assert.strictEqual(res.isp, 'Mercedes-Benz Group')
223+
done()
224+
} catch (err) {
225+
done(err)
226+
}
227+
})
228+
}).timeout(7000)
229+
230+
it('getIpConnectionType: retrieves ips connection type', (done) => {
231+
const query = {
232+
action: 'getIpConnectionType',
233+
args: ['53.1.34.21']
234+
}
235+
236+
client.request(query, (err, data) => {
237+
try {
238+
if (err) throw err
239+
240+
assert.strictEqual(
241+
data[0], '53.1.34.21', 'result contains queried ip'
242+
)
243+
244+
const res = data[1]
245+
assert.strictEqual(res.connection_type, 'Cable/DSL')
246+
done()
247+
} catch (err) {
248+
done(err)
249+
}
250+
})
251+
}).timeout(7000)
201252
})

‎update-connection-type-db.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
trap '[[ -f "${TMPFILE}" ]] && rm --force "${TMPFILE}"' 'EXIT'
4+
5+
readonly DESTDIR="${DESTDIR:-mmdb}"
6+
readonly ARCHIVE="${DESTDIR}/GeoIP2-Connection-Type.tar.gz"
7+
readonly LOCAL_TIMESTAMP="$(date +'%s' --reference="${ARCHIVE}" 2>/dev/null)"
8+
readonly MAXMIND_LICENSE="${MAXMIND_LICENSE:-$(head --quiet --lines=1 'MAXMIND_KEY' 2>/dev/null)}"
9+
10+
[[ "${MAXMIND_LICENSE:-}" =~ ^[[:alnum:]]+$ ]] || {
11+
echo "${BASH_SOURCE[0]##*/}: fatal error: license key has not been provided." >&2
12+
exit 1
13+
}
14+
15+
[[ -d "${DESTDIR}" ]] \
16+
|| mkdir --parents "${DESTDIR}"
17+
18+
curl --silent --fail --output "${ARCHIVE}" --time-cond "${ARCHIVE}" --location "https://download.maxmind.com/app/geoip_download?edition_id=GeoIP2-Connection-Type&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" || {
19+
echo "${BASH_SOURCE[0]##*/}: fatal error: download failed." >&2
20+
exit 1
21+
}
22+
23+
[[ "${LOCAL_TIMESTAMP}" = "$(date +'%s' --reference="${ARCHIVE}" 2>/dev/null)" ]] || {
24+
readonly TMPFILE="$(mktemp -p "${DESTDIR}" -t 'GeoIP2-Connection-Type.mmdb.XXXXXXXXXXXXXXXX')"
25+
26+
tar --extract --directory "${DESTDIR}" --strip-components=1 --wildcards '*.mmdb' \
27+
--transform='flags=rSH;s|GeoIP2-Connection-Type\.mmdb$|'"${TMPFILE##*/}"'|g' --file "${ARCHIVE}" || exit 1
28+
29+
cat "${TMPFILE}" >"${DESTDIR}/GeoIP2-Connection-Type.mmdb"
30+
}

‎update-isp-db.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
trap '[[ -f "${TMPFILE}" ]] && rm --force "${TMPFILE}"' 'EXIT'
4+
5+
readonly DESTDIR="${DESTDIR:-mmdb}"
6+
readonly ARCHIVE="${DESTDIR}/GeoIP2-ISP.tar.gz"
7+
readonly LOCAL_TIMESTAMP="$(date +'%s' --reference="${ARCHIVE}" 2>/dev/null)"
8+
readonly MAXMIND_LICENSE="${MAXMIND_LICENSE:-$(head --quiet --lines=1 'MAXMIND_KEY' 2>/dev/null)}"
9+
10+
[[ "${MAXMIND_LICENSE:-}" =~ ^[[:alnum:]]+$ ]] || {
11+
echo "${BASH_SOURCE[0]##*/}: fatal error: license key has not been provided." >&2
12+
exit 1
13+
}
14+
15+
[[ -d "${DESTDIR}" ]] \
16+
|| mkdir --parents "${DESTDIR}"
17+
18+
curl --silent --fail --output "${ARCHIVE}" --time-cond "${ARCHIVE}" --location "https://download.maxmind.com/app/geoip_download?edition_id=GeoIP2-ISP&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" || {
19+
echo "${BASH_SOURCE[0]##*/}: fatal error: download failed." >&2
20+
exit 1
21+
}
22+
23+
[[ "${LOCAL_TIMESTAMP}" = "$(date +'%s' --reference="${ARCHIVE}" 2>/dev/null)" ]] || {
24+
readonly TMPFILE="$(mktemp -p "${DESTDIR}" -t 'GeoIP2-ISP.mmdb.XXXXXXXXXXXXXXXX')"
25+
26+
tar --extract --directory "${DESTDIR}" --strip-components=1 --wildcards '*.mmdb' \
27+
--transform='flags=rSH;s|GeoIP2-ISP\.mmdb$|'"${TMPFILE##*/}"'|g' --file "${ARCHIVE}" || exit 1
28+
29+
cat "${TMPFILE}" >"${DESTDIR}/GeoIP2-ISP.mmdb"
30+
}

‎workers/api.net.util.wrk.js‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ const { WrkApi } = require('bfx-wrk-api')
1212
const geoIp = require('geoip-lite')
1313
const fs = require('fs')
1414

15-
const newDb = path.join(__dirname, '..', 'mmdb', 'GeoLite2-ASN.mmdb')
15+
const asnMMDB = path.join(__dirname, '..', 'mmdb', 'GeoLite2-ASN.mmdb')
16+
const ispMMDB = path.join(__dirname, '..', 'mmdb', 'GeoIP2-ISP.mmdb')
17+
const connectionTypeMMDB = path.join(__dirname, '..', 'mmdb', 'GeoIP2-Connection-Type.mmdb')
1618

1719
class WrkUtilNetApi extends WrkApi {
1820
constructor (conf, ctx) {
@@ -35,8 +37,10 @@ class WrkUtilNetApi extends WrkApi {
3537
switch (type) {
3638
case 'api_bfx':
3739
ctx.lru_0 = this.lru_0
38-
ctx.asn_db = this.asnDb
40+
ctx.asnDb = this.asnDb
3941
ctx.geoIp = this.geoIp
42+
ctx.ispDb = this.ispDb
43+
ctx.connectionTypeDb = this.connectionTypeDb
4044
break
4145
}
4246

@@ -62,14 +66,30 @@ class WrkUtilNetApi extends WrkApi {
6266

6367
this.geoIp = geoIp
6468

65-
this.asnDb = await maxmind.open(newDb, {
69+
this.asnDb = await maxmind.open(asnMMDB, {
6670
watchForUpdates: true,
6771
watchForUpdatesNonPersistent: true,
6872
watchForUpdatesHook: () => {
6973
process.stdout.write('ASN database has been reloaded\n')
7074
}
7175
})
7276

77+
this.ispDb = await maxmind.open(ispMMDB, {
78+
watchForUpdates: true,
79+
watchForUpdatesNonPersistent: true,
80+
watchForUpdatesHook: () => {
81+
process.stdout.write('ISP database has been reloaded\n')
82+
}
83+
})
84+
85+
this.connectionTypeDb = await maxmind.open(connectionTypeMMDB, {
86+
watchForUpdates: true,
87+
watchForUpdatesNonPersistent: true,
88+
watchForUpdatesHook: () => {
89+
process.stdout.write('Connection type database has been reloaded\n')
90+
}
91+
})
92+
7393
this.geoIp.startWatchingDataUpdate((err) => {
7494
if (err instanceof Error) {
7595
process.stderr.write(`ERR: ${err.message}\n`)
@@ -92,9 +112,17 @@ class WrkUtilNetApi extends WrkApi {
92112
}
93113

94114
testIfDbExists () {
95-
if (!fs.existsSync(newDb)) {
115+
if (!fs.existsSync(asnMMDB)) {
96116
throw new Error('GEO_DB_NOT_INSTALLED - run `npm run update-asn-data`')
97117
}
118+
119+
if (!fs.existsSync(ispMMDB)) {
120+
throw new Error('ISP_DB_NOT_INSTALLED - run `npm run update-isp-data`')
121+
}
122+
123+
if (!fs.existsSync(connectionTypeMMDB)) {
124+
throw new Error('CONNECTION_TYPE_DB_NOT_INSTALLED - run `npm run update-connection-type-data`')
125+
}
98126
}
99127
}
100128

‎workers/loc.api/net.util.js‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,16 @@ class UtilNet extends Api {
1313

1414
getIpInfo (space, ip, cb) {
1515
const geoData = this.ctx.geoIp.lookup(ip)
16-
const asnData = this.ctx.asn_db.get(ip)
16+
const asnData = this.ctx.asnDb.get(ip)
17+
const ispData = this.ctx.ispDb.get(ip)
18+
const connectionTypeData = this.ctx.connectionTypeDb.get(ip)
1719

1820
dns.reverse(ip, (err, dnsData) => {
1921
if (err) return cb(err)
2022

2123
const res = [
2224
ip,
23-
{ geo: geoData, dns: dnsData, asn: asnData }
25+
{ geo: geoData, dns: dnsData, asn: asnData, isp: ispData, connectionType: connectionTypeData }
2426
]
2527

2628
cb(null, res)
@@ -49,7 +51,19 @@ class UtilNet extends Api {
4951
}
5052

5153
getIpAsn (space, ip, cb) {
52-
const res = this.ctx.asn_db.get(ip)
54+
const res = this.ctx.asnDb.get(ip)
55+
56+
cb(null, [ip, res])
57+
}
58+
59+
getIpIsp (space, ip, cb) {
60+
const res = this.ctx.ispDb.get(ip)
61+
62+
cb(null, [ip, res])
63+
}
64+
65+
getIpConnectionType (space, ip, cb) {
66+
const res = this.ctx.connectionTypeDb.get(ip)
5367

5468
cb(null, [ip, res])
5569
}

0 commit comments

Comments
 (0)