Skip to content

Commit ccbbbc9

Browse files
2shortplanks2shortplanks
authored
Merge pull request maxmind#79 from maxmind/greg/network
Add support for network output and improve performance
2 parents ba112f1 + 740e941 commit ccbbbc9

File tree

11 files changed

+353
-102
lines changed

11 files changed

+353
-102
lines changed

‎HISTORY.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ History
66
2.10.0
77
++++++++++++++++++
88

9+
* The ``network`` attribute was added to ``geoip2.record.Traits``,
10+
``geoip2.model.AnonymousIP``, ``geoip2.model.ASN``,
11+
``geoip2.model.ConnectionType``, ``geoip2.model.Domain``,
12+
and ``geoip2.model.ISP``. This is an ``ipaddress.IPv4Network`` or an
13+
``ipaddress.IPv6Network``. This is the largest network where all of the
14+
fields besides ``ip_address`` have the same value. GitHub #79.
915
* Python 3.3 and 3.4 are no longer supported.
1016
* Updated documentation of anonymizer attributes - ``is_anonymous_vpn`` and
1117
``is_hosting_provider`` - to be more descriptive.

‎README.rst‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ Web Service Example
9595
44.9733
9696
>>> response.location.longitude
9797
-93.2323
98+
>>>
99+
>>> response.traits.network
100+
IPv4Network('128.101.101.101/32')
98101
99102
Web Service Client Exceptions
100103
-----------------------------
@@ -156,6 +159,10 @@ City Database
156159
44.9733
157160
>>> response.location.longitude
158161
-93.2323
162+
>>>
163+
>>> response.traits.network
164+
IPv4Network('128.101.101.0/24')
165+
>>>
159166
>>> reader.close()
160167
161168
Anonymous IP Database
@@ -182,7 +189,9 @@ Anonymous IP Database
182189
>>> response.is_tor_exit_node
183190
True
184191
>>> response.ip_address
185-
'128.101.101.101'
192+
'85.25.43.84'
193+
>>> response.network
194+
IPv4Network('85.25.43.0/24')
186195
>>> reader.close()
187196
188197
ASN Database
@@ -218,6 +227,8 @@ Connection-Type Database
218227
'Corporate'
219228
>>> response.ip_address
220229
'128.101.101.101'
230+
>>> response.network
231+
IPv4Network('128.101.101.101/24')
221232
>>> reader.close()
222233
223234
@@ -284,6 +295,10 @@ Enterprise Database
284295
44.9733
285296
>>> response.location.longitude
286297
-93.2323
298+
>>>
299+
>>> response.traits.network
300+
IPv4Network('128.101.101.0/24')
301+
287302
288303
ISP Database
289304
^^^^^^^^^^^^
@@ -307,7 +322,9 @@ ISP Database
307322
>>> response.organization
308323
'Telstra Internet'
309324
>>> response.ip_address
310-
'128.101.101.101'
325+
'1.128.0.0'
326+
>>> response.network
327+
IPv4Network('1.128.0.0/16')
311328
>>> reader.close()
312329
313330
Database Reader Exceptions

‎examples/benchmark.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/python
2+
# -*- coding: utf-8 -*-
3+
4+
from __future__ import print_function
5+
6+
import argparse
7+
import geoip2.database
8+
import random
9+
import socket
10+
import struct
11+
import timeit
12+
13+
parser = argparse.ArgumentParser(description='Benchmark maxminddb.')
14+
parser.add_argument(
15+
'--count', default=250000, type=int, help='number of lookups')
16+
parser.add_argument('--mode', default=0, type=int, help='reader mode to use')
17+
parser.add_argument(
18+
'--file', default='GeoIP2-City.mmdb', help='path to mmdb file')
19+
20+
args = parser.parse_args()
21+
22+
reader = geoip2.database.Reader(args.file, mode=args.mode)
23+
24+
25+
def lookup_ip_address():
26+
ip = socket.inet_ntoa(struct.pack('!L', random.getrandbits(32)))
27+
try:
28+
record = reader.city(str(ip))
29+
except geoip2.errors.AddressNotFoundError:
30+
pass
31+
32+
33+
elapsed = timeit.timeit(
34+
'lookup_ip_address()',
35+
setup='from __main__ import lookup_ip_address',
36+
number=args.count)
37+
38+
print(args.count / elapsed, 'lookups per second')

‎geoip2/compat.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,18 @@ def compat_ip_address(address):
1212
if isinstance(address, bytes):
1313
address = address.decode()
1414
return ipaddress.ip_address(address)
15+
16+
def compat_ip_network(network, strict=True):
17+
"""Intended for internal use only."""
18+
if isinstance(network, bytes):
19+
network = network.decode()
20+
return ipaddress.ip_network(network, strict)
1521
else:
1622

1723
def compat_ip_address(address):
1824
"""Intended for internal use only."""
1925
return ipaddress.ip_address(address)
26+
27+
def compat_ip_network(network, strict=True):
28+
"""Intended for internal use only."""
29+
return ipaddress.ip_network(network, strict)

‎geoip2/database.py‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ def __init__(self, fileish, locales=None, mode=MODE_AUTO):
8383
if locales is None:
8484
locales = ['en']
8585
self._db_reader = maxminddb.open_database(fileish, mode)
86+
self._db_type = self._db_reader.metadata().database_type
8687
self._locales = locales
8788

8889
def __enter__(self):
@@ -179,25 +180,27 @@ def isp(self, ip_address):
179180
ip_address)
180181

181182
def _get(self, database_type, ip_address):
182-
if database_type not in self.metadata().database_type:
183+
if database_type not in self._db_type:
183184
caller = inspect.stack()[2][3]
184185
raise TypeError("The %s method cannot be used with the "
185-
"%s database" %
186-
(caller, self.metadata().database_type))
187-
record = self._db_reader.get(ip_address)
186+
"%s database" % (caller, self._db_type))
187+
(record, prefix_len) = self._db_reader.get_with_prefix_len(ip_address)
188188
if record is None:
189189
raise geoip2.errors.AddressNotFoundError(
190190
"The address %s is not in the database." % ip_address)
191-
return record
191+
return (record, prefix_len)
192192

193193
def _model_for(self, model_class, types, ip_address):
194-
record = self._get(types, ip_address)
195-
record.setdefault('traits', {})['ip_address'] = ip_address
194+
(record, prefix_len) = self._get(types, ip_address)
195+
traits = record.setdefault('traits', {})
196+
traits['ip_address'] = ip_address
197+
traits['prefix_len'] = prefix_len
196198
return model_class(record, locales=self._locales)
197199

198200
def _flat_model_for(self, model_class, types, ip_address):
199-
record = self._get(types, ip_address)
201+
(record, prefix_len) = self._get(types, ip_address)
200202
record['ip_address'] = ip_address
203+
record['prefix_len'] = prefix_len
201204
return model_class(record)
202205

203206
def metadata(self):

‎geoip2/models.py‎

Lines changed: 70 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@
1111
1212
"""
1313
# pylint: disable=too-many-instance-attributes,too-few-public-methods
14+
import ipaddress
1415
from abc import ABCMeta
1516

1617
import geoip2.records
18+
from geoip2.compat import compat_ip_network
1719
from geoip2.mixins import SimpleEquality
1820

1921

@@ -305,13 +307,37 @@ class SimpleModel(SimpleEquality):
305307

306308
__metaclass__ = ABCMeta
307309

310+
def __init__(self, raw):
311+
self.raw = raw
312+
self._network = None
313+
self._prefix_len = raw.get('prefix_len')
314+
self.ip_address = raw.get('ip_address')
315+
308316
def __repr__(self):
309317
# pylint: disable=no-member
310318
return '{module}.{class_name}({data})'.format(
311319
module=self.__module__,
312320
class_name=self.__class__.__name__,
313321
data=str(self.raw))
314322

323+
@property
324+
def network(self):
325+
"""The network for the record"""
326+
# This code is duplicated for performance reasons
327+
# pylint: disable=duplicate-code
328+
network = self._network
329+
if isinstance(network, (ipaddress.IPv4Network, ipaddress.IPv6Network)):
330+
return network
331+
332+
ip_address = self.ip_address
333+
prefix_len = self._prefix_len
334+
if ip_address is None or prefix_len is None:
335+
return None
336+
network = compat_ip_network("{}/{}".format(ip_address, prefix_len),
337+
False)
338+
self._network = network
339+
return network
340+
315341

316342
class AnonymousIP(SimpleModel):
317343
"""Model class for the GeoIP2 Anonymous IP.
@@ -359,17 +385,23 @@ class AnonymousIP(SimpleModel):
359385
The IP address used in the lookup.
360386
361387
:type: unicode
388+
389+
.. attribute:: network
390+
391+
The network associated with the record. In particular, this is the
392+
largest network where all of the fields besides ip_address have the same
393+
value.
394+
395+
:type: ipaddress.IPv4Network or ipaddress.IPv6Network
362396
"""
363397
def __init__(self, raw):
398+
super(AnonymousIP, self).__init__(raw)
364399
self.is_anonymous = raw.get('is_anonymous', False)
365400
self.is_anonymous_vpn = raw.get('is_anonymous_vpn', False)
366401
self.is_hosting_provider = raw.get('is_hosting_provider', False)
367402
self.is_public_proxy = raw.get('is_public_proxy', False)
368403
self.is_tor_exit_node = raw.get('is_tor_exit_node', False)
369404

370-
self.ip_address = raw.get('ip_address')
371-
self.raw = raw
372-
373405

374406
class ASN(SimpleModel):
375407
"""Model class for the GeoLite2 ASN.
@@ -394,15 +426,22 @@ class ASN(SimpleModel):
394426
The IP address used in the lookup.
395427
396428
:type: unicode
429+
430+
.. attribute:: network
431+
432+
The network associated with the record. In particular, this is the
433+
largest network where all of the fields besides ip_address have the same
434+
value.
435+
436+
:type: ipaddress.IPv4Network or ipaddress.IPv6Network
397437
"""
398438

399439
# pylint:disable=too-many-arguments
400440
def __init__(self, raw):
441+
super(ASN, self).__init__(raw)
401442
self.autonomous_system_number = raw.get('autonomous_system_number')
402443
self.autonomous_system_organization = raw.get(
403444
'autonomous_system_organization')
404-
self.ip_address = raw.get('ip_address')
405-
self.raw = raw
406445

407446

408447
class ConnectionType(SimpleModel):
@@ -428,11 +467,18 @@ class ConnectionType(SimpleModel):
428467
The IP address used in the lookup.
429468
430469
:type: unicode
470+
471+
.. attribute:: network
472+
473+
The network associated with the record. In particular, this is the
474+
largest network where all of the fields besides ip_address have the same
475+
value.
476+
477+
:type: ipaddress.IPv4Network or ipaddress.IPv6Network
431478
"""
432479
def __init__(self, raw):
480+
super(ConnectionType, self).__init__(raw)
433481
self.connection_type = raw.get('connection_type')
434-
self.ip_address = raw.get('ip_address')
435-
self.raw = raw
436482

437483

438484
class Domain(SimpleModel):
@@ -452,11 +498,18 @@ class Domain(SimpleModel):
452498
453499
:type: unicode
454500
501+
.. attribute:: network
502+
503+
The network associated with the record. In particular, this is the
504+
largest network where all of the fields besides ip_address have the same
505+
value.
506+
507+
:type: ipaddress.IPv4Network or ipaddress.IPv6Network
508+
455509
"""
456510
def __init__(self, raw):
511+
super(Domain, self).__init__(raw)
457512
self.domain = raw.get('domain')
458-
self.ip_address = raw.get('ip_address')
459-
self.raw = raw
460513

461514

462515
class ISP(ASN):
@@ -494,6 +547,14 @@ class ISP(ASN):
494547
The IP address used in the lookup.
495548
496549
:type: unicode
550+
551+
.. attribute:: network
552+
553+
The network associated with the record. In particular, this is the
554+
largest network where all of the fields besides ip_address have the same
555+
value.
556+
557+
:type: ipaddress.IPv4Network or ipaddress.IPv6Network
497558
"""
498559

499560
# pylint:disable=too-many-arguments

0 commit comments

Comments
 (0)