From 5f44f4331c814613876dca9ba496c4841747c3a8 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 17 Nov 2021 09:01:42 -0800 Subject: [PATCH 1/6] Update test databases --- tests/data | 2 +- tests/database_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/data b/tests/data index cbaa463..2b37923 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit cbaa463dc6950ababbf678ca85fb3833b81c76d3 +Subproject commit 2b37923df61aa3b5fb6c7edfbf4dc5fafa10258a diff --git a/tests/database_test.py b/tests/database_test.py index 4084ad4..5f0956d 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -117,7 +117,7 @@ def test_city(self) -> None: self.assertEqual( record.country.name, "United Kingdom", "The default locale is en" ) - self.assertEqual(record.country.is_in_european_union, True) + self.assertEqual(record.country.is_in_european_union, False) self.assertEqual( record.location.accuracy_radius, 100, "The accuracy_radius is populated" ) @@ -156,7 +156,7 @@ def test_country(self) -> None: record.traits.ip_address, "81.2.69.160", "IP address is added to model" ) self.assertEqual(record.traits.network, ipaddress.ip_network("81.2.69.160/27")) - self.assertEqual(record.country.is_in_european_union, True) + self.assertEqual(record.country.is_in_european_union, False) self.assertEqual(record.registered_country.is_in_european_union, False) reader.close() From 96073708f018a20340dd18aafe193071d420e310 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 17 Nov 2021 09:34:59 -0800 Subject: [PATCH 2/6] Add MCC/MNC support --- HISTORY.rst | 11 +++++++++++ geoip2/models.py | 21 +++++++++++++++++++++ geoip2/records.py | 26 ++++++++++++++++++++++++- tests/database_test.py | 43 +++++++++++++++++++++++++----------------- 4 files changed, 83 insertions(+), 18 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 580ff98..44f1779 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,8 +1,19 @@ + .. :changelog: History ------- +4.5.0 +++++++++++++++++++ + +* Support for mobile country code (MCC) and mobile network codes (MNC) was + added for the GeoIP2 ISP and Enterprise databases as well as the GeoIP2 + City and Insights web services. ``mobile_country_code`` and + ``mobile_network_code`` attributes were added to ``geoip2.model.ISP`` + for the GeoIP2 ISP database and ``geoip2.record.Traits`` for the + Enterprise database and the GeoIP2 City and Insights web services. + 4.4.0 (2021-09-24) ++++++++++++++++++ diff --git a/geoip2/models.py b/geoip2/models.py index 07c607a..2e177cd 100644 --- a/geoip2/models.py +++ b/geoip2/models.py @@ -577,6 +577,23 @@ class ISP(ASN): :type: unicode + .. attribute: mobile_country_code + + The `mobile country code (MCC) + `_ associated with the + IP address and ISP. + + :type: str + + .. attribute: mobile_network_code + + The `mobile network code (MCC) + `_ associated with the + IP address and ISP. + + :type: str + + .. attribute:: organization The name of the organization associated with the IP address. @@ -599,10 +616,14 @@ class ISP(ASN): """ isp: Optional[str] + mobile_country_code: Optional[str] + mobile_network_code: Optional[str] organization: Optional[str] # pylint:disable=too-many-arguments def __init__(self, raw: Dict[str, Union[str, int]]) -> None: super().__init__(raw) self.isp = cast(Optional[str], raw.get("isp")) + self.mobile_country_code = cast(Optional[str], raw.get("mobile_country_code")) + self.mobile_network_code = cast(Optional[str], raw.get("mobile_network_code")) self.organization = cast(Optional[str], raw.get("organization")) diff --git a/geoip2/records.py b/geoip2/records.py index d3599bf..42fd373 100644 --- a/geoip2/records.py +++ b/geoip2/records.py @@ -713,7 +713,7 @@ class Traits(Record): .. attribute:: is_tor_exit_node - This is true if the IP address is a Tor exit node. This attribute is + This is true if the IP address is a Tor exit node. This attribute is only available from GeoIP2 Precision Insights. :type: bool @@ -726,6 +726,24 @@ class Traits(Record): :type: unicode + .. attribute: mobile_country_code + + The `mobile country code (MCC) + `_ associated with the + IP address and ISP. This attribute is available from the City and + Insights web services and the GeoIP2 Enterprise database. + + :type: str + + .. attribute: mobile_network_code + + The `mobile network code (MCC) + `_ associated with the + IP address and ISP. This attribute is available from the City and + Insights web services and the GeoIP2 Enterprise database. + + :type: str + .. attribute:: network The network associated with the record. In particular, this is the @@ -808,6 +826,8 @@ class Traits(Record): is_tor_exit_node: bool isp: Optional[str] ip_address: Optional[str] + mobile_country_code: Optional[str] + mobile_network_code: Optional[str] organization: Optional[str] static_ip_score: Optional[float] user_count: Optional[int] @@ -838,6 +858,8 @@ def __init__( static_ip_score: Optional[float] = None, user_count: Optional[int] = None, user_type: Optional[str] = None, + mobile_country_code: Optional[str] = None, + mobile_network_code: Optional[str] = None, **_, ) -> None: self.autonomous_system_number = autonomous_system_number @@ -854,6 +876,8 @@ def __init__( self.is_satellite_provider = is_satellite_provider self.is_tor_exit_node = is_tor_exit_node self.isp = isp + self.mobile_country_code = mobile_country_code + self.mobile_network_code = mobile_network_code self.organization = organization self.static_ip_score = static_ip_score self.user_type = user_type diff --git a/tests/database_test.py b/tests/database_test.py index 5f0956d..cf091df 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -199,27 +199,36 @@ def test_enterprise(self) -> None: record.traits.network, ipaddress.ip_network("74.209.16.0/20") ) - def test_isp(self) -> None: - reader = geoip2.database.Reader("tests/data/test-data/GeoIP2-ISP-Test.mmdb") + record = reader.enterprise("149.101.100.0") - ip_address = "1.128.0.0" - record = reader.isp(ip_address) - self.assertEqual(record, eval(repr(record)), "ISP repr can be eval'd") + self.assertEqual(record.traits.mobile_country_code, "310") + self.assertEqual(record.traits.mobile_network_code, "004") - self.assertEqual(record.autonomous_system_number, 1221) - self.assertEqual(record.autonomous_system_organization, "Telstra Pty Ltd") - self.assertEqual(record.isp, "Telstra Internet") - self.assertEqual(record.organization, "Telstra Internet") - self.assertEqual(record.ip_address, ip_address) - self.assertEqual(record.network, ipaddress.ip_network("1.128.0.0/11")) + def test_isp(self) -> None: + with geoip2.database.Reader( + "tests/data/test-data/GeoIP2-ISP-Test.mmdb" + ) as reader: + ip_address = "1.128.0.0" + record = reader.isp(ip_address) + self.assertEqual(record, eval(repr(record)), "ISP repr can be eval'd") + + self.assertEqual(record.autonomous_system_number, 1221) + self.assertEqual(record.autonomous_system_organization, "Telstra Pty Ltd") + self.assertEqual(record.isp, "Telstra Internet") + self.assertEqual(record.organization, "Telstra Internet") + self.assertEqual(record.ip_address, ip_address) + self.assertEqual(record.network, ipaddress.ip_network("1.128.0.0/11")) + + self.assertRegex( + str(record), + r"ISP\(\{.*Telstra.*\}\)", + "ISP str representation is reasonable", + ) - self.assertRegex( - str(record), - r"ISP\(\{.*Telstra.*\}\)", - "ISP str representation is reasonable", - ) + record = reader.isp("149.101.100.0") - reader.close() + self.assertEqual(record.mobile_country_code, "310") + self.assertEqual(record.mobile_network_code, "004") def test_context_manager(self) -> None: with geoip2.database.Reader( From 9f3e6af3117e3928cbd68a1bf06c042b87aff295 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 17 Nov 2021 09:35:43 -0800 Subject: [PATCH 3/6] Replace unicode type with str in docs "unicode" is a holdover from Python 2. --- geoip2/models.py | 22 +++++++++++----------- geoip2/records.py | 38 +++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/geoip2/models.py b/geoip2/models.py index 2e177cd..9642669 100644 --- a/geoip2/models.py +++ b/geoip2/models.py @@ -404,7 +404,7 @@ class AnonymousIP(SimpleModel): The IP address used in the lookup. - :type: unicode + :type: str .. attribute:: network @@ -448,13 +448,13 @@ class ASN(SimpleModel): The organization associated with the registered autonomous system number for the IP address. - :type: unicode + :type: str .. attribute:: ip_address The IP address used in the lookup. - :type: unicode + :type: str .. attribute:: network @@ -495,13 +495,13 @@ class ConnectionType(SimpleModel): Additional values may be added in the future. - :type: unicode + :type: str .. attribute:: ip_address The IP address used in the lookup. - :type: unicode + :type: str .. attribute:: network @@ -528,13 +528,13 @@ class Domain(SimpleModel): The domain associated with the IP address. - :type: unicode + :type: str .. attribute:: ip_address The IP address used in the lookup. - :type: unicode + :type: str .. attribute:: network @@ -569,13 +569,13 @@ class ISP(ASN): The organization associated with the registered autonomous system number for the IP address. - :type: unicode + :type: str .. attribute:: isp The name of the ISP associated with the IP address. - :type: unicode + :type: str .. attribute: mobile_country_code @@ -598,13 +598,13 @@ class ISP(ASN): The name of the organization associated with the IP address. - :type: unicode + :type: str .. attribute:: ip_address The IP address used in the lookup. - :type: unicode + :type: str .. attribute:: network diff --git a/geoip2/records.py b/geoip2/records.py index 42fd373..10d1148 100644 --- a/geoip2/records.py +++ b/geoip2/records.py @@ -76,7 +76,7 @@ class City(PlaceRecord): The name of the city based on the locales list passed to the constructor. - :type: unicode + :type: str .. attribute:: names @@ -117,7 +117,7 @@ class Continent(PlaceRecord): A two character continent code like "NA" (North America) or "OC" (Oceania). - :type: unicode + :type: str .. attribute:: geoname_id @@ -130,7 +130,7 @@ class Continent(PlaceRecord): Returns the name of the continent based on the locales list passed to the constructor. - :type: unicode + :type: str .. attribute:: names @@ -191,14 +191,14 @@ class Country(PlaceRecord): `_ alpha code for the country. - :type: unicode + :type: str .. attribute:: name The name of the country based on the locales list passed to the constructor. - :type: unicode + :type: str .. attribute:: names @@ -266,14 +266,14 @@ class RepresentedCountry(Country): The two-character `ISO 3166-1 `_ alpha code for the country. - :type: unicode + :type: str .. attribute:: name The name of the country based on the locales list passed to the constructor. - :type: unicode + :type: str .. attribute:: names @@ -289,7 +289,7 @@ class RepresentedCountry(Country): country. Currently we only return ``military`` but this could expand to include other types in the future. - :type: unicode + :type: str """ @@ -376,7 +376,7 @@ class Location(Record): Zone Database `_, e.g., "America/New_York". - :type: unicode + :type: str """ @@ -443,7 +443,7 @@ class Postal(Record): codes are not available for all countries. In some countries, this will only contain part of the postal code. - :type: unicode + :type: str .. attribute:: confidence @@ -496,14 +496,14 @@ class Subdivision(PlaceRecord): contain the subdivision portion of the `ISO 3166-2 code `_. - :type: unicode + :type: str .. attribute:: name The name of the subdivision based on the locales list passed to the constructor. - :type: unicode + :type: str .. attribute:: names @@ -598,7 +598,7 @@ class Traits(Record): the IP address. This attribute is only available from the City and Insights web service end points and the GeoIP2 Enterprise database. - :type: unicode + :type: str .. attribute:: connection_type @@ -613,7 +613,7 @@ class Traits(Record): This attribute is only available in the GeoIP2 Enterprise database. - :type: unicode + :type: str .. attribute:: domain @@ -623,7 +623,7 @@ class Traits(Record): from the City and Insights web service end points and the GeoIP2 Enterprise database. - :type: unicode + :type: str .. attribute:: ip_address @@ -633,7 +633,7 @@ class Traits(Record): running on. If the system is behind a NAT, this may differ from the IP address locally assigned to it. - :type: unicode + :type: str .. attribute:: is_anonymous @@ -724,7 +724,7 @@ class Traits(Record): only available from the City and Insights web service end points and the GeoIP2 Enterprise database. - :type: unicode + :type: str .. attribute: mobile_country_code @@ -758,7 +758,7 @@ class Traits(Record): attribute is only available from the City and Insights web service end points and the GeoIP2 Enterprise database. - :type: unicode + :type: str .. attribute:: static_ip_score @@ -807,7 +807,7 @@ class Traits(Record): This attribute is only available from the Insights end point and the GeoIP2 Enterprise database. - :type: unicode + :type: str """ From 2e8f56b3f2f6802a65721065f0019744c564d7f5 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 17 Nov 2021 09:41:17 -0800 Subject: [PATCH 4/6] Test on 3.10 and list it as supported --- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 2 +- setup.cfg | 5 +++-- setup.py | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a88db19..54b43f1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: "3.10" - name: Install dependencies run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2088bc..fc3d77f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: # We don't test on Windows currently as it appears mocket may not # work there. platform: [ubuntu-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] name: Python ${{ matrix.python-version }} on ${{ matrix.platform }} runs-on: ${{ matrix.platform }} diff --git a/setup.cfg b/setup.cfg index bc2fd08..599ad15 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,14 +12,15 @@ universal = 1 disable = duplicate-code [tox:tox] -envlist = py36, py37, py38, py39, mypy +envlist = py36, py37, py38, py39, py310, mypy [gh-actions] python = 3.6: py36 3.7: py37 3.8: py38 - 3.9: py39, mypy + 3.9: py39 + "3.10": py310, mypy [testenv] deps = diff --git a/setup.py b/setup.py index 7a89b6c..4f72a4a 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python", "Topic :: Internet :: Proxy Servers", "Topic :: Internet", From c255dad13bac1466b2ed3343c002a28dcf8bd283 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 17 Nov 2021 10:13:45 -0800 Subject: [PATCH 5/6] Add a note that the MCC/MNC data will not be available until Jan --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 44f1779..8058af0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ History ``mobile_network_code`` attributes were added to ``geoip2.model.ISP`` for the GeoIP2 ISP database and ``geoip2.record.Traits`` for the Enterprise database and the GeoIP2 City and Insights web services. + We expect this data to be available by late January, 2022. 4.4.0 (2021-09-24) ++++++++++++++++++ From 7b15d13426dcf8aaa886e888f7752ea610140811 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 17 Nov 2021 12:11:30 -0800 Subject: [PATCH 6/6] Fix typo and whitespace --- geoip2/models.py | 3 +-- geoip2/records.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/geoip2/models.py b/geoip2/models.py index 9642669..d16393d 100644 --- a/geoip2/models.py +++ b/geoip2/models.py @@ -587,13 +587,12 @@ class ISP(ASN): .. attribute: mobile_network_code - The `mobile network code (MCC) + The `mobile network code (MNC) `_ associated with the IP address and ISP. :type: str - .. attribute:: organization The name of the organization associated with the IP address. diff --git a/geoip2/records.py b/geoip2/records.py index 10d1148..bf8164b 100644 --- a/geoip2/records.py +++ b/geoip2/records.py @@ -737,7 +737,7 @@ class Traits(Record): .. attribute: mobile_network_code - The `mobile network code (MCC) + The `mobile network code (MNC) `_ associated with the IP address and ISP. This attribute is available from the City and Insights web services and the GeoIP2 Enterprise database.