Launchpad itself

Merge lp:~cjwatson/launchpad/geoip-country into lp:launchpad

  1. geoip-country
  2. Merge into devel
Proposed by Colin Watson
Status: Merged
Merged at revision: 19029
Proposed branch: lp:~cjwatson/launchpad/geoip-country
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/geoip-ip-address-handling
Diff against target: 159 lines (+16/-35)
8 files modified
configs/development/launchpad-lazr.conf (+0/-1)
configs/testrunner/launchpad-lazr.conf (+0/-1)
lib/lp/services/apachelogparser/base.py (+2/-5)
lib/lp/services/apachelogparser/tests/apache-log-files/launchpadlibrarian.net.access-log (+1/-1)
lib/lp/services/config/schema-lazr.conf (+1/-1)
lib/lp/services/geoip/doc/geoip.txt (+6/-19)
lib/lp/services/geoip/interfaces.py (+2/-2)
lib/lp/services/geoip/model.py (+4/-5)
To merge this branch: bzr merge lp:~cjwatson/launchpad/geoip-country
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+371044@code.launchpad.net

Commit message

Switch to the GeoIP country database.

Description of the change

The only piece of information we use from this now is the country code, so there's no point reading the much larger city database.

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'configs/development/launchpad-lazr.conf'
2--- configs/development/launchpad-lazr.conf 2019-06-18 17:30:52 +0000
3+++ configs/development/launchpad-lazr.conf 2019-08-07 15:43:20 +0000
4@@ -109,7 +109,6 @@
5 summary_list_size: 5
6 max_bug_feed_cache_minutes: 30
7 bzr_imports_root_url: file:///tmp/bazaar-branches
8-geoip_database: /usr/share/GeoIP/GeoLiteCity.dat
9 feature_flags_endpoint: http://xmlrpc-private.launchpad.test:8087/featureflags/
10
11 [launchpad_session]
12
13=== modified file 'configs/testrunner/launchpad-lazr.conf'
14--- configs/testrunner/launchpad-lazr.conf 2019-05-22 14:57:45 +0000
15+++ configs/testrunner/launchpad-lazr.conf 2019-08-07 15:43:20 +0000
16@@ -103,7 +103,6 @@
17 [launchpad]
18 basic_auth_password: test
19 max_attachment_size: 1024
20-geoip_database: /usr/share/GeoIP/GeoLiteCity.dat
21 logparser_max_parsed_lines: 100000
22 homepage_recent_posts_feed: http://launchpad.test:8093/blog-feed
23 openid_canonical_root: http://testopenid.test/
24
25=== modified file 'lib/lp/services/apachelogparser/base.py'
26--- lib/lp/services/apachelogparser/base.py 2019-07-25 15:00:18 +0000
27+++ lib/lp/services/apachelogparser/base.py 2019-08-07 15:43:20 +0000
28@@ -1,4 +1,4 @@
29-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
30+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
31 # GNU Affero General Public License version 3 (see the file LICENSE).
32
33 from datetime import datetime
34@@ -148,10 +148,7 @@
35 file_downloads[day] = {}
36 daily_downloads = file_downloads[day]
37
38- country_code = None
39- geoip_record = geoip.getRecordByAddress(host)
40- if geoip_record is not None:
41- country_code = geoip_record['country_code']
42+ country_code = geoip.getCountryCodeByAddr(host)
43 if country_code not in daily_downloads:
44 daily_downloads[country_code] = 0
45 daily_downloads[country_code] += 1
46
47=== modified file 'lib/lp/services/apachelogparser/tests/apache-log-files/launchpadlibrarian.net.access-log'
48--- lib/lp/services/apachelogparser/tests/apache-log-files/launchpadlibrarian.net.access-log 2010-06-17 15:23:40 +0000
49+++ lib/lp/services/apachelogparser/tests/apache-log-files/launchpadlibrarian.net.access-log 2019-08-07 15:43:20 +0000
50@@ -2,5 +2,5 @@
51 121.44.28.210 - - [13/Jun/2008:14:55:06 +0100] "GET /9096290/me-tv-icon-14x14.png HTTP/1.1" 200 730 "https://launchpad.net/me-tv" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008060900 Firefox/3.0"
52 121.44.28.210 - - [13/Jun/2008:14:55:13 +0100] "GET /12060796/me-tv-icon-64x64.png HTTP/1.1" 200 6378 "https://launchpad.net/me-tv" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008060900 Firefox/3.0"
53 157.92.18.21 - - [13/Jun/2008:14:55:15 +0100] "GET /8196569/mediumubuntulogo.png HTTP/1.1" 200 3420 "https://bugs.launchpad.net/ubuntu/+source/acpi-support/+bug/59695/comments/14" "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9) Gecko/2008052623 Iceweasel/3.0 (Debian-3.0~rc1-1)"
54-210.79.178.243 - - [13/Jun/2008:14:55:21 +0100] "GET /8196569/mediumubuntulogo.png HTTP/1.1" 200 3420 "https://launchpad.net/ubuntu/hardy/hppa/xserver-xorg-driver-savage" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008060309 Firefox/3.0"
55+202.214.194.147 - - [13/Jun/2008:14:55:21 +0100] "GET /8196569/mediumubuntulogo.png HTTP/1.1" 200 3420 "https://launchpad.net/ubuntu/hardy/hppa/xserver-xorg-driver-savage" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008060309 Firefox/3.0"
56 69.233.136.42 - - [13/Jun/2008:14:55:22 +0100] "GET /15018215/ul_logo_64x64.png HTTP/1.1" 200 2261 "https://launchpad.net/~ubuntulite/+archive" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9b5) Gecko/2008041514 Firefox/3.0b5"
57
58=== modified file 'lib/lp/services/config/schema-lazr.conf'
59--- lib/lp/services/config/schema-lazr.conf 2019-07-11 13:27:09 +0000
60+++ lib/lp/services/config/schema-lazr.conf 2019-08-07 15:43:20 +0000
61@@ -1112,7 +1112,7 @@
62 code_homepage_product_cloud_size: 120
63
64 # The database used by our GeoIP library.
65-geoip_database: /usr/share/GeoIP/GeoIPCity.dat
66+geoip_database: /usr/share/GeoIP/GeoIP.dat
67
68 # The maximum number of lines that should be parsed by the launchpad
69 # log parser. The default value of None means there is no maximum.
70
71=== modified file 'lib/lp/services/geoip/doc/geoip.txt'
72--- lib/lp/services/geoip/doc/geoip.txt 2019-08-07 15:42:37 +0000
73+++ lib/lp/services/geoip/doc/geoip.txt 2019-08-07 15:43:20 +0000
74@@ -37,32 +37,19 @@
75 >>> print(geoip.getCountryByAddr('fc00::1'))
76 None
77
78-IGeoIP also provides a getRecordByAddress() method, which returns a dict
79-containing a bunch more information about that IP's location.
80+IGeoIP also provides a getCountryCodeByAddr() method, which returns just the
81+country code without looking it up in the database.
82
83- >>> record = geoip.getRecordByAddress('201.13.165.145')
84- >>> for key, value in sorted(record.items()):
85- ... print "%s: %s" % (key, value)
86- area_code: ...
87- city: ...
88- country_code: BR
89- country_code3: BRA
90- country_name: Brazil
91- dma_code: ...
92- latitude: ...
93- longitude: ...
94- postal_code: ...
95- region: ...
96- region_name: ...
97- time_zone: ...
98+ >>> print(geoip.getCountryCodeByAddr('201.13.165.145'))
99+ BR
100
101 And again we'll return None if the address is private.
102
103- >>> print(geoip.getRecordByAddress('127.0.0.1'))
104+ >>> print(geoip.getCountryCodeByAddr('127.0.0.1'))
105 None
106
107 If it can't find a GeoIP record for the given IP address, it will return
108 None.
109
110- >>> print geoip.getRecordByAddress('255.255.255.255')
111+ >>> print(geoip.getCountryCodeByAddr('255.255.255.255'))
112 None
113
114=== modified file 'lib/lp/services/geoip/interfaces.py'
115--- lib/lp/services/geoip/interfaces.py 2019-08-07 15:42:37 +0000
116+++ lib/lp/services/geoip/interfaces.py 2019-08-07 15:43:20 +0000
117@@ -14,8 +14,8 @@
118 class IGeoIP(Interface):
119 """The GeoIP utility, which represents the GeoIP database."""
120
121- def getRecordByAddress(ip_address):
122- """Return the GeoIP record for the given IP address, or None."""
123+ def getCountryCodeByAddr(ip_address):
124+ """Return the country code for the given IP address, or None."""
125
126 def getCountryByAddr(ip_address):
127 """Find and return an ICountry based on the given IP address.
128
129=== modified file 'lib/lp/services/geoip/model.py'
130--- lib/lp/services/geoip/model.py 2019-08-07 15:42:37 +0000
131+++ lib/lp/services/geoip/model.py 2019-08-07 15:43:20 +0000
132@@ -41,12 +41,12 @@
133 return libGeoIP.open(
134 config.launchpad.geoip_database, libGeoIP.GEOIP_MEMORY_CACHE)
135
136- def getRecordByAddress(self, ip_address):
137+ def getCountryCodeByAddr(self, ip_address):
138 """See `IGeoIP`."""
139 if not ipaddress_is_global(ip_address):
140 return None
141 try:
142- return self._gi.record_by_addr(ip_address)
143+ return self._gi.country_code_by_addr(ip_address)
144 except SystemError:
145 # libGeoIP may raise a SystemError if it doesn't find a record for
146 # some IP addresses (e.g. 255.255.255.255), so we need to catch
147@@ -57,10 +57,9 @@
148 """See `IGeoIP`."""
149 if not ipaddress_is_global(ip_address):
150 return None
151- geoip_record = self.getRecordByAddress(ip_address)
152- if geoip_record is None:
153+ countrycode = self.getCountryCodeByAddr(ip_address)
154+ if countrycode is None:
155 return None
156- countrycode = geoip_record['country_code']
157
158 countryset = getUtility(ICountrySet)
159 try: