[GeoIP] Use a helper script to decide when the upstream databases have changed (or at least their timestamps

[Philip Prindeville]

commit b338986e9f1caf7ca14103fce4c649b75342b0c8
Author: Philip A. Prindeville <philipp at redfish-solutions.com>
Date:   Wed Nov 6 13:53:23 2013 -0700

    Use a helper script to decide when the upstream databases have changed
    (or at least their timestamps have). We need to do this because there
    is no versioning on the unsupported IPv6 databases at this time.

 GeoIP.spec        |    7 ++++++
 geoipupdate6.cron |   39 ++++++++++++++++++++++++++++++------
 lastmod.pl        |   56 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 95 insertions(+), 7 deletions(-)
---
diff --git a/GeoIP.spec b/GeoIP.spec
index fe6b43a..9526f42 100644
--- a/GeoIP.spec
+++ b/GeoIP.spec
@@ -13,6 +13,7 @@ Source2:	fetch-geoipdata-city.pl
 Source3:	fetch-geoipdata.pl
 Source5:	geoipupdate.cron
 Source6:	geoipupdate6.cron
+Source7:	lastmod.pl
 # Data sources indexed at http://dev.maxmind.com/geoip/legacy/geolite
 Source10:	http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 Source11:	http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz
@@ -82,6 +83,7 @@ install -p -m 644 %{SOURCE2} fetch-geoipdata-city.pl
 install -p -m 644 %{SOURCE3} fetch-geoipdata.pl
 install -p -m 755 %{SOURCE5} geoipupdate.cron
 install -p -m 755 %{SOURCE6} geoipupdate6.cron
+install -p -m 755 %{SOURCE7} lastmod.pl
 
 # Data
 install -p -m 644 %{SOURCE10} data/GeoLiteCountry.dat.gz;	gunzip data/GeoLiteCountry.dat
@@ -140,6 +142,10 @@ do
 	install -p -m 644 data/$db %{buildroot}%{_datadir}/GeoIP/
 done
 
+%{__mkdir_p} %{buildroot}%{_libexecdir}
+
+install -p -m 755 lastmod.pl %{buildroot}%{_libexecdir}/
+
 # make the default GeoIP.dat a symlink to GeoLiteCountry.dat,
 # since it's actually an old snapshot of that database
 ln -sf GeoLiteCountry.dat %{buildroot}%{_datadir}/GeoIP/GeoIP.dat
@@ -196,6 +202,7 @@ rm -rf %{buildroot}
 
 %files update6
 %{_sysconfdir}/cron.weekly/geoipupdate6
+%{_libexecdir}/lastmod.pl
 
 %files devel
 # LGPLv2+
diff --git a/geoipupdate6.cron b/geoipupdate6.cron
index 4273968..c464c83 100644
--- a/geoipupdate6.cron
+++ b/geoipupdate6.cron
@@ -5,16 +5,41 @@
 # It would be much better if we could use geoipupdate for this as it only
 # downloads the data if it has changed and checks it after download
 
+PATH=/usr/libexec:$PATH
+
 DB_LOC=http://geolite.maxmind.com/download/geoip/database
 ASDB_LOC=http://download.maxmind.com/download/geoip/database/asnum
 
-cd /usr/share/GeoIP
+GEOIP_DIR=/usr/share/GeoIP
+
+DATABASES="$DB_LOC/GeoIPv6.dat.gz $DB_LOC/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz $ASDB_LOC/GeoIPASNumv6.dat.gz"
+
+cd $GEOIP_DIR
+
+status=0
+
+for URL in $DATABASES; do
+    RENEW=
+
+    BASENAME=$(basename $URL .gz)
+    ONLINE=$(lastmod.pl $URL)
 
-# IPv6 Country database
-wget --quiet $DB_LOC/GeoIPv6.dat.gz && mv -f GeoIPv6.dat GeoIPv6.dat.old && gunzip GeoIPv6.dat
+    if [ ! -f $BASENAME || ! -f .$BASENAME.timestamp ]; then
+	RENEW=1
+    else
+	TIMESTAMP=$(cat .$BASENAME.timestamp)
+	if [ $ONLINE -gt $TIMESTAMP ]; then
+	    RENEW=1
+	fi
+    fi
 
-# IPv6 City database
-wget --quiet $DB_LOC/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz && mv -f GeoLiteCityv6.dat GeoLiteCityv6.dat.old && gunzip GeoLiteCityv6.dat
+    if [ -n "$RENEW" ]; then
+	( wget -q -N $URL \
+	  && mv -f $BASENAME $BASENAME.old \
+	  && gunzip $BASENAME.gz \
+	  && echo $ONLINE > .$BASENAME.timestamp ) \
+	|| { status=1 ; echo "Failed to update $BASENAME" >&2 }
+    fi
+done
 
-# IPv6 AS Number database
-wget --quiet $ASDB_LOC/GeoIPASNumv6.dat.gz && mv -f GeoIPASNumv6.dat GeoIPASNumv6.dat.old && gunzip GeoIPASNumv6.dat
+exit $status
diff --git a/lastmod.pl b/lastmod.pl
new file mode 100755
index 0000000..735762a
--- /dev/null
+++ b/lastmod.pl
@@ -0,0 +1,56 @@
+#!/usr/bin/perl -w
+#===============================================================================
+#
+#         FILE: lastmod.pl
+#
+#        USAGE: lastmod.pl url
+#
+#  DESCRIPTION: Fetch an HTTP URL's HEAD and dump its Last-Modified field in
+#               local time format (e.g. seconds since the epoch).
+#
+#      OPTIONS: none
+# REQUIREMENTS: File::Basename, Net::HTTP, URI, HTTP::Headers
+#         BUGS: unknown
+#        NOTES: none
+#       AUTHOR: Philip Prindeville <philipp at fedoraproject.org>
+# ORGANIZATION: Fedora Project
+#      VERSION: 1.0
+#      CREATED: 08/06/2013 12:52:32 PM
+#     REVISION: 0
+#===============================================================================
+
+use strict;
+use warnings;
+
+use File::Basename;
+use Net::HTTP;
+use URI;
+use HTTP::Headers;
+
+if (@ARGV != 1) {
+    die "usage: " . basename($0) . " url\n";
+}
+
+my $uri = URI->new($ARGV[0]) || die "Couldn't parse URL\n";
+
+die "Not an HTTP URL\n" unless ($uri->scheme eq 'http');
+
+my $host = $uri->host() || die "No host in URL\n";
+
+my $s = Net::HTTP->new(Host => $host) || die "Can't construct Net::HTTP object\n";
+
+$s->write_request(HEAD => $uri->as_string()) || die "Can't send HEAD request\n";
+
+my ($code, $mess, %h) = $s->read_response_headers;
+
+if ($code ne '200') {
+    die "Response: " . $code . ": " . $mess . "\n";
+}
+
+my $h = HTTP::Headers->new(%h) || die "Couldn't parse headers\n";
+
+print $h->last_modified(), "\n";
+
+exit 0;
+
+# vim:ts=4