pghmcfc pushed to GeoIP (el5). "Use a helper script to decide when the upstream databases have changed (..more)"

[notifications at fedoraproject.org]

>From b338986e9f1caf7ca14103fce4c649b75342b0c8 Mon Sep 17 00:00:00 2001
From: "Philip A. Prindeville" <philipp at redfish-solutions.com>
Date: Wed, 6 Nov 2013 13:53:23 -0700
Subject: Use a helper script to decide when the upstream databases have
 changed (or at least their timestamps have). We need to do this because there
 is no versioning on the unsupported IPv6 databases at this time.


diff --git a/GeoIP.spec b/GeoIP.spec
index fe6b43a..9526f42 100644
--- a/GeoIP.spec
+++ b/GeoIP.spec
@@ -13,6 +13,7 @@ Source2:	fetch-geoipdata-city.pl
 Source3:	fetch-geoipdata.pl
 Source5:	geoipupdate.cron
 Source6:	geoipupdate6.cron
+Source7:	lastmod.pl
 # Data sources indexed at http://dev.maxmind.com/geoip/legacy/geolite
 Source10:	http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 Source11:	http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz
@@ -82,6 +83,7 @@ install -p -m 644 %{SOURCE2} fetch-geoipdata-city.pl
 install -p -m 644 %{SOURCE3} fetch-geoipdata.pl
 install -p -m 755 %{SOURCE5} geoipupdate.cron
 install -p -m 755 %{SOURCE6} geoipupdate6.cron
+install -p -m 755 %{SOURCE7} lastmod.pl
 
 # Data
 install -p -m 644 %{SOURCE10} data/GeoLiteCountry.dat.gz;	gunzip data/GeoLiteCountry.dat
@@ -140,6 +142,10 @@ do
 	install -p -m 644 data/$db %{buildroot}%{_datadir}/GeoIP/
 done
 
+%{__mkdir_p} %{buildroot}%{_libexecdir}
+
+install -p -m 755 lastmod.pl %{buildroot}%{_libexecdir}/
+
 # make the default GeoIP.dat a symlink to GeoLiteCountry.dat,
 # since it's actually an old snapshot of that database
 ln -sf GeoLiteCountry.dat %{buildroot}%{_datadir}/GeoIP/GeoIP.dat
@@ -196,6 +202,7 @@ rm -rf %{buildroot}
 
 %files update6
 %{_sysconfdir}/cron.weekly/geoipupdate6
+%{_libexecdir}/lastmod.pl
 
 %files devel
 # LGPLv2+
diff --git a/geoipupdate6.cron b/geoipupdate6.cron
index 4273968..c464c83 100644
--- a/geoipupdate6.cron
+++ b/geoipupdate6.cron
@@ -5,16 +5,41 @@
 # It would be much better if we could use geoipupdate for this as it only
 # downloads the data if it has changed and checks it after download
 
+PATH=/usr/libexec:$PATH
+
 DB_LOC=http://geolite.maxmind.com/download/geoip/database
 ASDB_LOC=http://download.maxmind.com/download/geoip/database/asnum
 
-cd /usr/share/GeoIP
+GEOIP_DIR=/usr/share/GeoIP
+
+DATABASES="$DB_LOC/GeoIPv6.dat.gz $DB_LOC/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz $ASDB_LOC/GeoIPASNumv6.dat.gz"
+
+cd $GEOIP_DIR
+
+status=0
+
+for URL in $DATABASES; do
+    RENEW=
+
+    BASENAME=$(basename $URL .gz)
+    ONLINE=$(lastmod.pl $URL)
 
-# IPv6 Country database
-wget --quiet $DB_LOC/GeoIPv6.dat.gz && mv -f GeoIPv6.dat GeoIPv6.dat.old && gunzip GeoIPv6.dat
+    if [ ! -f $BASENAME || ! -f .$BASENAME.timestamp ]; then
+	RENEW=1
+    else
+	TIMESTAMP=$(cat .$BASENAME.timestamp)
+	if [ $ONLINE -gt $TIMESTAMP ]; then
+	    RENEW=1
+	fi
+    fi
 
-# IPv6 City database
-wget --quiet $DB_LOC/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz && mv -f GeoLiteCityv6.dat GeoLiteCityv6.dat.old && gunzip GeoLiteCityv6.dat
+    if [ -n "$RENEW" ]; then
+	( wget -q -N $URL \
+	  && mv -f $BASENAME $BASENAME.old \
+	  && gunzip $BASENAME.gz \
+	  && echo $ONLINE > .$BASENAME.timestamp ) \
+	|| { status=1 ; echo "Failed to update $BASENAME" >&2 }
+    fi
+done
 
-# IPv6 AS Number database
-wget --quiet $ASDB_LOC/GeoIPASNumv6.dat.gz && mv -f GeoIPASNumv6.dat GeoIPASNumv6.dat.old && gunzip GeoIPASNumv6.dat
+exit $status
diff --git a/lastmod.pl b/lastmod.pl
new file mode 100755
index 0000000..735762a
--- /dev/null
+++ b/lastmod.pl
@@ -0,0 +1,56 @@
+#!/usr/bin/perl -w
+#===============================================================================
+#
+#         FILE: lastmod.pl
+#
+#        USAGE: lastmod.pl url
+#
+#  DESCRIPTION: Fetch an HTTP URL's HEAD and dump its Last-Modified field in
+#               local time format (e.g. seconds since the epoch).
+#
+#      OPTIONS: none
+# REQUIREMENTS: File::Basename, Net::HTTP, URI, HTTP::Headers
+#         BUGS: unknown
+#        NOTES: none
+#       AUTHOR: Philip Prindeville <philipp at fedoraproject.org>
+# ORGANIZATION: Fedora Project
+#      VERSION: 1.0
+#      CREATED: 08/06/2013 12:52:32 PM
+#     REVISION: 0
+#===============================================================================
+
+use strict;
+use warnings;
+
+use File::Basename;
+use Net::HTTP;
+use URI;
+use HTTP::Headers;
+
+if (@ARGV != 1) {
+    die "usage: " . basename($0) . " url\n";
+}
+
+my $uri = URI->new($ARGV[0]) || die "Couldn't parse URL\n";
+
+die "Not an HTTP URL\n" unless ($uri->scheme eq 'http');
+
+my $host = $uri->host() || die "No host in URL\n";
+
+my $s = Net::HTTP->new(Host => $host) || die "Can't construct Net::HTTP object\n";
+
+$s->write_request(HEAD => $uri->as_string()) || die "Can't send HEAD request\n";
+
+my ($code, $mess, %h) = $s->read_response_headers;
+
+if ($code ne '200') {
+    die "Response: " . $code . ": " . $mess . "\n";
+}
+
+my $h = HTTP::Headers->new(%h) || die "Couldn't parse headers\n";
+
+print $h->last_modified(), "\n";
+
+exit 0;
+
+# vim:ts=4
-- 
cgit v0.10.2


	http://pkgs.fedoraproject.org/cgit/GeoIP.git/commit/?h=el5&id=b338986e9f1caf7ca14103fce4c649b75342b0c8