4:19 p.m.
On Thu, 30 Jul 2015 16:55:18 +0200
Adrian Reber <adrian(a)lisas.de> wrote:
This is a try to fix the s3sync on mm-backend01. I have tested the
script and it successfully syncs the content to S3. I have, however,
not tested if the ansible rules work. Please have a look at following
patch and if the ansible rules are correct.
They look ok to me from a quick eyeball.
We can of course run a 'ansible-playbook --syntax-check' before
pushing.
In addition to this patch it is necessary to install the new
mirrormanager2 release (0.4.2) in prod. It has now changes except an
additional subpackage mirrormanager2-client which provides
report_mirror which is run after the sync to S3.
+1 here.
kevin
--
Date: Thu, 30 Jul 2015 13:47:48 +0000
Subject: [PATCH] Fix s3sync and start it via cron.
This changes the s3sync setup at multiple places:
* The s3sync script was using wrong paths which are now corrected
* After the sync report_mirror was started which is now installed
* The necessary report_mirror files are installed
* The s3sync script is now running from cron instead as a service
* Before touching any files fedmsg is queried to see if new files
are available for syncing
* logrotate has been adapted to the new setup
---
roles/s3-mirror/files/s3-mirror.init | 77
-------------------- roles/s3-mirror/files/s3-mirror.logrotate
| 3 - roles/s3-mirror/files/s3sync | 100
+++++++++++++-------------
roles/s3-mirror/tasks/main.yml | 19 ++++-
roles/s3-mirror/templates/report_mirror.conf | 60 +++++++++++++++ 5
files changed, 126 insertions(+), 133 deletions(-) delete mode 100644
roles/s3-mirror/files/s3-mirror.init create mode 100644
roles/s3-mirror/templates/report_mirror.conf
diff --git a/roles/s3-mirror/files/s3-mirror.init
b/roles/s3-mirror/files/s3-mirror.init deleted file mode 100644
index 59dbb39..0000000
--- a/roles/s3-mirror/files/s3-mirror.init
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/sh
-#
-# s3-mirror - sync content to S3
-#
-# chkconfig: - 99 99
-# description: sync content to S3
-
-# http://fedoraproject.org/wiki/FCNewInit/Initscripts
-### BEGIN INIT INFO
-# Provides:
-# Required-Start: $network $named $remote_fs
-# Required-Stop:
-# Should-Start:
-# Should-Stop:
-# Default-Start:
-# Default-Stop:
-# Short-Description:
-# Description:
-### END INIT INFO
-
-# Source function library.
-. /etc/rc.d/init.d/functions
-
-exec="/usr/local/bin/s3sync"
-prog=${exec##*/}
-
-lockfile=/var/lock/subsys/$prog
-pidfile=/var/run/s3-mirror/pid
-
-start() {
- echo -n $"Starting $prog: "
- mkdir -p /var/run/s3-mirror
- chown -R s3-mirror:s3-mirror /var/run/s3-mirror
- daemon --user s3-mirror --pidfile $pidfile $exec &
- retval=$?
- echo
- [ $retval -eq 0 ] && touch $lockfile
- return $retval
-}
-
-stop() {
- echo -n $"Stopping $prog: "
- killproc -p $pidfile
- retval=$?
- echo
- [ $retval -eq 0 ] && rm -f $lockfile
- return $retval
-}
-
-restart() {
- stop
- start
-}
-
-case "$1" in
- start|stop|restart)
- $1
- ;;
- force-reload)
- restart
- ;;
- status)
- status $prog
- ;;
- try-restart|condrestart)
- if status $prog >/dev/null ; then
- restart
- fi
- ;;
- reload)
- action $"Service ${0##*/} does not support the reload
action: " /bin/false
- exit 3
- ;;
- *)
- echo $"Usage: $0
{start|stop|status|restart|try-restart|force-reload}"
- exit 2
-esac
diff --git a/roles/s3-mirror/files/s3-mirror.logrotate
b/roles/s3-mirror/files/s3-mirror.logrotate index 12e66aa..7549028
100644 --- a/roles/s3-mirror/files/s3-mirror.logrotate
+++ b/roles/s3-mirror/files/s3-mirror.logrotate
@@ -9,7 +9,4 @@
compressext .bz2
dateext
copytruncate
- postrotate
- /bin/kill -HUP `cat /var/run/s3-mirror/pid 2>/dev/null`
2>/dev/null || true
- endscript
}
diff --git a/roles/s3-mirror/files/s3sync
b/roles/s3-mirror/files/s3sync index 6d0c552..f8d0413 100755
--- a/roles/s3-mirror/files/s3sync
+++ b/roles/s3-mirror/files/s3sync
@@ -1,16 +1,16 @@
#!/bin/bash
-pidfile=/var/run/s3-mirror/pid
logfile="/var/log/s3-mirror/s3sync.log"
+tree="/srv/pub/"
+s3cmd=/usr/bin/s3cmd
+curdate=`date +%s`
-trap sighup_handler HUP
-trap "rm -f ${pidfile}" QUIT EXIT INT TERM
-
-function sighup_handler()
-{
- exec 1>>${logfile} 2>&1
-}
+if [ ! -d ${tree} ]; then
+ echo "Specified root of the directory tree (${tree}) does not
exist. Exiting." | tee ${logfile}
+ exit 1
+fi
+exec 1>>${logfile} 2>&1
function newer()
{
@@ -20,16 +20,6 @@ function newer()
return 1
}
-function repeat()
-{
- while :; do
- $1
- /bin/sleep $((${RANDOM} % 300))
- done
-}
-
-s3cmd=/usr/bin/s3cmd
-
S3CMD_ARGS="sync \
--verbose \
--preserve \
@@ -44,41 +34,51 @@ S3CMD_ARGS="sync \
content="epel"
targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2
s3-mirror-eu-west-1 s3-mirror-ap-northeast-1" +report=0
-function parallel_sync_full()
-{
- report=0
- for c in ${content}; do
- if
$(newer /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ;
then
- echo
"=============================================================="
- echo -n "Starting at "
- date
- time $s3cmd $S3CMD_ARGS \
- --cache-file /var/lib/s3-mirror/${c}.cache \
- --delete-removed \
- --delete-after \
- --add-destination
s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
- --add-destination
s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
- --add-destination
s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
- --add-destination
s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
- /pub/${c}/
s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
- report=1
- cp
-a /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
- fi
- done
+if [ -e /var/run/s3-mirror/lastrun ]; then
+ . /var/run/s3-mirror/lastrun
+else
+ # 48 hours -> 172800 seconds
+ let lastrun=curdate-172800
+fi
+
+let delta=curdate-lastrun
- if [ ${report} -ne 0 ] ; then
- for target in ${targets} ; do
- report_mirror
-c /etc/mirrormanager-client/report_mirror_${target}.conf
--exclude-from /usr/local/etc/s3-mirror-excludes.txt
- done
- echo -n "Ending at "
+# hardcoded to epel only '-e'
+/usr/local/bin/last-sync -d ${delta} -e
+
+if [ "$?" -ne "0" ]; then
+ # no changes since the last sync
+ # abort
+ exit 0
+fi
+
+for c in ${content}; do
+ if $(newer
${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
+ echo
"=============================================================="
+ echo -n "Starting at "
date
+ time $s3cmd $S3CMD_ARGS \
+ --cache-file /var/lib/s3-mirror/${c}.cache \
+ --delete-removed \
+ --delete-after \
+ --add-destination
s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
+ --add-destination
s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
+ --add-destination
s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
+ --add-destination
s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
+ ${tree}/${c}/
s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
+ report=1
+ cp -a
${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
+ echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun
fi
-}
+done
-# send stdout/stderr to logfile
-sighup_handler
-mkdir -p $(dirname ${pidfile})
-echo "$$" > ${pidfile}
-repeat parallel_sync_full
+if [ ${report} -ne 0 ] ; then
+ for target in ${targets} ; do
+ report_mirror
-c /etc/mirrormanager-client/report_mirror_${target}.conf
--exclude-from /usr/local/etc/s3-mirror-excludes.txt
+ done
+ echo -n "Ending at "
+ date
+fi
diff --git a/roles/s3-mirror/tasks/main.yml
b/roles/s3-mirror/tasks/main.yml index 7900457..3da4b8d 100644
--- a/roles/s3-mirror/tasks/main.yml
+++ b/roles/s3-mirror/tasks/main.yml
@@ -11,6 +11,16 @@
yum: pkg={{ item }} state=installed
with_items:
- s3cmd
+ - mirrormanager2-client
+
+- name: Create report_mirror config files
+ template: src=report_mirror.conf
dest=/etc/mirrormanager-client/report_mirror_{{ item }}
owner=s3-mirror group=s3-mirror mode=0400
+ with_items:
+ - s3-mirror-us-east-1
+ - s3-mirror-us-west-1
+ - s3-mirror-us-west-2
+ - s3-mirror-eu-west-1
+ - s3-mirror-ap-northeast-1
- name: Create needed directories
file: path={{item}} owner=s3-mirror group=s3-mirror
state=directory mode=0755 @@ -33,12 +43,15 @@
- name: s3-mirror logrotate
copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror
owner=s3-mirror group=s3-mirror mode=0644
-- name: s3-mirror init.d
- copy: src=s3-mirror.init dest=/etc/init.d/s3-mirror
owner=s3-mirror group=s3-mirror mode=0755 -
- name: s3sync-logs
copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs
owner=s3-mirror group=s3-mirror mode=0755
+- name: s3sync cron
+ cron: name="s3sync" minute="3,18,33,48"
user="s3-mirror"
+ job="/usr/local/bin/lock-wrapper
s3sync /usr/local/bin/s3sync"
+ cron_file=s3sync
+ when: env != 'staging'
+
- name: s3sync-logs cron
cron: name="s3sync-logs" hour="0" user="root"
job='/usr/local/bin/lock-wrapper s3sync-logs "/bin/sleep
$((${RANDOM} \% 300)); /usr/local/bin/s3sync-logs
>> /var/log/s3-mirror-logs/s3sync-logs.log 2>&1"
>> | /usr/local/bin/nag-once s3mirror-logs 12h 2>&1' diff --git
>> a/roles/s3-mirror/templates/report_mirror.conf
>> b/roles/s3-mirror/templates/report_mirror.conf new file mode 100644
index 0000000..e51e1d9
--- /dev/null
+++ b/roles/s3-mirror/templates/report_mirror.conf
@@ -0,0 +1,60 @@
+[global]
+# if enabled=0, no data is sent to the database
+enabled=1
+# server= is the URL to the MirrorManager XML-RPC interface
+server=https://admin.fedoraproject.org/mirrormanager/xmlrpc
+
+
+[site]
+# if enabled=0, no data about this site is sent to the database
+enabled=1
+# Name and Password fields need to match the Site name and password
+# fields you entered for your Site in the MirrorManager database at
+# https://admin.fedoraproject.org/mirrormanager
+name={{ s3_mirror_report_mirror_site }}
+password={{ s3_mirror_report_mirror_pw }}
+
+[host]
+# if enabled=0, no data about this host is sent to the database
+enabled=1
+# Name field need to match the Host name field you entered for your
+# Host in the MirrorManager database at
+# https://admin.fedoraproject.org/mirrormanager
+name={{ item }}.fedoraproject.org
+# if user_active=0, no data about this category is given to the
public +# This can be used to toggle between serving and not serving
data, +# such enabled during the nighttime (when you have more idle
bandwidth +# available) and disabled during the daytime.
+# By not specifying user_active, the database will not be updated.
+# user_active=1
+
+[stats]
+# Stats are only sent when run with the -s option
+# and when this section is enabled.
+# This feature is not presently implemented
+enabled=0
+apache=/var/log/httpd/access_log
+vsftpd=/var/log/vsftpd.log
+# remember to enable log file and transfer logging in rsyncd.conf
+rsyncd=/var/log/rsyncd.log
+
+
+# Content Categories
+# These sections match the Categories for content tracked by
MirrorManager. +#
+# enabled=1 means information about this category will be sent to
the database. +# enabled=0, no data about this host is sent to the
database. If the +# database already has information for you for
this Category, it will +# remain unchanged. This can be used to
update the database after you +# have manually synced some
infrequently-updated content, such as +# historical releases.
+#
+# path= is the path on your local disk to the top-level directory
for this Category +
+[Fedora Linux]
+enabled=0
+path=/srv/pub/fedora/linux
+
+[Fedora EPEL]
+enabled=1
+path=/srv/pub/epel