Freeze break request: s3sync
Adrian Reber
adrian at lisas.de
Thu Jul 30 14:55:18 UTC 2015
This is a try to fix the s3sync on mm-backend01. I have tested the
script and it successfully syncs the content to S3. I have, however, not
tested if the ansible rules work. Please have a look at following patch
and if the ansible rules are correct.
In addition to this patch it is necessary to install the new
mirrormanager2 release (0.4.2) in prod. It has now changes except an
additional subpackage mirrormanager2-client which provides report_mirror
which is run after the sync to S3.
Date: Thu, 30 Jul 2015 13:47:48 +0000
Subject: [PATCH] Fix s3sync and start it via cron.
This changes the s3sync setup at multiple places:
* The s3sync script was using wrong paths which are now corrected
* After the sync report_mirror was started which is now installed
* The necessary report_mirror files are installed
* The s3sync script is now running from cron instead as a service
* Before touching any files fedmsg is queried to see if new files
are available for syncing
* logrotate has been adapted to the new setup
---
roles/s3-mirror/files/s3-mirror.init | 77 --------------------
roles/s3-mirror/files/s3-mirror.logrotate | 3 -
roles/s3-mirror/files/s3sync | 100 +++++++++++++-------------
roles/s3-mirror/tasks/main.yml | 19 ++++-
roles/s3-mirror/templates/report_mirror.conf | 60 +++++++++++++++
5 files changed, 126 insertions(+), 133 deletions(-)
delete mode 100644 roles/s3-mirror/files/s3-mirror.init
create mode 100644 roles/s3-mirror/templates/report_mirror.conf
diff --git a/roles/s3-mirror/files/s3-mirror.init b/roles/s3-mirror/files/s3-mirror.init
deleted file mode 100644
index 59dbb39..0000000
--- a/roles/s3-mirror/files/s3-mirror.init
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/sh
-#
-# s3-mirror - sync content to S3
-#
-# chkconfig: - 99 99
-# description: sync content to S3
-
-# http://fedoraproject.org/wiki/FCNewInit/Initscripts
-### BEGIN INIT INFO
-# Provides:
-# Required-Start: $network $named $remote_fs
-# Required-Stop:
-# Should-Start:
-# Should-Stop:
-# Default-Start:
-# Default-Stop:
-# Short-Description:
-# Description:
-### END INIT INFO
-
-# Source function library.
-. /etc/rc.d/init.d/functions
-
-exec="/usr/local/bin/s3sync"
-prog=${exec##*/}
-
-lockfile=/var/lock/subsys/$prog
-pidfile=/var/run/s3-mirror/pid
-
-start() {
- echo -n $"Starting $prog: "
- mkdir -p /var/run/s3-mirror
- chown -R s3-mirror:s3-mirror /var/run/s3-mirror
- daemon --user s3-mirror --pidfile $pidfile $exec &
- retval=$?
- echo
- [ $retval -eq 0 ] && touch $lockfile
- return $retval
-}
-
-stop() {
- echo -n $"Stopping $prog: "
- killproc -p $pidfile
- retval=$?
- echo
- [ $retval -eq 0 ] && rm -f $lockfile
- return $retval
-}
-
-restart() {
- stop
- start
-}
-
-case "$1" in
- start|stop|restart)
- $1
- ;;
- force-reload)
- restart
- ;;
- status)
- status $prog
- ;;
- try-restart|condrestart)
- if status $prog >/dev/null ; then
- restart
- fi
- ;;
- reload)
- action $"Service ${0##*/} does not support the reload action: " /bin/false
- exit 3
- ;;
- *)
- echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}"
- exit 2
-esac
diff --git a/roles/s3-mirror/files/s3-mirror.logrotate b/roles/s3-mirror/files/s3-mirror.logrotate
index 12e66aa..7549028 100644
--- a/roles/s3-mirror/files/s3-mirror.logrotate
+++ b/roles/s3-mirror/files/s3-mirror.logrotate
@@ -9,7 +9,4 @@
compressext .bz2
dateext
copytruncate
- postrotate
- /bin/kill -HUP `cat /var/run/s3-mirror/pid 2>/dev/null` 2>/dev/null || true
- endscript
}
diff --git a/roles/s3-mirror/files/s3sync b/roles/s3-mirror/files/s3sync
index 6d0c552..f8d0413 100755
--- a/roles/s3-mirror/files/s3sync
+++ b/roles/s3-mirror/files/s3sync
@@ -1,16 +1,16 @@
#!/bin/bash
-pidfile=/var/run/s3-mirror/pid
logfile="/var/log/s3-mirror/s3sync.log"
+tree="/srv/pub/"
+s3cmd=/usr/bin/s3cmd
+curdate=`date +%s`
-trap sighup_handler HUP
-trap "rm -f ${pidfile}" QUIT EXIT INT TERM
-
-function sighup_handler()
-{
- exec 1>>${logfile} 2>&1
-}
+if [ ! -d ${tree} ]; then
+ echo "Specified root of the directory tree (${tree}) does not exist. Exiting." | tee ${logfile}
+ exit 1
+fi
+exec 1>>${logfile} 2>&1
function newer()
{
@@ -20,16 +20,6 @@ function newer()
return 1
}
-function repeat()
-{
- while :; do
- $1
- /bin/sleep $((${RANDOM} % 300))
- done
-}
-
-s3cmd=/usr/bin/s3cmd
-
S3CMD_ARGS="sync \
--verbose \
--preserve \
@@ -44,41 +34,51 @@ S3CMD_ARGS="sync \
content="epel"
targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
+report=0
-function parallel_sync_full()
-{
- report=0
- for c in ${content}; do
- if $(newer /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
- echo "=============================================================="
- echo -n "Starting at "
- date
- time $s3cmd $S3CMD_ARGS \
- --cache-file /var/lib/s3-mirror/${c}.cache \
- --delete-removed \
- --delete-after \
- --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
- --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
- --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
- --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
- /pub/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
- report=1
- cp -a /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
- fi
- done
+if [ -e /var/run/s3-mirror/lastrun ]; then
+ . /var/run/s3-mirror/lastrun
+else
+ # 48 hours -> 172800 seconds
+ let lastrun=curdate-172800
+fi
+
+let delta=curdate-lastrun
- if [ ${report} -ne 0 ] ; then
- for target in ${targets} ; do
- report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt
- done
- echo -n "Ending at "
+# hardcoded to epel only '-e'
+/usr/local/bin/last-sync -d ${delta} -e
+
+if [ "$?" -ne "0" ]; then
+ # no changes since the last sync
+ # abort
+ exit 0
+fi
+
+for c in ${content}; do
+ if $(newer ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
+ echo "=============================================================="
+ echo -n "Starting at "
date
+ time $s3cmd $S3CMD_ARGS \
+ --cache-file /var/lib/s3-mirror/${c}.cache \
+ --delete-removed \
+ --delete-after \
+ --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
+ --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
+ --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
+ --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
+ ${tree}/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
+ report=1
+ cp -a ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
+ echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun
fi
-}
+done
-# send stdout/stderr to logfile
-sighup_handler
-mkdir -p $(dirname ${pidfile})
-echo "$$" > ${pidfile}
-repeat parallel_sync_full
+if [ ${report} -ne 0 ] ; then
+ for target in ${targets} ; do
+ report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt
+ done
+ echo -n "Ending at "
+ date
+fi
diff --git a/roles/s3-mirror/tasks/main.yml b/roles/s3-mirror/tasks/main.yml
index 7900457..3da4b8d 100644
--- a/roles/s3-mirror/tasks/main.yml
+++ b/roles/s3-mirror/tasks/main.yml
@@ -11,6 +11,16 @@
yum: pkg={{ item }} state=installed
with_items:
- s3cmd
+ - mirrormanager2-client
+
+- name: Create report_mirror config files
+ template: src=report_mirror.conf dest=/etc/mirrormanager-client/report_mirror_{{ item }} owner=s3-mirror group=s3-mirror mode=0400
+ with_items:
+ - s3-mirror-us-east-1
+ - s3-mirror-us-west-1
+ - s3-mirror-us-west-2
+ - s3-mirror-eu-west-1
+ - s3-mirror-ap-northeast-1
- name: Create needed directories
file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755
@@ -33,12 +43,15 @@
- name: s3-mirror logrotate
copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0644
-- name: s3-mirror init.d
- copy: src=s3-mirror.init dest=/etc/init.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0755
-
- name: s3sync-logs
copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs owner=s3-mirror group=s3-mirror mode=0755
+- name: s3sync cron
+ cron: name="s3sync" minute="3,18,33,48" user="s3-mirror"
+ job="/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3sync"
+ cron_file=s3sync
+ when: env != 'staging'
+
- name: s3sync-logs cron
cron: name="s3sync-logs" hour="0" user="root"
job='/usr/local/bin/lock-wrapper s3sync-logs "/bin/sleep $((${RANDOM} \% 300)); /usr/local/bin/s3sync-logs >> /var/log/s3-mirror-logs/s3sync-logs.log 2>&1" | /usr/local/bin/nag-once s3mirror-logs 12h 2>&1'
diff --git a/roles/s3-mirror/templates/report_mirror.conf b/roles/s3-mirror/templates/report_mirror.conf
new file mode 100644
index 0000000..e51e1d9
--- /dev/null
+++ b/roles/s3-mirror/templates/report_mirror.conf
@@ -0,0 +1,60 @@
+[global]
+# if enabled=0, no data is sent to the database
+enabled=1
+# server= is the URL to the MirrorManager XML-RPC interface
+server=https://admin.fedoraproject.org/mirrormanager/xmlrpc
+
+
+[site]
+# if enabled=0, no data about this site is sent to the database
+enabled=1
+# Name and Password fields need to match the Site name and password
+# fields you entered for your Site in the MirrorManager database at
+# https://admin.fedoraproject.org/mirrormanager
+name={{ s3_mirror_report_mirror_site }}
+password={{ s3_mirror_report_mirror_pw }}
+
+[host]
+# if enabled=0, no data about this host is sent to the database
+enabled=1
+# Name field need to match the Host name field you entered for your
+# Host in the MirrorManager database at
+# https://admin.fedoraproject.org/mirrormanager
+name={{ item }}.fedoraproject.org
+# if user_active=0, no data about this category is given to the public
+# This can be used to toggle between serving and not serving data,
+# such enabled during the nighttime (when you have more idle bandwidth
+# available) and disabled during the daytime.
+# By not specifying user_active, the database will not be updated.
+# user_active=1
+
+[stats]
+# Stats are only sent when run with the -s option
+# and when this section is enabled.
+# This feature is not presently implemented
+enabled=0
+apache=/var/log/httpd/access_log
+vsftpd=/var/log/vsftpd.log
+# remember to enable log file and transfer logging in rsyncd.conf
+rsyncd=/var/log/rsyncd.log
+
+
+# Content Categories
+# These sections match the Categories for content tracked by MirrorManager.
+#
+# enabled=1 means information about this category will be sent to the database.
+# enabled=0, no data about this host is sent to the database. If the
+# database already has information for you for this Category, it will
+# remain unchanged. This can be used to update the database after you
+# have manually synced some infrequently-updated content, such as
+# historical releases.
+#
+# path= is the path on your local disk to the top-level directory for this Category
+
+[Fedora Linux]
+enabled=0
+path=/srv/pub/fedora/linux
+
+[Fedora EPEL]
+enabled=1
+path=/srv/pub/epel
--
1.7.2.1
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 811 bytes
Desc: not available
URL: <http://lists.fedoraproject.org/pipermail/infrastructure/attachments/20150730/fce48464/attachment.sig>
More information about the infrastructure
mailing list