Freeze break request: s3sync
Kevin Fenzi
kevin at scrye.com
Thu Jul 30 21:19:00 UTC 2015
On Thu, 30 Jul 2015 16:55:18 +0200
Adrian Reber <adrian at lisas.de> wrote:
> This is a try to fix the s3sync on mm-backend01. I have tested the
> script and it successfully syncs the content to S3. I have, however,
> not tested if the ansible rules work. Please have a look at following
> patch and if the ansible rules are correct.
They look ok to me from a quick eyeball.
We can of course run a 'ansible-playbook --syntax-check' before
pushing.
>
> In addition to this patch it is necessary to install the new
> mirrormanager2 release (0.4.2) in prod. It has now changes except an
> additional subpackage mirrormanager2-client which provides
> report_mirror which is run after the sync to S3.
+1 here.
kevin
--
>
> Date: Thu, 30 Jul 2015 13:47:48 +0000
> Subject: [PATCH] Fix s3sync and start it via cron.
>
> This changes the s3sync setup at multiple places:
>
> * The s3sync script was using wrong paths which are now corrected
> * After the sync report_mirror was started which is now installed
> * The necessary report_mirror files are installed
> * The s3sync script is now running from cron instead as a service
> * Before touching any files fedmsg is queried to see if new files
> are available for syncing
> * logrotate has been adapted to the new setup
> ---
> roles/s3-mirror/files/s3-mirror.init | 77
> -------------------- roles/s3-mirror/files/s3-mirror.logrotate
> | 3 - roles/s3-mirror/files/s3sync | 100
> +++++++++++++-------------
> roles/s3-mirror/tasks/main.yml | 19 ++++-
> roles/s3-mirror/templates/report_mirror.conf | 60 +++++++++++++++ 5
> files changed, 126 insertions(+), 133 deletions(-) delete mode 100644
> roles/s3-mirror/files/s3-mirror.init create mode 100644
> roles/s3-mirror/templates/report_mirror.conf
>
> diff --git a/roles/s3-mirror/files/s3-mirror.init
> b/roles/s3-mirror/files/s3-mirror.init deleted file mode 100644
> index 59dbb39..0000000
> --- a/roles/s3-mirror/files/s3-mirror.init
> +++ /dev/null
> @@ -1,77 +0,0 @@
> -#!/bin/sh
> -#
> -# s3-mirror - sync content to S3
> -#
> -# chkconfig: - 99 99
> -# description: sync content to S3
> -
> -# http://fedoraproject.org/wiki/FCNewInit/Initscripts
> -### BEGIN INIT INFO
> -# Provides:
> -# Required-Start: $network $named $remote_fs
> -# Required-Stop:
> -# Should-Start:
> -# Should-Stop:
> -# Default-Start:
> -# Default-Stop:
> -# Short-Description:
> -# Description:
> -### END INIT INFO
> -
> -# Source function library.
> -. /etc/rc.d/init.d/functions
> -
> -exec="/usr/local/bin/s3sync"
> -prog=${exec##*/}
> -
> -lockfile=/var/lock/subsys/$prog
> -pidfile=/var/run/s3-mirror/pid
> -
> -start() {
> - echo -n $"Starting $prog: "
> - mkdir -p /var/run/s3-mirror
> - chown -R s3-mirror:s3-mirror /var/run/s3-mirror
> - daemon --user s3-mirror --pidfile $pidfile $exec &
> - retval=$?
> - echo
> - [ $retval -eq 0 ] && touch $lockfile
> - return $retval
> -}
> -
> -stop() {
> - echo -n $"Stopping $prog: "
> - killproc -p $pidfile
> - retval=$?
> - echo
> - [ $retval -eq 0 ] && rm -f $lockfile
> - return $retval
> -}
> -
> -restart() {
> - stop
> - start
> -}
> -
> -case "$1" in
> - start|stop|restart)
> - $1
> - ;;
> - force-reload)
> - restart
> - ;;
> - status)
> - status $prog
> - ;;
> - try-restart|condrestart)
> - if status $prog >/dev/null ; then
> - restart
> - fi
> - ;;
> - reload)
> - action $"Service ${0##*/} does not support the reload
> action: " /bin/false
> - exit 3
> - ;;
> - *)
> - echo $"Usage: $0
> {start|stop|status|restart|try-restart|force-reload}"
> - exit 2
> -esac
> diff --git a/roles/s3-mirror/files/s3-mirror.logrotate
> b/roles/s3-mirror/files/s3-mirror.logrotate index 12e66aa..7549028
> 100644 --- a/roles/s3-mirror/files/s3-mirror.logrotate
> +++ b/roles/s3-mirror/files/s3-mirror.logrotate
> @@ -9,7 +9,4 @@
> compressext .bz2
> dateext
> copytruncate
> - postrotate
> - /bin/kill -HUP `cat /var/run/s3-mirror/pid 2>/dev/null`
> 2>/dev/null || true
> - endscript
> }
> diff --git a/roles/s3-mirror/files/s3sync
> b/roles/s3-mirror/files/s3sync index 6d0c552..f8d0413 100755
> --- a/roles/s3-mirror/files/s3sync
> +++ b/roles/s3-mirror/files/s3sync
> @@ -1,16 +1,16 @@
> #!/bin/bash
>
> -pidfile=/var/run/s3-mirror/pid
> logfile="/var/log/s3-mirror/s3sync.log"
> +tree="/srv/pub/"
> +s3cmd=/usr/bin/s3cmd
> +curdate=`date +%s`
>
> -trap sighup_handler HUP
> -trap "rm -f ${pidfile}" QUIT EXIT INT TERM
> -
> -function sighup_handler()
> -{
> - exec 1>>${logfile} 2>&1
> -}
> +if [ ! -d ${tree} ]; then
> + echo "Specified root of the directory tree (${tree}) does not
> exist. Exiting." | tee ${logfile}
> + exit 1
> +fi
>
> +exec 1>>${logfile} 2>&1
>
> function newer()
> {
> @@ -20,16 +20,6 @@ function newer()
> return 1
> }
>
> -function repeat()
> -{
> - while :; do
> - $1
> - /bin/sleep $((${RANDOM} % 300))
> - done
> -}
> -
> -s3cmd=/usr/bin/s3cmd
> -
> S3CMD_ARGS="sync \
> --verbose \
> --preserve \
> @@ -44,41 +34,51 @@ S3CMD_ARGS="sync \
>
> content="epel"
> targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2
> s3-mirror-eu-west-1 s3-mirror-ap-northeast-1" +report=0
>
> -function parallel_sync_full()
> -{
> - report=0
> - for c in ${content}; do
> - if
> $(newer /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ;
> then
> - echo
> "=============================================================="
> - echo -n "Starting at "
> - date
> - time $s3cmd $S3CMD_ARGS \
> - --cache-file /var/lib/s3-mirror/${c}.cache \
> - --delete-removed \
> - --delete-after \
> - --add-destination
> s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
> - --add-destination
> s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
> - --add-destination
> s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
> - --add-destination
> s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
> - /pub/${c}/
> s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
> - report=1
> - cp
> -a /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
> - fi
> - done
> +if [ -e /var/run/s3-mirror/lastrun ]; then
> + . /var/run/s3-mirror/lastrun
> +else
> + # 48 hours -> 172800 seconds
> + let lastrun=curdate-172800
> +fi
> +
> +let delta=curdate-lastrun
>
> - if [ ${report} -ne 0 ] ; then
> - for target in ${targets} ; do
> - report_mirror
> -c /etc/mirrormanager-client/report_mirror_${target}.conf
> --exclude-from /usr/local/etc/s3-mirror-excludes.txt
> - done
> - echo -n "Ending at "
> +# hardcoded to epel only '-e'
> +/usr/local/bin/last-sync -d ${delta} -e
> +
> +if [ "$?" -ne "0" ]; then
> + # no changes since the last sync
> + # abort
> + exit 0
> +fi
> +
> +for c in ${content}; do
> + if $(newer
> ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
> + echo
> "=============================================================="
> + echo -n "Starting at "
> date
> + time $s3cmd $S3CMD_ARGS \
> + --cache-file /var/lib/s3-mirror/${c}.cache \
> + --delete-removed \
> + --delete-after \
> + --add-destination
> s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
> + --add-destination
> s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
> + --add-destination
> s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
> + --add-destination
> s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
> + ${tree}/${c}/
> s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
> + report=1
> + cp -a
> ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
> + echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun
> fi
> -}
> +done
>
>
> -# send stdout/stderr to logfile
> -sighup_handler
> -mkdir -p $(dirname ${pidfile})
> -echo "$$" > ${pidfile}
> -repeat parallel_sync_full
> +if [ ${report} -ne 0 ] ; then
> + for target in ${targets} ; do
> + report_mirror
> -c /etc/mirrormanager-client/report_mirror_${target}.conf
> --exclude-from /usr/local/etc/s3-mirror-excludes.txt
> + done
> + echo -n "Ending at "
> + date
> +fi
> diff --git a/roles/s3-mirror/tasks/main.yml
> b/roles/s3-mirror/tasks/main.yml index 7900457..3da4b8d 100644
> --- a/roles/s3-mirror/tasks/main.yml
> +++ b/roles/s3-mirror/tasks/main.yml
> @@ -11,6 +11,16 @@
> yum: pkg={{ item }} state=installed
> with_items:
> - s3cmd
> + - mirrormanager2-client
> +
> +- name: Create report_mirror config files
> + template: src=report_mirror.conf
> dest=/etc/mirrormanager-client/report_mirror_{{ item }}
> owner=s3-mirror group=s3-mirror mode=0400
> + with_items:
> + - s3-mirror-us-east-1
> + - s3-mirror-us-west-1
> + - s3-mirror-us-west-2
> + - s3-mirror-eu-west-1
> + - s3-mirror-ap-northeast-1
>
> - name: Create needed directories
> file: path={{item}} owner=s3-mirror group=s3-mirror
> state=directory mode=0755 @@ -33,12 +43,15 @@
> - name: s3-mirror logrotate
> copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror
> owner=s3-mirror group=s3-mirror mode=0644
> -- name: s3-mirror init.d
> - copy: src=s3-mirror.init dest=/etc/init.d/s3-mirror
> owner=s3-mirror group=s3-mirror mode=0755 -
> - name: s3sync-logs
> copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs
> owner=s3-mirror group=s3-mirror mode=0755
> +- name: s3sync cron
> + cron: name="s3sync" minute="3,18,33,48" user="s3-mirror"
> + job="/usr/local/bin/lock-wrapper
> s3sync /usr/local/bin/s3sync"
> + cron_file=s3sync
> + when: env != 'staging'
> +
> - name: s3sync-logs cron
> cron: name="s3sync-logs" hour="0" user="root"
> job='/usr/local/bin/lock-wrapper s3sync-logs "/bin/sleep
> $((${RANDOM} \% 300)); /usr/local/bin/s3sync-logs
> >> /var/log/s3-mirror-logs/s3sync-logs.log 2>&1"
> >> | /usr/local/bin/nag-once s3mirror-logs 12h 2>&1' diff --git
> >> a/roles/s3-mirror/templates/report_mirror.conf
> >> b/roles/s3-mirror/templates/report_mirror.conf new file mode 100644
> index 0000000..e51e1d9
> --- /dev/null
> +++ b/roles/s3-mirror/templates/report_mirror.conf
> @@ -0,0 +1,60 @@
> +[global]
> +# if enabled=0, no data is sent to the database
> +enabled=1
> +# server= is the URL to the MirrorManager XML-RPC interface
> +server=https://admin.fedoraproject.org/mirrormanager/xmlrpc
> +
> +
> +[site]
> +# if enabled=0, no data about this site is sent to the database
> +enabled=1
> +# Name and Password fields need to match the Site name and password
> +# fields you entered for your Site in the MirrorManager database at
> +# https://admin.fedoraproject.org/mirrormanager
> +name={{ s3_mirror_report_mirror_site }}
> +password={{ s3_mirror_report_mirror_pw }}
> +
> +[host]
> +# if enabled=0, no data about this host is sent to the database
> +enabled=1
> +# Name field need to match the Host name field you entered for your
> +# Host in the MirrorManager database at
> +# https://admin.fedoraproject.org/mirrormanager
> +name={{ item }}.fedoraproject.org
> +# if user_active=0, no data about this category is given to the
> public +# This can be used to toggle between serving and not serving
> data, +# such enabled during the nighttime (when you have more idle
> bandwidth +# available) and disabled during the daytime.
> +# By not specifying user_active, the database will not be updated.
> +# user_active=1
> +
> +[stats]
> +# Stats are only sent when run with the -s option
> +# and when this section is enabled.
> +# This feature is not presently implemented
> +enabled=0
> +apache=/var/log/httpd/access_log
> +vsftpd=/var/log/vsftpd.log
> +# remember to enable log file and transfer logging in rsyncd.conf
> +rsyncd=/var/log/rsyncd.log
> +
> +
> +# Content Categories
> +# These sections match the Categories for content tracked by
> MirrorManager. +#
> +# enabled=1 means information about this category will be sent to
> the database. +# enabled=0, no data about this host is sent to the
> database. If the +# database already has information for you for
> this Category, it will +# remain unchanged. This can be used to
> update the database after you +# have manually synced some
> infrequently-updated content, such as +# historical releases.
> +#
> +# path= is the path on your local disk to the top-level directory
> for this Category +
> +[Fedora Linux]
> +enabled=0
> +path=/srv/pub/fedora/linux
> +
> +[Fedora EPEL]
> +enabled=1
> +path=/srv/pub/epel
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: OpenPGP digital signature
URL: <http://lists.fedoraproject.org/pipermail/infrastructure/attachments/20150730/e550548a/attachment.sig>
More information about the infrastructure
mailing list