diff --git a/roles/s3-mirror/files/s3-mirror.init b/roles/s3-mirror/files/s3-mirror.init deleted file mode 100644 index 59dbb39c12..0000000000 --- a/roles/s3-mirror/files/s3-mirror.init +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -# -# s3-mirror - sync content to S3 -# -# chkconfig: - 99 99 -# description: sync content to S3 - -# http://fedoraproject.org/wiki/FCNewInit/Initscripts -### BEGIN INIT INFO -# Provides: -# Required-Start: $network $named $remote_fs -# Required-Stop: -# Should-Start: -# Should-Stop: -# Default-Start: -# Default-Stop: -# Short-Description: -# Description: -### END INIT INFO - -# Source function library. -. /etc/rc.d/init.d/functions - -exec="/usr/local/bin/s3sync" -prog=${exec##*/} - -lockfile=/var/lock/subsys/$prog -pidfile=/var/run/s3-mirror/pid - -start() { - echo -n $"Starting $prog: " - mkdir -p /var/run/s3-mirror - chown -R s3-mirror:s3-mirror /var/run/s3-mirror - daemon --user s3-mirror --pidfile $pidfile $exec & - retval=$? - echo - [ $retval -eq 0 ] && touch $lockfile - return $retval -} - -stop() { - echo -n $"Stopping $prog: " - killproc -p $pidfile - retval=$? - echo - [ $retval -eq 0 ] && rm -f $lockfile - return $retval -} - -restart() { - stop - start -} - -case "$1" in - start|stop|restart) - $1 - ;; - force-reload) - restart - ;; - status) - status $prog - ;; - try-restart|condrestart) - if status $prog >/dev/null ; then - restart - fi - ;; - reload) - action $"Service ${0##*/} does not support the reload action: " /bin/false - exit 3 - ;; - *) - echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}" - exit 2 -esac diff --git a/roles/s3-mirror/files/s3-mirror.logrotate b/roles/s3-mirror/files/s3-mirror.logrotate index 12e66aa61e..754902880b 100644 --- a/roles/s3-mirror/files/s3-mirror.logrotate +++ b/roles/s3-mirror/files/s3-mirror.logrotate @@ -9,7 +9,4 @@ compressext .bz2 dateext copytruncate - postrotate - /bin/kill -HUP `cat /var/run/s3-mirror/pid 2>/dev/null` 2>/dev/null || true - endscript } diff --git a/roles/s3-mirror/files/s3sync b/roles/s3-mirror/files/s3sync index 6d0c552195..f8d04137ca 100755 --- a/roles/s3-mirror/files/s3sync +++ b/roles/s3-mirror/files/s3sync @@ -1,16 +1,16 @@ #!/bin/bash -pidfile=/var/run/s3-mirror/pid logfile="/var/log/s3-mirror/s3sync.log" +tree="/srv/pub/" +s3cmd=/usr/bin/s3cmd +curdate=`date +%s` -trap sighup_handler HUP -trap "rm -f ${pidfile}" QUIT EXIT INT TERM - -function sighup_handler() -{ - exec 1>>${logfile} 2>&1 -} +if [ ! -d ${tree} ]; then + echo "Specified root of the directory tree (${tree}) does not exist. Exiting." | tee ${logfile} + exit 1 +fi +exec 1>>${logfile} 2>&1 function newer() { @@ -20,16 +20,6 @@ function newer() return 1 } -function repeat() -{ - while :; do - $1 - /bin/sleep $((${RANDOM} % 300)) - done -} - -s3cmd=/usr/bin/s3cmd - S3CMD_ARGS="sync \ --verbose \ --preserve \ @@ -44,41 +34,51 @@ S3CMD_ARGS="sync \ content="epel" targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1" +report=0 -function parallel_sync_full() -{ - report=0 - for c in ${content}; do - if $(newer /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then - echo "==============================================================" - echo -n "Starting at " - date - time $s3cmd $S3CMD_ARGS \ - --cache-file /var/lib/s3-mirror/${c}.cache \ - --delete-removed \ - --delete-after \ - --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \ - --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \ - --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \ - --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \ - /pub/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/ - report=1 - cp -a /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist - fi - done +if [ -e /var/run/s3-mirror/lastrun ]; then + . /var/run/s3-mirror/lastrun +else + # 48 hours -> 172800 seconds + let lastrun=curdate-172800 +fi - if [ ${report} -ne 0 ] ; then - for target in ${targets} ; do - report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt - done - echo -n "Ending at " +let delta=curdate-lastrun + +# hardcoded to epel only '-e' +/usr/local/bin/last-sync -d ${delta} -e + +if [ "$?" -ne "0" ]; then + # no changes since the last sync + # abort + exit 0 +fi + +for c in ${content}; do + if $(newer ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then + echo "==============================================================" + echo -n "Starting at " date + time $s3cmd $S3CMD_ARGS \ + --cache-file /var/lib/s3-mirror/${c}.cache \ + --delete-removed \ + --delete-after \ + --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \ + --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \ + --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \ + --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \ + ${tree}/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/ + report=1 + cp -a ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist + echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun fi -} +done -# send stdout/stderr to logfile -sighup_handler -mkdir -p $(dirname ${pidfile}) -echo "$$" > ${pidfile} -repeat parallel_sync_full +if [ ${report} -ne 0 ] ; then + for target in ${targets} ; do + report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt + done + echo -n "Ending at " + date +fi diff --git a/roles/s3-mirror/tasks/main.yml b/roles/s3-mirror/tasks/main.yml index 7900457aa9..3da4b8d45c 100644 --- a/roles/s3-mirror/tasks/main.yml +++ b/roles/s3-mirror/tasks/main.yml @@ -11,6 +11,16 @@ yum: pkg={{ item }} state=installed with_items: - s3cmd + - mirrormanager2-client + +- name: Create report_mirror config files + template: src=report_mirror.conf dest=/etc/mirrormanager-client/report_mirror_{{ item }} owner=s3-mirror group=s3-mirror mode=0400 + with_items: + - s3-mirror-us-east-1 + - s3-mirror-us-west-1 + - s3-mirror-us-west-2 + - s3-mirror-eu-west-1 + - s3-mirror-ap-northeast-1 - name: Create needed directories file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755 @@ -33,12 +43,15 @@ - name: s3-mirror logrotate copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0644 -- name: s3-mirror init.d - copy: src=s3-mirror.init dest=/etc/init.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0755 - - name: s3sync-logs copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs owner=s3-mirror group=s3-mirror mode=0755 +- name: s3sync cron + cron: name="s3sync" minute="3,18,33,48" user="s3-mirror" + job="/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3sync" + cron_file=s3sync + when: env != 'staging' + - name: s3sync-logs cron cron: name="s3sync-logs" hour="0" user="root" job='/usr/local/bin/lock-wrapper s3sync-logs "/bin/sleep $((${RANDOM} \% 300)); /usr/local/bin/s3sync-logs >> /var/log/s3-mirror-logs/s3sync-logs.log 2>&1" | /usr/local/bin/nag-once s3mirror-logs 12h 2>&1' diff --git a/roles/s3-mirror/templates/report_mirror.conf b/roles/s3-mirror/templates/report_mirror.conf new file mode 100644 index 0000000000..e51e1d9744 --- /dev/null +++ b/roles/s3-mirror/templates/report_mirror.conf @@ -0,0 +1,60 @@ +[global] +# if enabled=0, no data is sent to the database +enabled=1 +# server= is the URL to the MirrorManager XML-RPC interface +server=https://admin.fedoraproject.org/mirrormanager/xmlrpc + + +[site] +# if enabled=0, no data about this site is sent to the database +enabled=1 +# Name and Password fields need to match the Site name and password +# fields you entered for your Site in the MirrorManager database at +# https://admin.fedoraproject.org/mirrormanager +name={{ s3_mirror_report_mirror_site }} +password={{ s3_mirror_report_mirror_pw }} + +[host] +# if enabled=0, no data about this host is sent to the database +enabled=1 +# Name field need to match the Host name field you entered for your +# Host in the MirrorManager database at +# https://admin.fedoraproject.org/mirrormanager +name={{ item }}.fedoraproject.org +# if user_active=0, no data about this category is given to the public +# This can be used to toggle between serving and not serving data, +# such enabled during the nighttime (when you have more idle bandwidth +# available) and disabled during the daytime. +# By not specifying user_active, the database will not be updated. +# user_active=1 + +[stats] +# Stats are only sent when run with the -s option +# and when this section is enabled. +# This feature is not presently implemented +enabled=0 +apache=/var/log/httpd/access_log +vsftpd=/var/log/vsftpd.log +# remember to enable log file and transfer logging in rsyncd.conf +rsyncd=/var/log/rsyncd.log + + +# Content Categories +# These sections match the Categories for content tracked by MirrorManager. +# +# enabled=1 means information about this category will be sent to the database. +# enabled=0, no data about this host is sent to the database. If the +# database already has information for you for this Category, it will +# remain unchanged. This can be used to update the database after you +# have manually synced some infrequently-updated content, such as +# historical releases. +# +# path= is the path on your local disk to the top-level directory for this Category + +[Fedora Linux] +enabled=0 +path=/srv/pub/fedora/linux + +[Fedora EPEL] +enabled=1 +path=/srv/pub/epel