Fix s3sync and start it via cron.

This changes the s3sync setup at multiple places: * The s3sync script was using wrong paths which are now corrected * After the sync report_mirror was started which is now installed * The necessary report_mirror files are installed * The s3sync script is now running from cron instead as a service * Before touching any files fedmsg is queried to see if new files are available for syncing * logrotate has been adapted to the new setup
2026-05-05 07:34:30 +08:00 · 2015-07-30 13:47:48 +00:00
parent f7ae8f9654
commit 7efbd7f504
5 changed files with 126 additions and 133 deletions
--- a/roles/s3-mirror/files/s3-mirror.init
+++ b/roles/s3-mirror/files/s3-mirror.init
@@ -1,77 +0,0 @@
-#!/bin/sh
-#
-# s3-mirror - sync content to S3
-#
-# chkconfig:   - 99 99
-# description: sync content to S3
-
-# http://fedoraproject.org/wiki/FCNewInit/Initscripts
-### BEGIN INIT INFO
-# Provides: 
-# Required-Start: $network $named $remote_fs
-# Required-Stop: 
-# Should-Start: 
-# Should-Stop: 
-# Default-Start: 
-# Default-Stop: 
-# Short-Description: 
-# Description: 
-### END INIT INFO
-
-# Source function library.
-. /etc/rc.d/init.d/functions
-
-exec="/usr/local/bin/s3sync"
-prog=${exec##*/}
-
-lockfile=/var/lock/subsys/$prog
-pidfile=/var/run/s3-mirror/pid
-
-start() {
-    echo -n $"Starting $prog: "
-    mkdir -p /var/run/s3-mirror
-    chown -R s3-mirror:s3-mirror /var/run/s3-mirror
-    daemon --user s3-mirror --pidfile $pidfile $exec &
-    retval=$?
-    echo
-    [ $retval -eq 0 ] && touch $lockfile
-    return $retval
-}
-
-stop() {
-    echo -n $"Stopping $prog: "
-    killproc -p $pidfile
-    retval=$?
-    echo
-    [ $retval -eq 0 ] && rm -f $lockfile
-    return $retval
-}
-
-restart() {
-    stop
-    start
-}
-
-case "$1" in
-    start|stop|restart)
-        $1
-        ;;
-    force-reload)
-        restart
-        ;;
-    status)
-        status $prog
-        ;;
-    try-restart|condrestart)
-        if status $prog >/dev/null ; then
-            restart
-        fi
-	;;
-    reload)
-        action $"Service ${0##*/} does not support the reload action: " /bin/false
-        exit 3
-        ;;
-    *)
-        echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}"
-        exit 2
-esac
--- a/roles/s3-mirror/files/s3-mirror.logrotate
+++ b/roles/s3-mirror/files/s3-mirror.logrotate
@@ -9,7 +9,4 @@
    compressext .bz2
    dateext
    copytruncate
-    postrotate
-        /bin/kill -HUP `cat /var/run/s3-mirror/pid 2>/dev/null` 2>/dev/null || true
-    endscript
 }
--- a/roles/s3-mirror/files/s3sync
+++ b/roles/s3-mirror/files/s3sync
@@ -1,16 +1,16 @@
 #!/bin/bash

-pidfile=/var/run/s3-mirror/pid
 logfile="/var/log/s3-mirror/s3sync.log"
+tree="/srv/pub/"
+s3cmd=/usr/bin/s3cmd
+curdate=`date +%s`

-trap sighup_handler HUP
-trap "rm -f ${pidfile}" QUIT EXIT INT TERM
-
-function sighup_handler()
-{
-    exec 1>>${logfile} 2>&1
-}
+if [ ! -d ${tree} ]; then
+    echo "Specified root of the directory tree (${tree}) does not exist. Exiting." | tee ${logfile}
+    exit 1
+fi

+exec 1>>${logfile} 2>&1

 function newer()
 {
@@ -20,16 +20,6 @@ function newer()
    return 1
 }

-function repeat()
-{
-    while :; do
-	$1
-        /bin/sleep $((${RANDOM} % 300))
-    done
-}
-
-s3cmd=/usr/bin/s3cmd
-
 S3CMD_ARGS="sync \
  --verbose \
  --preserve \
@@ -44,41 +34,51 @@ S3CMD_ARGS="sync \

 content="epel"
 targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
+report=0

-function parallel_sync_full()
-{
-    report=0
-    for c in ${content}; do
-	if $(newer /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
-	    echo "=============================================================="
-	    echo -n "Starting at "
-	    date
-            time $s3cmd $S3CMD_ARGS  \
-		       --cache-file /var/lib/s3-mirror/${c}.cache \
-                       --delete-removed \
-                       --delete-after \
-                       --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
-                       --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
-                       --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
-                       --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
-                              /pub/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
-	    report=1
-	    cp -a /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
-	fi
-    done
+if [ -e /var/run/s3-mirror/lastrun ]; then
+        . /var/run/s3-mirror/lastrun
+else
+        # 48 hours -> 172800 seconds
+        let lastrun=curdate-172800
+fi

-    if [ ${report} -ne 0 ] ; then
-	for target in ${targets} ; do
-           report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt
-	done
-        echo -n "Ending at "
+let delta=curdate-lastrun
+
+# hardcoded to epel only '-e'
+/usr/local/bin/last-sync -d ${delta} -e
+
+if [ "$?" -ne "0" ]; then
+    # no changes since the last sync
+    # abort
+    exit 0
+fi
+
+for c in ${content}; do
+    if $(newer ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
+        echo "=============================================================="
+        echo -n "Starting at "
        date
+        time $s3cmd $S3CMD_ARGS  \
+    	       --cache-file /var/lib/s3-mirror/${c}.cache \
+                   --delete-removed \
+                   --delete-after \
+                   --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
+                   --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
+                   --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
+                   --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
+                          ${tree}/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
+        report=1
+        cp -a ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
+        echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun
    fi
-}
+done


-# send stdout/stderr to logfile
-sighup_handler
-mkdir -p $(dirname ${pidfile})
-echo "$$" > ${pidfile}
-repeat parallel_sync_full
+if [ ${report} -ne 0 ] ; then
+    for target in ${targets} ; do
+       report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt
+    done
+    echo -n "Ending at "
+    date
+fi
--- a/roles/s3-mirror/tasks/main.yml
+++ b/roles/s3-mirror/tasks/main.yml
@@ -11,6 +11,16 @@
  yum: pkg={{ item }} state=installed
  with_items:
  - s3cmd
+  - mirrormanager2-client
+
+- name: Create report_mirror config files
+  template: src=report_mirror.conf dest=/etc/mirrormanager-client/report_mirror_{{ item }} owner=s3-mirror group=s3-mirror mode=0400
+  with_items:
+  - s3-mirror-us-east-1
+  - s3-mirror-us-west-1
+  - s3-mirror-us-west-2
+  - s3-mirror-eu-west-1
+  - s3-mirror-ap-northeast-1

 - name: Create needed directories
  file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755
@@ -33,12 +43,15 @@
 - name: s3-mirror logrotate
  copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0644

- name: s3-mirror init.d
-  copy: src=s3-mirror.init dest=/etc/init.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0755
-
 - name: s3sync-logs
  copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs owner=s3-mirror group=s3-mirror mode=0755

+- name: s3sync cron
+  cron: name="s3sync" minute="3,18,33,48" user="s3-mirror"
+        job="/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3sync"
+        cron_file=s3sync
+  when: env != 'staging'
+
 - name: s3sync-logs cron
  cron: name="s3sync-logs" hour="0" user="root"
        job='/usr/local/bin/lock-wrapper s3sync-logs "/bin/sleep $((${RANDOM} \% 300)); /usr/local/bin/s3sync-logs >> /var/log/s3-mirror-logs/s3sync-logs.log 2>&1" | /usr/local/bin/nag-once s3mirror-logs 12h 2>&1'
--- a/roles/s3-mirror/templates/report_mirror.conf
+++ b/roles/s3-mirror/templates/report_mirror.conf
@@ -0,0 +1,60 @@
+[global]
+# if enabled=0, no data is sent to the database
+enabled=1
+# server= is the URL to the MirrorManager XML-RPC interface
+server=https://admin.fedoraproject.org/mirrormanager/xmlrpc
+
+
+[site]
+# if enabled=0, no data about this site is sent to the database
+enabled=1
+# Name and Password fields need to match the Site name and password
+# fields you entered for your Site in the MirrorManager database at
+# https://admin.fedoraproject.org/mirrormanager
+name={{ s3_mirror_report_mirror_site }}
+password={{ s3_mirror_report_mirror_pw }}
+
+[host]
+# if enabled=0, no data about this host is sent to the database
+enabled=1
+# Name field need to match the Host name field you entered for your
+# Host in the MirrorManager database at
+# https://admin.fedoraproject.org/mirrormanager
+name={{ item }}.fedoraproject.org
+# if user_active=0, no data about this category is given to the public
+# This can be used to toggle between serving and not serving data,
+# such enabled during the nighttime (when you have more idle bandwidth
+# available) and disabled during the daytime.
+# By not specifying user_active, the database will not be updated.
+# user_active=1
+
+[stats]
+# Stats are only sent when run with the -s option
+# and when this section is enabled.
+# This feature is not presently implemented
+enabled=0
+apache=/var/log/httpd/access_log
+vsftpd=/var/log/vsftpd.log
+# remember to enable log file and transfer logging in rsyncd.conf
+rsyncd=/var/log/rsyncd.log
+
+
+# Content Categories
+# These sections match the Categories for content tracked by MirrorManager.
+#
+# enabled=1 means information about this category will be sent to the database.
+# enabled=0, no data about this host is sent to the database.  If the
+# database already has information for you for this Category, it will
+# remain unchanged.  This can be used to update the database after you
+# have manually synced some infrequently-updated content, such as
+# historical releases.
+#
+# path= is the path on your local disk to the top-level directory for this Category
+
+[Fedora Linux]
+enabled=0
+path=/srv/pub/fedora/linux
+
+[Fedora EPEL]
+enabled=1
+path=/srv/pub/epel