Merge lp:~jpds/launchpad/archivepublisher-log-to-file into lp:launchpad

[Jonathan Davies]

Made cron.publish-ftpmaster log messages to a file rather than a cron mail spool.

= Summary =

This branch makes the cron.publish-ftpmaster cronjob log all of its output to a log file in:

/srv/launchpad.net/production-logs/lp_publish/publish-ftpmaster.log

...rather than having cron send all the output to the lp_publish user's mail spool file where it's building up and hard to find what ran at what time.

It also rotates the log when it's completed a run and keep up to a month's worth of logs.

All actions are additionally timestamped.

[Graham Binns (gmb)]

review: Approve (code)

[Tom Haddon (mthaddon)]

For reasons discussed on IRC I'd rather this wasn't merged. We should just do this in the crontab rather than in the script itself.

[Julian Edwards (julian-edwards)]

Can you paste the conversation in here for posterity please?

Unmerged revisions

10464. By Jonathan Davies on 2010-03-09

Timestamp these two last things.

10463. By Jonathan Davies on 2010-03-09

Log all output of cron.publish-ftpmaster to file instead of via cron emails and
timestamp everything we do.

=== modified file 'cronscripts/publishing/cron.publish-ftpmaster'
--- cronscripts/publishing/cron.publish-ftpmaster	2010-03-02 17:48:37 +0000
+++ cronscripts/publishing/cron.publish-ftpmaster	2010-03-09 15:48:26 +0000
@@ -43,27 +43,37 @@
 # Configuration options.
 LAUNCHPADROOT=/srv/launchpad.net/codelines/current
 LOCKFILE=/srv/launchpad.net/ubuntu-archive/cron.daily.lock
+LOGDIR=/srv/launchpad.net/production-logs
+LOGFILE=$LOGDIR/lp_publish/publish-ftpmaster.log
 DISTRONAME=ubuntu
 TRACEFILE=$ARCHIVEROOT/project/trace/$(hostname --fqdn)
 DSYNCLIST=$CACHEROOT/dsync.list
 MD5LIST=$INDICES/md5sums.gz
 
+# Mirrors to push at the end of the publishing run.
+MASTERMIRRORS="syowa frei scandium"
+
 # Manipulate the environment.
 export GNUPGHOME
 PATH=$PATH:$LAUNCHPADROOT/scripts:$LAUNCHPADROOT/cronscripts:$LAUNCHPADROOT/cronscripts/publishing:$LAUNCHPADROOT/scripts/ftpmaster-tools
 
+# Log all output to file.
+exec >> $LOGFILE 2>&1
+
 # Claim the lock.
 if ! lockfile -r1 $LOCKFILE; then
   echo "Could not claim lock file."
   exit 1
 fi
 
+echo "$(date -R): Initiating archive publishing operations..."
+
 DONEPUB=no
 
 cleanup () {
-  echo "Cleaning up lockfile."
+  echo "$(date -R): Cleaning up lockfile."
   rm -f $LOCKFILE
-  echo "Moving dists backup to safe keeping for next time."
+  echo "$(date -R): Moving dists backup to safe keeping for next time."
   if [ "x$DONEPUB" = "xyes" ]; then
     if [ -d ${DISTSROOT}.old ]; then
       mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
@@ -82,6 +92,10 @@
   if [ "$SECURITY_PUBLISHER" = "yes" ]; then
     exec $0
   fi
+
+  # Keep up to a month of log files.
+  echo "$(date -R): Rotating log file."
+  savelog -c 750 $LOGFILE > /dev/null 2>&1
 }
 
 trap cleanup EXIT
@@ -89,11 +103,13 @@
 # Lock claimed.
 
 # Process the accepted queue into the publishing records.
+echo "$(date -R): Processing the accepted queue into the publishing records..."
 process-accepted.py -v -v -v $DISTRONAME
 
 # If doing a security run, find out which suites are pending publication.
 SUITEOPTS=""
 if [ "$SECURITY_PUBLISHER" = "yes" ]; then
+    echo "$(date -R): Querying which suites are pending publication..."
     SUITES=$(lp-query-distro.py pending_suites)
     SECURITY_SUITES="no"
     if [ -n "$SUITES" ]; then
@@ -109,7 +125,7 @@
         done
     fi
     if [ "$SECURITY_SUITES" != "yes" ]; then
-        echo "Nothing to do for security publisher; exiting."
+        echo "$(date -R): Nothing to do for security publisher; exiting."
         exit 0
     fi
 fi
@@ -122,21 +138,27 @@
 # This should achieve the same as copying, only faster.
 
 # Create backup dists folder, if this is the first time.
+echo "$(date -R): Creating backup dists directories..."
 mkdir -p ${DISTSCOPYROOT}/dists
 mkdir -p ${DISTSCOPYROOT_PARTNER}/dists
 
 # Move the backup dists folder into place.
+echo "$(date -R): Moving backup dists into place..."
 mv ${DISTSCOPYROOT}/dists ${ARCHIVEROOT}/dists.new
 mv ${DISTSCOPYROOT_PARTNER}/dists ${ARCHIVEROOT_PARTNER}/dists.new
 
 # Bring it up-to-date efficiently with rsync. --delete is required to
 # ensure we don't ressurect things previously deleted, bug 58835.
+echo "$(date -R): Updating dists directories..."
 rsync -aH --delete ${DISTSROOT}/ ${ARCHIVEROOT}/dists.new
 rsync -aH --delete ${DISTSROOT_PARTNER}/ ${ARCHIVEROOT_PARTNER}/dists.new
 
 # Publish the results for all archives (except PPA).
 # The -R only affects the primary and the partner archive.
+echo "$(date -R): Publishing the $DISTRONAME partner archive..."
 publish-distro.py -v -v --partner -d $DISTRONAME -R ${DISTSROOT_PARTNER}.new
+
+echo "$(date -R): Publishing the $DISTRONAME archive..."
 publish-distro.py -v -v -d $DISTRONAME $SUITEOPTS -R ${DISTSROOT}.new
 
 set +x
@@ -148,39 +170,40 @@
        $(find ${DISTSROOT}.new/*/*/dist-upgrader* -name "*.tar.gz"); do
   #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ]
   if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
-    echo "(re-)signing $CANDIDATE"
+    echo "$(date -R): (re-)signing $CANDIDATE"
     gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
   else
-    echo "Not re-signing $CANDIDATE"
+    echo "$(date -R): Not re-signing $CANDIDATE"
   fi
 done
 SIGNLIST_PARTNER=$(find ${DISTSROOT_PARTNER}.new -maxdepth 2 -name Release)
 for CANDIDATE in $SIGNLIST_PARTNER; do
   #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ].
   if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
-    echo "(re-)signing $CANDIDATE"
+    echo "$(date -R): (re-)signing $CANDIDATE"
     gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
   else
-    echo "Not re-signing $CANDIDATE"
+    echo "$(date -R): Not re-signing $CANDIDATE"
   fi
 done
 
 # The Packages and Sources files are very large and would cripple our
 # mirrors, so we remove them now that the uncompressed MD5SUMS are in the
 # Release files.
-echo Removing uncompressed Packages and Sources files
+echo "$(date -R): Removing uncompressed Packages and Sources files"
 find ${DISTSROOT}.new \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
 find ${DISTSROOT_PARTNER} \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
 
 # Copy in the indices.
 if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
-    echo Copying the indices into place.
+    echo "$(date -R): Copying the indices into place."
     rm -f $INDICES/override.*
     cp $OVERRIDEROOT/override.* $INDICES
 fi
 
 # As close to atomically as possible, put the new dists into place for the
 # primary and partner archives.
+echo "$(date -R): Placing the new dists into place..."
 mv $DISTSROOT ${DISTSROOT}.old
 mv ${DISTSROOT}.new $DISTSROOT
 mv $DISTSROOT_PARTNER ${DISTSROOT_PARTNER}.old
@@ -197,6 +220,7 @@
 # Timestamp our trace file to track when the last archive publisher run took
 # place.
 if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
+    echo "$(date -R): Timestamping trace file..."
     date -u > "$TRACEFILE"
 fi
 
@@ -204,6 +228,7 @@
 if [ "$SECURITY_PUBLISHER" != "yes" ]; then
 
     # Make the lslr because we all love those.
+    echo "$(date -R): Creating ls-lR.gz..."
     LSLR=ls-lR.gz
     ( cd $ARCHIVEROOT ; \
       rm -f .$LSLR.new ; \
@@ -215,25 +240,35 @@
       mv -f .$LSLR.new $LSLR )
 
     # Run dsync over primary archive only.
+    echo "$(date -R): Running dsync over primary archive..."
     ( cd $ARCHIVEROOT ; \
       dsync-flist -q generate $DSYNCLIST -e 'Packages*' -e 'Sources*' -e 'Release*' --md5 ; \
       (dsync-flist -q md5sums $DSYNCLIST; find dists '(' -name 'Packages*' -o -name 'Sources*' -o -name 'Release*' ')' -print | xargs -r md5sum) | gzip -9n > ${MD5LIST} ; \
       dsync-flist -q link-dups $DSYNCLIST || true )
 
     # Clear out empty and thus redundant dirs.
+    echo "$(date -R): Clearing out empty directories..."
     find $ARCHIVEROOT -type d -empty | xargs -r rmdir
     find $ARCHIVEROOT_PARTNER -type d -empty | xargs -r rmdir
 
     # Invoke cron.germinate to fill ubuntu tasks and ignore failures,
     # the mirrors should be always triggered.
-    cron.germinate || echo "cron.germinate failed with exit code $?" >&2
+    echo "$(date -R): Running cron.germinate..."
+    cron.germinate || echo "$(date -R): cron.germinate failed with exit code $?"
 
 # End of block skipped by security publishing.
 fi
 
 # Trigger master mirrors.
 if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
-    ssh archvsync@syowa
-    ssh archvsync@frei
-    ssh archvsync@scandium
+    echo "$(date -R): Triggering master mirrors..."
+
+    for HOST in $MASTERMIRRORS; do
+        echo "$(date -R): Triggering $HOST:"
+        ssh archvsync@$HOST
+    done
+
+    echo "$(date -R): Master mirror triggers completed."
 fi
+
+echo "$(date -R): Archive publishing operations completed."