Merge lp:~ev/whoopsie/exponential-backoff into lp:whoopsie

[Evan]

This branch adds exponential backing off of report submissions when one of the submissions fails. So normally whoopsie will process outstanding reports every two hours. If one of the reports in the handful it's currently processing fails, it will retry two hours and 4 minutes later, then two hours and 8 minutes if that fails, two hours and 16, and so on.

This will help us when we and if we have to bring daisy.ubuntu.com down for any length of time in that it will prevent a flood of systems hitting us all at once.

Unmerged revisions

560. By Evan on 2013-06-20

Back off exponentially.

=== modified file 'Makefile'
--- Makefile	2013-02-26 10:36:50 +0000
+++ Makefile	2013-06-20 13:54:24 +0000
@@ -12,7 +12,7 @@
 whoopsie_CFLAGS=$(shell pkg-config --cflags gio-2.0 glib-2.0 libcurl) \
 	   -g -Ilib -Wall -Werror -Os -DVERSION=\"$(VERSION)\" \
 	   $(CFLAGS) $(CPPFLAGS)
-whoopsie_LIBS=$(shell pkg-config --libs gio-2.0 glib-2.0 libcurl) -lcap \
+whoopsie_LIBS=$(shell pkg-config --libs gio-2.0 glib-2.0 libcurl) -lm -lcap \
 			  $(LDFLAGS)
 SOURCES=src/whoopsie.c \
 		src/utils.c \

=== modified file 'src/tests/Makefile'
--- src/tests/Makefile	2013-02-26 11:02:35 +0000
+++ src/tests/Makefile	2013-06-20 13:54:24 +0000
@@ -2,7 +2,7 @@
 CFLAGS=$(shell pkg-config --cflags gio-2.0 glib-2.0 libcurl) \
 		$(shell libgcrypt-config --cflags) \
 		-g -I../../lib -I.. -DVERSION=\"$(VERSION)\" -DTEST -DTEST_DIR=\"$(CURDIR)\" -Wall
-LIBS=$(shell pkg-config --libs gio-2.0 glib-2.0 libcurl) -lcap \
+LIBS=$(shell pkg-config --libs gio-2.0 glib-2.0 libcurl) -lm -lcap \
 		$(shell libgcrypt-config --libs)
 
 test_parse_report_SOURCES=test_parse_report.c \

=== modified file 'src/whoopsie.c'
--- src/whoopsie.c	2013-02-08 11:58:46 +0000
+++ src/whoopsie.c	2013-06-20 13:54:24 +0000
@@ -39,6 +39,7 @@
 #include <sys/mount.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <math.h>
 
 #include "bson/bson.h"
 #include "whoopsie.h"
@@ -47,10 +48,16 @@
 #include "monitor.h"
 #include "identifier.h"
 
-/* The length of time to wait before processing outstanding crashes, in seconds
- */
+/* The length of time to wait before processing outstanding crashes (retrying),
+ * in seconds. */
 #define PROCESS_OUTSTANDING_TIMEOUT 7200
 
+/* The starting point for exponential backoff, in seconds. */
+#define BASE_BACKOFF 60
+
+/* The number of times we've retried the upload of reports */
+static int retry_count = 0;
+
 /* If true, we have an active Internet connection. True by default in case we
  * can't bring up GNetworkMonitor */
 static gboolean online_state = TRUE;
@@ -638,7 +645,7 @@
     return success;
 }
 
-void
+gboolean
 process_existing_files (const char* report_dir)
 {
     GDir* dir = NULL;
@@ -646,6 +653,7 @@
     const gchar* ext = NULL;
     char* upload_file = NULL;
     char* crash_file = NULL;
+    gboolean failed = FALSE;
 
     dir = g_dir_open (report_dir, 0, NULL);
     while ((file = g_dir_read_name (dir)) != NULL) {
@@ -671,15 +679,29 @@
             g_free (crash_file);
             continue;
         } else if (online_state && parse_and_upload_report (crash_file)) {
+            retry_count = 0;
             if (!mark_handled (crash_file))
                 g_print ("Unable to mark report as seen (%s)\n", crash_file);
 
-        } 
+        } else {
+            failed = TRUE;
+        }
 
         g_free (upload_file);
         g_free (crash_file);
     }
     g_dir_close (dir);
+
+    if (failed) {
+        retry_count += 1;
+        /* Exponential backoff. */
+        g_timeout_add_seconds (PROCESS_OUTSTANDING_TIMEOUT + BASE_BACKOFF * pow(2, retry_count),
+                              (GSourceFunc) process_existing_files,
+                              (gpointer) report_dir);
+        /* Cancel the existing timeout, since we've just created a new one. */
+        return FALSE;
+    }
+    return TRUE;
 }
 
 void daemonize (void)

=== modified file 'src/whoopsie.h'
--- src/whoopsie.h	2012-12-11 17:56:52 +0000
+++ src/whoopsie.h	2013-06-20 13:54:24 +0000
@@ -24,6 +24,6 @@
 char* get_crash_db_url (void);
 void destroy_key_and_value (gpointer key, gpointer value, gpointer user_data);
 void drop_privileges (GError** error);
-void process_existing_files (const char* report_dir);
+gboolean process_existing_files (const char* report_dir);
 
 #endif /* WHOOPSIE_H */