Merge lp:~chipaca/account-polld/intrainvocatory-coalescence into lp:~ubuntu-push-hackers/account-polld/trunk

[John Lenton]

first pass at grouping notifications by plugin (next pass: make gmail track threads, not messages).

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:97
http://jenkins.qa.ubuntu.com/job/account-polld-ci/85/
Executed test runs:
    SUCCESS: http://jenkins.qa.ubuntu.com/job/account-polld-utopic-amd64-ci/85
    SUCCESS: http://jenkins.qa.ubuntu.com/job/account-polld-utopic-armhf-ci/86
        deb: http://jenkins.qa.ubuntu.com/job/account-polld-utopic-armhf-ci/86/artifact/work/output/*zip*/output.zip
    SUCCESS: http://jenkins.qa.ubuntu.com/job/account-polld-utopic-i386-ci/85

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/account-polld-ci/85/rebuild

review: Approve (continuous-integration)

[Roberto Alsina (ralsina)]

review: Approve

[lp:~chipaca/account-polld/intrainvocatory-coalescence]

98. By John Lenton on 2014-12-09

fixes lp:1400749

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:98
http://jenkins.qa.ubuntu.com/job/account-polld-ci/86/
Executed test runs:
    SUCCESS: http://jenkins.qa.ubuntu.com/job/account-polld-vivid-amd64-ci/1
    SUCCESS: http://jenkins.qa.ubuntu.com/job/account-polld-vivid-armhf-ci/1
        deb: http://jenkins.qa.ubuntu.com/job/account-polld-vivid-armhf-ci/1/artifact/work/output/*zip*/output.zip
    SUCCESS: http://jenkins.qa.ubuntu.com/job/account-polld-vivid-i386-ci/1

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/account-polld-ci/86/rebuild

review: Approve (continuous-integration)

=== modified file 'cmd/account-polld/account_manager.go'
--- cmd/account-polld/account_manager.go	2014-10-02 13:58:49 +0000
+++ cmd/account-polld/account_manager.go	2014-12-09 15:20:23 +0000
@@ -120,7 +120,7 @@
 		return
 	}
 
-	if n, err := a.plugin.Poll(&a.authData); err != nil {
+	if bs, err := a.plugin.Poll(&a.authData); err != nil {
 		log.Print("Error while polling ", a.authData.AccountId, ": ", err)
 
 		// If the error indicates that the authentication
@@ -132,10 +132,10 @@
 		}
 		a.doneChan <- err
 	} else {
-		log.Println("Account", a.authData.AccountId, "has", len(n), "updates to report")
-		if len(n) > 0 {
-			a.postWatch <- &PostWatch{messages: n, appId: a.plugin.ApplicationId()}
+		for _, b := range bs {
+			log.Println("Account", a.authData.AccountId, "has", len(b.Messages), b.Tag, "updates to report")
 		}
+		a.postWatch <- &PostWatch{batches: bs, appId: a.plugin.ApplicationId()}
 		a.doneChan <- nil
 	}
 }

=== modified file 'cmd/account-polld/main.go'
--- cmd/account-polld/main.go	2014-10-02 13:58:49 +0000
+++ cmd/account-polld/main.go	2014-12-09 15:20:23 +0000
@@ -37,8 +37,8 @@
 )
 
 type PostWatch struct {
-	appId    plugins.ApplicationId
-	messages []plugins.PushMessage
+	appId   plugins.ApplicationId
+	batches []*plugins.PushMessageBatch
 }
 
 /* Use identifiers and API keys provided by the respective webapps which are the official
@@ -152,18 +152,66 @@
 
 func postOffice(bus *dbus.Connection, postWatch chan *PostWatch) {
 	for post := range postWatch {
-		for _, n := range post.messages {
-			var pushMessage string
-			if out, err := json.Marshal(n); err == nil {
-				pushMessage = string(out)
-			} else {
-				log.Printf("Cannot marshall %#v to json: %s", n, err)
-				continue
-			}
-			obj := bus.Object(POSTAL_SERVICE, pushObjectPath(post.appId))
-			if _, err := obj.Call(POSTAL_INTERFACE, "Post", post.appId, pushMessage); err != nil {
-				log.Println("Cannot call the Post Office:", err)
-				log.Println("Message missed posting:", pushMessage)
+		obj := bus.Object(POSTAL_SERVICE, pushObjectPath(post.appId))
+		pers, err := obj.Call(POSTAL_INTERFACE, "ListPersistent", post.appId)
+		if err != nil {
+			log.Println("Could not list previous messages:", err)
+			continue
+		}
+		var tags []string
+		tagmap := make(map[string]int)
+		if err := pers.Args(&tags); err != nil {
+			log.Println("Could not get tags:", err)
+			continue
+		}
+		log.Printf("Previous messages: %#v\n", tags)
+		for _, tag := range tags {
+			tagmap[tag]++
+		}
+
+		for _, batch := range post.batches {
+			// add individual notifications upto the batch limit
+			// (minus currently presented notifications). If
+			// overflowed and no overflow present, present that.
+			var notifs []*plugins.PushMessage
+			add := batch.Limit - tagmap[batch.Tag]
+			if add > 0 {
+				// there are less notifications presented than
+				// the limit; we can show some
+				if len(batch.Messages) < add {
+					notifs = batch.Messages
+				} else {
+					notifs = batch.Messages[:add]
+				}
+			}
+			for _, n := range notifs {
+				n.Notification.Tag = batch.Tag
+			}
+			ofTag := batch.Tag + "-overflow"
+			if len(notifs) < len(batch.Messages) {
+				// overflow
+				n := batch.OverflowHandler(batch.Messages[len(notifs):])
+				n.Notification.Tag = ofTag
+				if tagmap[ofTag] != 0 {
+					// sneakily, don't replace the overflow card,
+					// but do play the sound & etc
+					n.Notification.Card = nil
+				}
+				notifs = append(notifs, n)
+			}
+
+			for _, n := range notifs {
+				var pushMessage string
+				if out, err := json.Marshal(n); err == nil {
+					pushMessage = string(out)
+				} else {
+					log.Printf("Cannot marshall %#v to json: %s", n, err)
+					continue
+				}
+				if _, err := obj.Call(POSTAL_INTERFACE, "Post", post.appId, pushMessage); err != nil {
+					log.Println("Cannot call the Post Office:", err)
+					log.Println("Message missed posting:", pushMessage)
+				}
 			}
 		}
 	}

=== modified file 'plugins/facebook/facebook.go'
--- plugins/facebook/facebook.go	2014-09-25 14:02:14 +0000
+++ plugins/facebook/facebook.go	2014-12-09 15:20:23 +0000
@@ -148,36 +148,20 @@
 	return validNotifications
 }
 
-func (p *fbPlugin) buildPushMessages(notifications []Notification, doc Document, max int, consolidatedIndexStart int) []plugins.PushMessage {
-	pushMsg := []plugins.PushMessage{}
-	for _, n := range notifications {
-		msg := n.buildPushMessage()
-		pushMsg = append(pushMsg, *msg)
-		if len(pushMsg) == max {
-			break
-		}
-	}
-	// Now we consolidate the remaining statuses
-	if len(notifications) > len(pushMsg) && len(notifications) >= consolidatedIndexStart {
-		usernamesMap := doc.getConsolidatedMessagesUsernames(consolidatedIndexStart)
-		usernames := []string{}
-		for _, v := range usernamesMap {
-			usernames = append(usernames, v)
-			// we don't too many usernames listed, this is a hard number
-			if len(usernames) > 10 {
-				usernames = append(usernames, "...")
-				break
-			}
-		}
-		if len(usernames) > 0 {
-			consolidatedMsg := doc.getConsolidatedMessage(usernames)
-			pushMsg = append(pushMsg, *consolidatedMsg)
-		}
-	}
-	return pushMsg
+func (p *fbPlugin) buildPushMessages(notifications []Notification, doc Document, max int, consolidatedIndexStart int) *plugins.PushMessageBatch {
+	pushMsg := make([]*plugins.PushMessage, len(notifications))
+	for i, n := range notifications {
+		pushMsg[i] = n.buildPushMessage()
+	}
+	return &plugins.PushMessageBatch{
+		Messages:        pushMsg,
+		Limit:           max,
+		OverflowHandler: doc.handleOverflow,
+		Tag:             doc.getTag(),
+	}
 }
 
-func (p *fbPlugin) parseResponse(resp *http.Response) ([]plugins.PushMessage, error) {
+func (p *fbPlugin) parseResponse(resp *http.Response) (*plugins.PushMessageBatch, error) {
 	var result notificationDoc
 	if err := p.decodeResponse(resp, &result); err != nil {
 		return nil, err
@@ -188,7 +172,7 @@
 	return pushMsgs, nil
 }
 
-func (p *fbPlugin) parseInboxResponse(resp *http.Response) ([]plugins.PushMessage, error) {
+func (p *fbPlugin) parseInboxResponse(resp *http.Response) (*plugins.PushMessageBatch, error) {
 	var result inboxDoc
 	if err := p.decodeResponse(resp, &result); err != nil {
 		return nil, err
@@ -198,7 +182,7 @@
 	return pushMsgs, nil
 }
 
-func (p *fbPlugin) getNotifications(authData *accounts.AuthData) ([]plugins.PushMessage, error) {
+func (p *fbPlugin) getNotifications(authData *accounts.AuthData) (*plugins.PushMessageBatch, error) {
 	resp, err := doRequest(authData, "me/notifications")
 	if err != nil {
 		log.Println("facebook plugin: notifications poll failed: ", err)
@@ -212,7 +196,7 @@
 	return notifications, nil
 }
 
-func (p *fbPlugin) getInbox(authData *accounts.AuthData) ([]plugins.PushMessage, error) {
+func (p *fbPlugin) getInbox(authData *accounts.AuthData) (*plugins.PushMessageBatch, error) {
 	resp, err := doRequest(authData, "me/inbox?fields=unread,unseen,comments.limit(1)")
 	if err != nil {
 		log.Println("facebook plugin: inbox poll failed: ", err)
@@ -226,7 +210,7 @@
 	return inbox, nil
 }
 
-func (p *fbPlugin) Poll(authData *accounts.AuthData) ([]plugins.PushMessage, error) {
+func (p *fbPlugin) Poll(authData *accounts.AuthData) ([]*plugins.PushMessageBatch, error) {
 	// This envvar check is to ease testing.
 	if token := os.Getenv("ACCOUNT_POLLD_TOKEN_FACEBOOK"); token != "" {
 		authData.AccessToken = token
@@ -237,8 +221,13 @@
 	if notifErr != nil && inboxErr != nil {
 		return nil, fmt.Errorf("poll failed with '%s' and '%s'", notifErr, inboxErr)
 	}
-	messages := append(notifications, inbox...)
-	return messages, nil
+	if notifErr != nil {
+		return []*plugins.PushMessageBatch{inbox}, nil
+	}
+	if inboxErr != nil {
+		return []*plugins.PushMessageBatch{notifications}, nil
+	}
+	return []*plugins.PushMessageBatch{notifications, inbox}, nil
 }
 
 func toEpoch(stamp timeStamp) int64 {
@@ -350,19 +339,19 @@
 	Message     string    `json:message`
 }
 
-func (doc *inboxDoc) getConsolidatedMessagesUsernames(idxStart int) map[string]string {
-	usernamesMap := make(map[string]string)
-	for _, t := range doc.Data[idxStart-1:] {
-		message := t.Comments.Data[0]
-		userId := message.From.Id
-		if _, ok := usernamesMap[userId]; !ok {
-			usernamesMap[userId] = message.From.Name
+func (doc *inboxDoc) getTag() string {
+	return "inbox"
+}
+
+func (doc *inboxDoc) handleOverflow(pushMsg []*plugins.PushMessage) *plugins.PushMessage {
+	usernames := []string{}
+	for _, m := range pushMsg {
+		usernames = append(usernames, m.Notification.Card.Summary)
+		if len(usernames) > 10 {
+			usernames[10] = "…"
+			break
 		}
 	}
-	return usernamesMap
-}
-
-func (doc *inboxDoc) getConsolidatedMessage(usernames []string) *plugins.PushMessage {
 	// TRANSLATORS: This represents a message summary about more facebook messages
 	summary := gettext.Gettext("Multiple more messages")
 	// TRANSLATORS: This represents a message body with the comma separated facebook usernames
@@ -401,17 +390,19 @@
 	return fmt.Sprintf("id: %s, dated: %s, unread: %d, unseen: %d", t.Id, t.UpdatedTime, t.Unread, t.Unseen)
 }
 
-func (doc *notificationDoc) getConsolidatedMessagesUsernames(idxStart int) map[string]string {
-	usernamesMap := make(map[string]string)
-	for _, n := range doc.Data[idxStart:] {
-		if _, ok := usernamesMap[n.From.Id]; !ok {
-			usernamesMap[n.From.Id] = n.From.Name
+func (doc *notificationDoc) getTag() string {
+	return "notification"
+}
+
+func (doc *notificationDoc) handleOverflow(pushMsg []*plugins.PushMessage) *plugins.PushMessage {
+	usernames := []string{}
+	for _, m := range pushMsg {
+		usernames = append(usernames, m.Notification.Card.Summary)
+		if len(usernames) > 10 {
+			usernames[10] = "…"
+			break
 		}
 	}
-	return usernamesMap
-}
-
-func (doc *notificationDoc) getConsolidatedMessage(usernames []string) *plugins.PushMessage {
 	// TRANSLATORS: This represents a notification summary about more facebook notifications
 	summary := gettext.Gettext("Multiple more notifications")
 	// TRANSLATORS: This represents a notification body with the comma separated facebook usernames
@@ -447,8 +438,8 @@
 }
 
 type Document interface {
-	getConsolidatedMessage([]string) *plugins.PushMessage
-	getConsolidatedMessagesUsernames(int) map[string]string
+	getTag() string
+	handleOverflow([]*plugins.PushMessage) *plugins.PushMessage
 	size() int
 	notification(int) Notification
 }

=== modified file 'plugins/facebook/facebook_test.go'
--- plugins/facebook/facebook_test.go	2014-09-02 18:32:07 +0000
+++ plugins/facebook/facebook_test.go	2014-12-09 15:20:23 +0000
@@ -354,8 +354,10 @@
 		Body:       closeWrapper{bytes.NewReader([]byte(notificationsBody))},
 	}
 	p := &fbPlugin{}
-	messages, err := p.parseResponse(resp)
+	batch, err := p.parseResponse(resp)
 	c.Assert(err, IsNil)
+	c.Assert(batch, NotNil)
+	messages := batch.Messages
 	c.Assert(len(messages), Equals, 2)
 	c.Check(messages[0].Notification.Card.Summary, Equals, "Sender")
 	c.Check(messages[0].Notification.Card.Body, Equals, "Sender posted on your timeline: \"The message...\"")
@@ -370,15 +372,18 @@
 		Body:       closeWrapper{bytes.NewReader([]byte(largeNotificationsBody))},
 	}
 	p := &fbPlugin{}
-	messages, err := p.parseResponse(resp)
+	batch, err := p.parseResponse(resp)
 	c.Assert(err, IsNil)
-	c.Assert(len(messages), Equals, 3)
+	c.Assert(batch, NotNil)
+	messages := batch.Messages
+	c.Assert(len(messages), Equals, 4)
 	c.Check(messages[0].Notification.Card.Summary, Equals, "Sender")
 	c.Check(messages[0].Notification.Card.Body, Equals, "Sender posted on your timeline: \"The message...\"")
 	c.Check(messages[1].Notification.Card.Summary, Equals, "Sender2")
 	c.Check(messages[1].Notification.Card.Body, Equals, "Sender2's birthday was on July 7.")
-	c.Check(messages[2].Notification.Card.Summary, Equals, "Multiple more notifications")
-	c.Check(messages[2].Notification.Card.Body, Equals, "From Sender3, Sender2")
+	ofMsg := batch.OverflowHandler(messages[2:])
+	c.Check(ofMsg.Notification.Card.Summary, Equals, "Multiple more notifications")
+	c.Check(ofMsg.Notification.Card.Body, Equals, "From Sender3, Sender2")
 	c.Check(p.state.LastUpdate, Equals, timeStamp("2014-07-12T09:51:57+0000"))
 }
 
@@ -388,8 +393,10 @@
 		Body:       closeWrapper{bytes.NewReader([]byte(notificationsBody))},
 	}
 	p := &fbPlugin{state: fbState{LastUpdate: "2014-07-08T06:17:52+0000"}}
-	messages, err := p.parseResponse(resp)
+	batch, err := p.parseResponse(resp)
 	c.Assert(err, IsNil)
+	c.Assert(batch, NotNil)
+	messages := batch.Messages
 	c.Assert(len(messages), Equals, 1)
 	c.Check(messages[0].Notification.Card.Summary, Equals, "Sender")
 	c.Check(messages[0].Notification.Card.Body, Equals, "Sender posted on your timeline: \"The message...\"")
@@ -456,15 +463,20 @@
 		Body:       closeWrapper{bytes.NewReader([]byte(inboxBody))},
 	}
 	p := &fbPlugin{}
-	messages, err := p.parseInboxResponse(resp)
+	batch, err := p.parseInboxResponse(resp)
 	c.Assert(err, IsNil)
+	c.Assert(batch, NotNil)
+	messages := batch.Messages
 	c.Assert(len(messages), Equals, 3)
 	c.Check(messages[0].Notification.Card.Summary, Equals, "Pollod Magnifico")
 	c.Check(messages[0].Notification.Card.Body, Equals, "Hola mundo!")
 	c.Check(messages[1].Notification.Card.Summary, Equals, "Pollitod Magnifico")
 	c.Check(messages[1].Notification.Card.Body, Equals, "Hola!")
-	c.Check(messages[2].Notification.Card.Summary, Equals, "Multiple more messages")
-	c.Check(messages[2].Notification.Card.Body, Equals, "From Pollitod Magnifico, A Friend")
+
+	ofMsg := batch.OverflowHandler(messages[batch.Limit:])
+
+	c.Check(ofMsg.Notification.Card.Summary, Equals, "Multiple more messages")
+	c.Check(ofMsg.Notification.Card.Body, Equals, "From A Friend")
 	c.Check(p.state.LastInboxUpdate, Equals, timeStamp("2014-08-25T18:39:32+0000"))
 }
 
@@ -518,8 +530,9 @@
 	var doc inboxDoc
 	p.decodeResponse(resp, &doc)
 	notifications := p.filterNotifications(&doc, &p.state.LastInboxUpdate)
-	pushMsgs := p.buildPushMessages(notifications, &doc, doc.size(), doc.size())
-	c.Check(pushMsgs, HasLen, doc.size())
+	batch := p.buildPushMessages(notifications, &doc, doc.size(), doc.size())
+	c.Assert(batch, NotNil)
+	c.Check(batch.Messages, HasLen, doc.size())
 }
 
 func (s *S) TestBuildPushMessagesConsolidate(c *C) {
@@ -533,9 +546,11 @@
 	p.decodeResponse(resp, &doc)
 	notifications := p.filterNotifications(&doc, &p.state.LastInboxUpdate)
 	max := doc.size() - 2
-	pushMsgs := p.buildPushMessages(notifications, &doc, max, max)
-	// we should get max + 1 messages
-	c.Check(pushMsgs, HasLen, max+1)
+	batch := p.buildPushMessages(notifications, &doc, max, max)
+	c.Assert(batch, NotNil)
+	// we should get all the messages in a batch with Limit=max
+	c.Check(batch.Messages, HasLen, len(notifications))
+	c.Check(batch.Limit, Equals, max)
 }
 
 func (s *S) TestStateFromStorageInitialState(c *C) {
@@ -639,8 +654,10 @@
 	authData := accounts.AuthData{}
 	authData.AccessToken = "foo"
 	p := &fbPlugin{state: state, accountId: 32}
-	msgs, err := p.getInbox(&authData)
-	c.Check(err, IsNil)
+	batch, err := p.getInbox(&authData)
+	c.Assert(err, IsNil)
+	c.Assert(batch, NotNil)
+	msgs := batch.Messages
 	c.Check(msgs, HasLen, 3)
 }
 
@@ -685,8 +702,10 @@
 	authData := accounts.AuthData{}
 	authData.AccessToken = "foo"
 	p := &fbPlugin{state: state, accountId: 32}
-	msgs, err := p.getNotifications(&authData)
-	c.Check(err, IsNil)
+	batch, err := p.getNotifications(&authData)
+	c.Assert(err, IsNil)
+	c.Assert(batch, NotNil)
+	msgs := batch.Messages
 	c.Check(msgs, HasLen, 2)
 }
 
@@ -735,9 +754,16 @@
 	authData := accounts.AuthData{}
 	authData.AccessToken = "foo"
 	p := &fbPlugin{state: state, accountId: 32}
-	msgs, err := p.Poll(&authData)
-	c.Check(err, IsNil)
-	c.Check(msgs, HasLen, 5)
+	batches, err := p.Poll(&authData)
+	c.Assert(err, IsNil)
+	c.Assert(batches, NotNil)
+	c.Assert(batches, HasLen, 2)
+	c.Assert(batches[0], NotNil)
+	c.Assert(batches[1], NotNil)
+	c.Check(batches[0].Tag, Equals, "notification")
+	c.Check(batches[0].Messages, HasLen, 2)
+	c.Check(batches[1].Tag, Equals, "inbox")
+	c.Check(batches[1].Messages, HasLen, 3)
 }
 
 func (s *S) TestPollSingleError(c *C) {
@@ -755,9 +781,11 @@
 	authData := accounts.AuthData{}
 	authData.AccessToken = "foo"
 	p := &fbPlugin{state: state, accountId: 32}
-	msgs, err := p.Poll(&authData)
-	c.Check(err, IsNil)
-	c.Check(msgs, HasLen, 3)
+	batches, err := p.Poll(&authData)
+	c.Assert(err, IsNil)
+	c.Assert(batches, HasLen, 1)
+	c.Assert(batches[0], NotNil)
+	c.Check(batches[0].Messages, HasLen, 3)
 }
 
 func (s *S) TestPollError(c *C) {

=== modified file 'plugins/gmail/api.go'
--- plugins/gmail/api.go	2014-08-05 18:30:24 +0000
+++ plugins/gmail/api.go	2014-12-09 15:20:23 +0000
@@ -26,7 +26,7 @@
 
 const gmailTime = "Mon, 2 Jan 2006 15:04:05 -0700"
 
-type pushes map[string]plugins.PushMessage
+type pushes map[string]*plugins.PushMessage
 type headers map[string]string
 
 // messageList holds a response to call to Users.messages: list

=== modified file 'plugins/gmail/gmail.go'
--- plugins/gmail/gmail.go	2014-08-23 20:00:19 +0000
+++ plugins/gmail/gmail.go	2014-12-09 15:20:23 +0000
@@ -100,7 +100,7 @@
 	return plugins.ApplicationId(APP_ID)
 }
 
-func (p *GmailPlugin) Poll(authData *accounts.AuthData) ([]plugins.PushMessage, error) {
+func (p *GmailPlugin) Poll(authData *accounts.AuthData) ([]*plugins.PushMessageBatch, error) {
 	// This envvar check is to ease testing.
 	if token := os.Getenv("ACCOUNT_POLLD_TOKEN_GMAIL"); token != "" {
 		authData.AccessToken = token
@@ -126,7 +126,18 @@
 			return nil, err
 		}
 	}
-	return p.createNotifications(messages)
+	notif, err := p.createNotifications(messages)
+	if err != nil {
+		return nil, err
+	}
+	return []*plugins.PushMessageBatch{
+		&plugins.PushMessageBatch{
+			Messages:        notif,
+			Limit:           individualNotificationsLimit,
+			OverflowHandler: p.handleOverflow,
+			Tag:             "gmail",
+		}}, nil
+
 }
 
 func (p *GmailPlugin) reported(id string) bool {
@@ -134,7 +145,7 @@
 	return ok
 }
 
-func (p *GmailPlugin) createNotifications(messages []message) ([]plugins.PushMessage, error) {
+func (p *GmailPlugin) createNotifications(messages []message) ([]*plugins.PushMessage, error) {
 	timestamp := time.Now()
 	pushMsgMap := make(pushes)
 
@@ -163,32 +174,30 @@
 			// fmt with label personal and threadId
 			action := fmt.Sprintf(gmailDispatchUrl, "personal", msg.ThreadId)
 			epoch := hdr.getEpoch()
-			pushMsgMap[msg.ThreadId] = *plugins.NewStandardPushMessage(summary, body, action, avatarPath, epoch)
+			pushMsgMap[msg.ThreadId] = plugins.NewStandardPushMessage(summary, body, action, avatarPath, epoch)
 		} else {
 			log.Print("gmail plugin ", p.accountId, ": skipping message id ", msg.Id, " with date ", msgStamp, " older than ", timeDelta)
 		}
 	}
-	var pushMsg []plugins.PushMessage
+	pushMsg := make([]*plugins.PushMessage, 0, len(pushMsgMap))
 	for _, v := range pushMsgMap {
 		pushMsg = append(pushMsg, v)
-		if len(pushMsg) == individualNotificationsLimit {
-			break
-		}
-	}
-	if len(pushMsgMap) > individualNotificationsLimit {
-		// TRANSLATORS: This represents a notification summary about more unread emails
-		summary := gettext.Gettext("More unread emails available")
-		// TODO it would probably be better to grab the estimate that google returns in the message list.
-		approxUnreadMessages := len(pushMsgMap) - individualNotificationsLimit
-		// TRANSLATORS: the first %d refers to approximate additionl email message count
-		body := fmt.Sprintf(gettext.Gettext("You have an approximate of %d additional unread messages"), approxUnreadMessages)
-		// fmt with label personal and no threadId
-		action := fmt.Sprintf(gmailDispatchUrl, "personal")
-		epoch := time.Now().Unix()
-		pushMsg = append(pushMsg, *plugins.NewStandardPushMessage(summary, body, action, "", epoch))
-	}
-
+	}
 	return pushMsg, nil
+
+}
+func (p *GmailPlugin) handleOverflow(pushMsg []*plugins.PushMessage) *plugins.PushMessage {
+	// TRANSLATORS: This represents a notification summary about more unread emails
+	summary := gettext.Gettext("More unread emails available")
+	// TODO it would probably be better to grab the estimate that google returns in the message list.
+	approxUnreadMessages := len(pushMsg)
+	// TRANSLATORS: the first %d refers to approximate additionl email message count
+	body := fmt.Sprintf(gettext.Gettext("You have an approximate of %d additional unread messages"), approxUnreadMessages)
+	// fmt with label personal and no threadId
+	action := fmt.Sprintf(gmailDispatchUrl, "personal")
+	epoch := time.Now().Unix()
+
+	return plugins.NewStandardPushMessage(summary, body, action, "", epoch)
 }
 
 func (p *GmailPlugin) parseMessageListResponse(resp *http.Response) ([]message, error) {

=== modified file 'plugins/plugins.go'
--- plugins/plugins.go	2014-08-27 23:08:59 +0000
+++ plugins/plugins.go	2014-12-09 15:20:23 +0000
@@ -44,7 +44,7 @@
 // ApplicationId returns the APP_ID of the delivery target for Post Office.
 type Plugin interface {
 	ApplicationId() ApplicationId
-	Poll(*accounts.AuthData) ([]PushMessage, error)
+	Poll(*accounts.AuthData) ([]*PushMessageBatch, error)
 }
 
 // AuthTokens is a map with tokens the plugins are to use to make requests.
@@ -76,6 +76,21 @@
 	return pm
 }
 
+// PushMessageBatch represents a logical grouping of PushMessages that
+// have a limit on the number of their notifications that want to be
+// presented to the user at the same time, and a way to handle the
+// overflow. All Notifications that are part of a Batch share the same
+// tag (Tag).  ${Tag}-overflow is the overflow notification tag.
+//
+// TODO: support notifications sharing just the prefix (so the app can
+// tell them apart by tag).
+type PushMessageBatch struct {
+	Messages        []*PushMessage
+	Limit           int
+	OverflowHandler func([]*PushMessage) *PushMessage
+	Tag             string
+}
+
 // PushMessage represents a data structure to be sent over to the
 // Post Office. It consists of a Notification and a Message.
 type PushMessage struct {

=== modified file 'plugins/twitter/twitter.go'
--- plugins/twitter/twitter.go	2014-08-05 20:32:03 +0000
+++ plugins/twitter/twitter.go	2014-12-09 15:20:23 +0000
@@ -78,7 +78,7 @@
 	return client.Get(http.DefaultClient, token, u.String(), query)
 }
 
-func (p *twitterPlugin) parseStatuses(resp *http.Response) ([]plugins.PushMessage, error) {
+func (p *twitterPlugin) parseStatuses(resp *http.Response) (*plugins.PushMessageBatch, error) {
 	defer resp.Body.Close()
 	decoder := json.NewDecoder(resp.Body)
 
@@ -107,35 +107,38 @@
 	}
 	p.lastMentionId = statuses[0].Id
 
-	pushMsg := []plugins.PushMessage{}
-	for _, s := range statuses {
+	pushMsg := make([]*plugins.PushMessage, len(statuses))
+	for i, s := range statuses {
 		// TRANSLATORS: The first %s refers to the twitter user's Name, the second %s to the username.
 		summary := fmt.Sprintf(gettext.Gettext("%s. @%s"), s.User.Name, s.User.ScreenName)
 		action := fmt.Sprintf("%s/%s/statuses/%d", twitterDispatchUrlBase, s.User.ScreenName, s.Id)
 		epoch := toEpoch(s.CreatedAt)
-		pushMsg = append(pushMsg, *plugins.NewStandardPushMessage(summary, s.Text, action, s.User.Image, epoch))
-		if len(pushMsg) == maxIndividualStatuses {
-			break
-		}
-	}
-	// Now we consolidate the remaining statuses
-	if len(statuses) > len(pushMsg) {
-		var screennames []string
-		for _, s := range statuses[consolidatedStatusIndexStart:] {
-			screennames = append(screennames, "@"+s.User.ScreenName)
-		}
-		// TRANSLATORS: This represents a notification summary about more twitter mentions available
-		summary := gettext.Gettext("Multiple more mentions")
-		// TRANSLATORS: This represents a notification body with the comma separated twitter usernames
-		body := fmt.Sprintf(gettext.Gettext("From %s"), strings.Join(screennames, ", "))
-		action := fmt.Sprintf("%s/i/connect", twitterDispatchUrlBase)
-		epoch := time.Now().Unix()
-		pushMsg = append(pushMsg, *plugins.NewStandardPushMessage(summary, body, action, "", epoch))
-	}
-	return pushMsg, nil
-}
-
-func (p *twitterPlugin) parseDirectMessages(resp *http.Response) ([]plugins.PushMessage, error) {
+		pushMsg[i] = plugins.NewStandardPushMessage(summary, s.Text, action, s.User.Image, epoch)
+	}
+	return &plugins.PushMessageBatch{
+		Messages:        pushMsg,
+		Limit:           maxIndividualStatuses,
+		OverflowHandler: p.consolidateStatuses,
+		Tag:             "status",
+	}, nil
+}
+
+func (p *twitterPlugin) consolidateStatuses(pushMsg []*plugins.PushMessage) *plugins.PushMessage {
+	screennames := make([]string, len(pushMsg))
+	for i, m := range pushMsg {
+		screennames[i] = m.Notification.Card.Summary
+	}
+	// TRANSLATORS: This represents a notification summary about more twitter mentions available
+	summary := gettext.Gettext("Multiple more mentions")
+	// TRANSLATORS: This represents a notification body with the comma separated twitter usernames
+	body := fmt.Sprintf(gettext.Gettext("From %s"), strings.Join(screennames, ", "))
+	action := fmt.Sprintf("%s/i/connect", twitterDispatchUrlBase)
+	epoch := time.Now().Unix()
+
+	return plugins.NewStandardPushMessage(summary, body, action, "", epoch)
+}
+
+func (p *twitterPlugin) parseDirectMessages(resp *http.Response) (*plugins.PushMessageBatch, error) {
 	defer resp.Body.Close()
 	decoder := json.NewDecoder(resp.Body)
 
@@ -164,35 +167,39 @@
 	}
 	p.lastDirectMessageId = dms[0].Id
 
-	pushMsg := []plugins.PushMessage{}
-	for _, m := range dms {
+	pushMsg := make([]*plugins.PushMessage, len(dms))
+	for i, m := range dms {
 		// TRANSLATORS: The first %s refers to the twitter user's Name, the second %s to the username.
 		summary := fmt.Sprintf(gettext.Gettext("%s. @%s"), m.Sender.Name, m.Sender.ScreenName)
 		action := fmt.Sprintf("%s/%s/messages", twitterDispatchUrlBase, m.Sender.ScreenName)
 		epoch := toEpoch(m.CreatedAt)
-		pushMsg = append(pushMsg, *plugins.NewStandardPushMessage(summary, m.Text, action, m.Sender.Image, epoch))
-		if len(pushMsg) == maxIndividualDirectMessages {
-			break
-		}
-	}
-	// Now we consolidate the remaining messages
-	if len(dms) > len(pushMsg) {
-		var senders []string
-		for _, m := range dms[consolidatedDirectMessageIndexStart:] {
-			senders = append(senders, "@"+m.Sender.ScreenName)
-		}
-		// TRANSLATORS: This represents a notification summary about more twitter direct messages available
-		summary := gettext.Gettext("Multiple direct messages available")
-		// TRANSLATORS: This represents a notification body with the comma separated twitter usernames
-		body := fmt.Sprintf(gettext.Gettext("From %s"), strings.Join(senders, ", "))
-		action := fmt.Sprintf("%s/messages", twitterDispatchUrlBase)
-		epoch := time.Now().Unix()
-		pushMsg = append(pushMsg, *plugins.NewStandardPushMessage(summary, body, action, "", epoch))
-	}
-	return pushMsg, nil
-}
-
-func (p *twitterPlugin) Poll(authData *accounts.AuthData) (messages []plugins.PushMessage, err error) {
+		pushMsg[i] = plugins.NewStandardPushMessage(summary, m.Text, action, m.Sender.Image, epoch)
+	}
+
+	return &plugins.PushMessageBatch{
+		Messages:        pushMsg,
+		Limit:           maxIndividualDirectMessages,
+		OverflowHandler: p.consolidateDirectMessages,
+		Tag:             "direct-message",
+	}, nil
+}
+
+func (p *twitterPlugin) consolidateDirectMessages(pushMsg []*plugins.PushMessage) *plugins.PushMessage {
+	senders := make([]string, len(pushMsg))
+	for i, m := range pushMsg {
+		senders[i] = m.Notification.Card.Summary
+	}
+	// TRANSLATORS: This represents a notification summary about more twitter direct messages available
+	summary := gettext.Gettext("Multiple direct messages available")
+	// TRANSLATORS: This represents a notification body with the comma separated twitter usernames
+	body := fmt.Sprintf(gettext.Gettext("From %s"), strings.Join(senders, ", "))
+	action := fmt.Sprintf("%s/messages", twitterDispatchUrlBase)
+	epoch := time.Now().Unix()
+
+	return plugins.NewStandardPushMessage(summary, body, action, "", epoch)
+}
+
+func (p *twitterPlugin) Poll(authData *accounts.AuthData) (batches []*plugins.PushMessageBatch, err error) {
 	url := "statuses/mentions_timeline.json"
 	if p.lastMentionId > 0 {
 		url = fmt.Sprintf("%s?since_id=%d", url, p.lastMentionId)
@@ -201,7 +208,7 @@
 	if err != nil {
 		return
 	}
-	messages, err = p.parseStatuses(resp)
+	statuses, err := p.parseStatuses(resp)
 	if err != nil {
 		return
 	}
@@ -222,7 +229,12 @@
 		p.bootstrap = true
 		return nil, nil
 	}
-	messages = append(messages, dms...)
+	if statuses != nil && len(statuses.Messages) > 0 {
+		batches = append(batches, statuses)
+	}
+	if dms != nil && len(dms.Messages) > 0 {
+		batches = append(batches, dms)
+	}
 	return
 }
 

=== modified file 'plugins/twitter/twitter_test.go'
--- plugins/twitter/twitter_test.go	2014-08-05 20:32:03 +0000
+++ plugins/twitter/twitter_test.go	2014-12-09 15:20:23 +0000
@@ -411,8 +411,10 @@
 		Body:       closeWrapper{bytes.NewReader([]byte(statusesBody))},
 	}
 	p := &twitterPlugin{}
-	messages, err := p.parseStatuses(resp)
+	batch, err := p.parseStatuses(resp)
 	c.Assert(err, IsNil)
+	c.Assert(batch, NotNil)
+	messages := batch.Messages
 	c.Assert(len(messages), Equals, 2)
 	c.Check(messages[0].Notification.Card.Summary, Equals, "Andrew Spode Miller. @spode")
 	c.Check(messages[0].Notification.Card.Body, Equals, "@jasoncosta @themattharris Hey! Going to be in Frisco in October. Was hoping to have a meeting to talk about @thinkwall if you're around?")
@@ -459,8 +461,10 @@
 		Body:       closeWrapper{bytes.NewReader([]byte(directMessagesBody))},
 	}
 	p := &twitterPlugin{}
-	messages, err := p.parseDirectMessages(resp)
+	batch, err := p.parseDirectMessages(resp)
 	c.Assert(err, IsNil)
+	c.Assert(batch, NotNil)
+	messages := batch.Messages
 	c.Assert(len(messages), Equals, 1)
 	c.Check(messages[0].Notification.Card.Summary, Equals, "Sean Cook. @theSeanCook")
 	c.Check(messages[0].Notification.Card.Body, Equals, "booyakasha")