Merge lp:~mardy/account-polld/dekko-gmail into lp:~ubuntu-push-hackers/account-polld/trunk

[Alberto Mardegan]

Add Dekko GMail plugin

Add Dekko GMail plugin

[system-apps-ci-bot (system-apps-ci-bot)]

PASSED: Continuous integration, rev:170
https://jenkins.canonical.com/system-apps/job/lp-account-polld-ci/15/
Executed test runs:
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build/1032
    SUCCESS: https://jenkins.canonical.com/system-apps/job/test-0-autopkgtest/label=phone-armhf,release=vivid+overlay,testname=default/200
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-0-fetch/1032
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-1-sourcepkg/release=vivid+overlay/929
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-1-sourcepkg/release=xenial+overlay/929
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-1-sourcepkg/release=yakkety/929
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=amd64,release=vivid+overlay/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=amd64,release=vivid+overlay/923/artifact/output/*zip*/output.zip
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=amd64,release=xenial+overlay/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=amd64,release=xenial+overlay/923/artifact/output/*zip*/output.zip
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=amd64,release=yakkety/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=amd64,release=yakkety/923/artifact/output/*zip*/output.zip
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=armhf,release=vivid+overlay/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=armhf,release=vivid+overlay/923/artifact/output/*zip*/output.zip
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=armhf,release=xenial+overlay/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=armhf,release=xenial+overlay/923/artifact/output/*zip*/output.zip
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=armhf,release=yakkety/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=armhf,release=yakkety/923/artifact/output/*zip*/output.zip
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=i386,release=vivid+overlay/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=i386,release=vivid+overlay/923/artifact/output/*zip*/output.zip
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=i386,release=xenial+overlay/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=i386,release=xenial+overlay/923/artifact/output/*zip*/output.zip
    SUCCESS: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=i386,release=yakkety/923
        deb: https://jenkins.canonical.com/system-apps/job/build-2-binpkg/arch=i386,release=yakkety/923/artifact/output/*zip*/output.zip

Click here to trigger a rebuild:
https://jenkins.canonical.com/system-apps/job/lp-account-polld-ci/15/rebuild

review: Approve (continuous-integration)

[Jonas G. Drange (jonas-drange)]

Everything looks okay, except for the duplication of some code from the gmail plugin. Have you looked at the possibility of sharing some of the code?

review: Needs Information

[Alberto Mardegan (mardy)]

> Everything looks okay, except for the duplication of some code from the gmail
> plugin. Have you looked at the possibility of sharing some of the code?

I haven't. :-) The reason is that the plan is to extend account-polld to support out-of-process plugins, after which we could reuse Dekko's own IMAP/POP3 client implementation and drop this plugin altogether.
You can read more about this in bug 1421923, from comment #18 on.

[Jonas G. Drange (jonas-drange)]

Then let's not invest too much time and energy in introducing some sort of code sharing between plugins—that may benefit from being completely separate packages anyway.

review: Approve

=== modified file 'cmd/account-polld/main.go'
--- cmd/account-polld/main.go	2016-07-22 11:44:01 +0000
+++ cmd/account-polld/main.go	2016-07-22 11:44:01 +0000
@@ -27,6 +27,7 @@
 	"launchpad.net/account-polld/accounts"
 	"launchpad.net/account-polld/gettext"
 	"launchpad.net/account-polld/plugins"
+	"launchpad.net/account-polld/plugins/dekko"
 	"launchpad.net/account-polld/plugins/gcalendar"
 	"launchpad.net/account-polld/plugins/gmail"
 	"launchpad.net/account-polld/plugins/twitter"
@@ -48,6 +49,7 @@
 /* Use identifiers and API keys provided by the respective webapps which are the official
    end points for the notifications */
 const (
+	SERVICENAME_DEKKO     = "dekko.dekkoproject_dekko"
 	SERVICENAME_GMAIL     = "com.ubuntu.developer.webapps.webapp-gmail_webapp-gmail"
 	SERVICENAME_TWITTER   = "com.ubuntu.developer.webapps.webapp-twitter_webapp-twitter"
 	SERVICENAME_GCALENDAR = "google-caldav"
@@ -99,6 +101,7 @@
 
 func monitorAccounts(postWatch chan *PostWatch, pollBus *pollbus.PollBus) {
 	watcher := accounts.NewWatcher()
+	watcher.AddService(SERVICENAME_DEKKO)
 	watcher.AddService(SERVICENAME_GMAIL)
 	watcher.AddService(SERVICENAME_GCALENDAR)
 	watcher.AddService(SERVICENAME_TWITTER)
@@ -131,6 +134,9 @@
 			var plugin plugins.Plugin
 			log.Println("Creating plugin for service: ", data.ServiceName)
 			switch data.ServiceName {
+			case SERVICENAME_DEKKO:
+				log.Println("Creating account with id", data.AccountId, "for", data.ServiceName)
+				plugin = dekko.New(data.AccountId)
 			case SERVICENAME_GMAIL:
 				log.Println("Creating account with id", data.AccountId, "for", data.ServiceName)
 				plugin = gmail.New(data.AccountId)

=== added directory 'plugins/dekko'
=== added file 'plugins/dekko/api.go'
--- plugins/dekko/api.go	1970-01-01 00:00:00 +0000
+++ plugins/dekko/api.go	2016-07-22 11:44:01 +0000
@@ -0,0 +1,127 @@
+/*
+ Copyright 2014 Canonical Ltd.
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License version 3, as published
+ by the Free Software Foundation.
+
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranties of
+ MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+ PURPOSE.  See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dekko
+
+import (
+	"fmt"
+	"time"
+
+	"launchpad.net/account-polld/plugins"
+)
+
+const gmailTime = "Mon, 2 Jan 2006 15:04:05 -0700"
+
+type pushes map[string]*plugins.PushMessage
+type headers map[string]string
+
+// messageList holds a response to call to Users.messages: list
+// defined in https://developers.google.com/gmail/api/v1/reference/users/messages/list
+type messageList struct {
+	// Messages holds a list of message.
+	Messages []message `json:"messages"`
+	// NextPageToken is used to retrieve the next page of results in the list.
+	NextPageToken string `json:"nextPageToken"`
+	// ResultSizeEstimage is the estimated total number of results.
+	ResultSizeEstimage uint64 `json:"resultSizeEstimate"`
+}
+
+// message holds a partial response for a Users.messages.
+// The full definition of a message is defined in
+// https://developers.google.com/gmail/api/v1/reference/users/messages#resource
+type message struct {
+	// Id is the immutable ID of the message.
+	Id string `json:"id"`
+	// ThreadId is the ID of the thread the message belongs to.
+	ThreadId string `json:"threadId"`
+	// HistoryId is the ID of the last history record that modified
+	// this message.
+	HistoryId string `json:"historyId"`
+	// Snippet is a short part of the message text. This text is
+	// used for the push message summary.
+	Snippet string `json:"snippet"`
+	// Payload represents the message payload.
+	Payload payload `json:"payload"`
+}
+
+func (m message) String() string {
+	return fmt.Sprintf("Id: %d, snippet: '%s'\n", m.Id, m.Snippet[:10])
+}
+
+// ById implements sort.Interface for []message based on
+// the Id field.
+type byId []message
+
+func (m byId) Len() int           { return len(m) }
+func (m byId) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
+func (m byId) Less(i, j int) bool { return m[i].Id < m[j].Id }
+
+// payload represents the message payload.
+type payload struct {
+	Headers []messageHeader `json:"headers"`
+}
+
+func (p *payload) mapHeaders() headers {
+	headers := make(map[string]string)
+	for _, hdr := range p.Headers {
+		headers[hdr.Name] = hdr.Value
+	}
+	return headers
+}
+
+func (hdr headers) getTimestamp() time.Time {
+	timestamp, ok := hdr[hdrDATE]
+	if !ok {
+		return time.Now()
+	}
+
+	if t, err := time.Parse(gmailTime, timestamp); err == nil {
+		return t
+	}
+	return time.Now()
+}
+
+func (hdr headers) getEpoch() int64 {
+	return hdr.getTimestamp().Unix()
+}
+
+// messageHeader represents the message headers.
+type messageHeader struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+type errorResp struct {
+	Err struct {
+		Code    uint64 `json:"code"`
+		Message string `json:"message"`
+		Errors  []struct {
+			Domain  string `json:"domain"`
+			Reason  string `json:"reason"`
+			Message string `json:"message"`
+		} `json:"errors"`
+	} `json:"error"`
+}
+
+func (err *errorResp) Error() string {
+	return fmt.Sprint("backend response:", err.Err.Message)
+}
+
+const (
+	hdrDATE    = "Date"
+	hdrSUBJECT = "Subject"
+	hdrFROM    = "From"
+)

=== added file 'plugins/dekko/dekko.go'
--- plugins/dekko/dekko.go	1970-01-01 00:00:00 +0000
+++ plugins/dekko/dekko.go	2016-07-22 11:44:01 +0000
@@ -0,0 +1,346 @@
+/*
+ Copyright 2014 Canonical Ltd.
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License version 3, as published
+ by the Free Software Foundation.
+
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranties of
+ MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+ PURPOSE.  See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dekko
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/mail"
+	"net/url"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"log"
+
+	"launchpad.net/account-polld/accounts"
+	"launchpad.net/account-polld/gettext"
+	"launchpad.net/account-polld/plugins"
+	"launchpad.net/account-polld/qtcontact"
+)
+
+const (
+	APP_ID           = "dekko.dekkoproject_dekko"
+	dekkoDispatchUrl = "dekko://notify/%d/%s/%s"
+	// If there's more than 10 emails in one batch, we don't show 10 notification
+	// bubbles, but instead show one summary. We always show all notifications in the
+	// indicator.
+	individualNotificationsLimit = 10
+	pluginName                   = "dekko"
+)
+
+type reportedIdMap map[string]time.Time
+
+var baseUrl, _ = url.Parse("https://www.googleapis.com/gmail/v1/users/me/")
+
+// timeDelta defines how old messages can be to be reported.
+var timeDelta = time.Duration(time.Hour * 24)
+
+// trackDelta defines how old messages can be before removed from tracking
+var trackDelta = time.Duration(time.Hour * 24 * 7)
+
+// relativeTimeDelta is the same as timeDelta
+var relativeTimeDelta string = "1d"
+
+// regexp for identifying non-ascii characters
+var nonAsciiChars, _ = regexp.Compile("[^\x00-\x7F]")
+
+type GmailPlugin struct {
+	// reportedIds holds the messages that have already been notified. This
+	// approach is taken against timestamps as it avoids needing to call
+	// get on the message.
+	reportedIds reportedIdMap
+	accountId   uint
+}
+
+func idsFromPersist(accountId uint) (ids reportedIdMap, err error) {
+	err = plugins.FromPersist(pluginName, accountId, &ids)
+	if err != nil {
+		return nil, err
+	}
+	// discard old ids
+	timestamp := time.Now()
+	for k, v := range ids {
+		delta := timestamp.Sub(v)
+		if delta > trackDelta {
+			log.Print("gmail plugin ", accountId, ": deleting ", k, " as ", delta, " is greater than ", trackDelta)
+			delete(ids, k)
+		}
+	}
+	return ids, nil
+}
+
+func (ids reportedIdMap) persist(accountId uint) (err error) {
+	err = plugins.Persist(pluginName, accountId, ids)
+	if err != nil {
+		log.Print("gmail plugin ", accountId, ": failed to save state: ", err)
+	}
+	return nil
+}
+
+func New(accountId uint) *GmailPlugin {
+	reportedIds, err := idsFromPersist(accountId)
+	if err != nil {
+		log.Print("gmail plugin ", accountId, ": cannot load previous state from storage: ", err)
+	} else {
+		log.Print("gmail plugin ", accountId, ": last state loaded from storage")
+	}
+	return &GmailPlugin{reportedIds: reportedIds, accountId: accountId}
+}
+
+func (p *GmailPlugin) ApplicationId() plugins.ApplicationId {
+	return plugins.ApplicationId(APP_ID)
+}
+
+func (p *GmailPlugin) Poll(authData *accounts.AuthData) ([]*plugins.PushMessageBatch, error) {
+	// This envvar check is to ease testing.
+	if token := os.Getenv("ACCOUNT_POLLD_TOKEN_GMAIL"); token != "" {
+		authData.AccessToken = token
+	}
+
+	resp, err := p.requestMessageList(authData.AccessToken)
+	if err != nil {
+		return nil, err
+	}
+	messages, err := p.parseMessageListResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO use the batching API defined in https://developers.google.com/gmail/api/guides/batch
+	for i := range messages {
+		resp, err := p.requestMessage(messages[i].Id, authData.AccessToken)
+		if err != nil {
+			return nil, err
+		}
+		messages[i], err = p.parseMessageResponse(resp)
+		if err != nil {
+			return nil, err
+		}
+	}
+	notif, err := p.createNotifications(messages)
+	if err != nil {
+		return nil, err
+	}
+	return []*plugins.PushMessageBatch{
+		&plugins.PushMessageBatch{
+			Messages:        notif,
+			Limit:           individualNotificationsLimit,
+			OverflowHandler: p.handleOverflow,
+			Tag:             "dekko",
+		}}, nil
+
+}
+
+func (p *GmailPlugin) reported(id string) bool {
+	_, ok := p.reportedIds[id]
+	return ok
+}
+
+func (p *GmailPlugin) createNotifications(messages []message) ([]*plugins.PushMessage, error) {
+	timestamp := time.Now()
+	pushMsgMap := make(pushes)
+
+	for _, msg := range messages {
+		hdr := msg.Payload.mapHeaders()
+
+		from := hdr[hdrFROM]
+		var avatarPath string
+
+		emailAddress, err := mail.ParseAddress(from)
+		if err != nil {
+			// If the email address contains non-ascii characters, we get an
+			// error so we're going to try again, this time mangling the name
+			// by removing all non-ascii characters. We only care about the email
+			// address here anyway.
+			// XXX: We can't check the error message due to [1]: the error
+			// message is different in go < 1.3 and > 1.5.
+			// [1] https://github.com/golang/go/issues/12492
+			mangledAddr := nonAsciiChars.ReplaceAllString(from, "")
+			mangledEmail, mangledParseError := mail.ParseAddress(mangledAddr)
+			if mangledParseError == nil {
+				emailAddress = mangledEmail
+			}
+		} else if emailAddress.Name != "" {
+			// We only want the Name if the first ParseAddress
+			// call was successful. I.e. we do not want the name
+			// from a mangled email address.
+			from = emailAddress.Name
+		}
+
+		if emailAddress != nil {
+			avatarPath = qtcontact.GetAvatar(emailAddress.Address)
+			// If icon path starts with a path separator, assume local file path,
+			// encode it and prepend file scheme defined in RFC 1738.
+			if strings.HasPrefix(avatarPath, string(os.PathSeparator)) {
+				avatarPath = url.QueryEscape(avatarPath)
+				avatarPath = "file://" + avatarPath
+			}
+		}
+
+		msgStamp := hdr.getTimestamp()
+
+		if _, ok := pushMsgMap[msg.ThreadId]; ok {
+			// TRANSLATORS: the %s is an appended "from" corresponding to an specific email thread
+			pushMsgMap[msg.ThreadId].Notification.Card.Summary += fmt.Sprintf(gettext.Gettext(", %s"), from)
+		} else if timestamp.Sub(msgStamp) < timeDelta {
+			// TRANSLATORS: the %s is the "from" header corresponding to a specific email
+			summary := fmt.Sprintf(gettext.Gettext("%s"), from)
+			// TRANSLATORS: the first %s refers to the email "subject", the second %s refers "from"
+			body := fmt.Sprintf(gettext.Gettext("%s\n%s"), hdr[hdrSUBJECT], msg.Snippet)
+			// fmt with label personal and threadId
+			action := fmt.Sprintf(dekkoDispatchUrl, p.accountId, "INBOX", msg.Id)
+			epoch := hdr.getEpoch()
+			pushMsgMap[msg.ThreadId] = plugins.NewStandardPushMessage(summary, body, action, avatarPath, epoch)
+		} else {
+			log.Print("gmail plugin ", p.accountId, ": skipping message id ", msg.Id, " with date ", msgStamp, " older than ", timeDelta)
+		}
+	}
+	pushMsg := make([]*plugins.PushMessage, 0, len(pushMsgMap))
+	for _, v := range pushMsgMap {
+		pushMsg = append(pushMsg, v)
+	}
+	return pushMsg, nil
+
+}
+func (p *GmailPlugin) handleOverflow(pushMsg []*plugins.PushMessage) *plugins.PushMessage {
+	// TODO it would probably be better to grab the estimate that google returns in the message list.
+	approxUnreadMessages := len(pushMsg)
+
+	// TRANSLATORS: the %d refers to the number of new email messages.
+	summary := fmt.Sprintf(gettext.Gettext("You have %d new messages"), approxUnreadMessages)
+
+	body := ""
+
+	// fmt with label personal and no threadId
+	action := fmt.Sprintf(dekkoDispatchUrl, p.accountId, "INBOX")
+	epoch := time.Now().Unix()
+
+	return plugins.NewStandardPushMessage(summary, body, action, "", epoch)
+}
+
+func (p *GmailPlugin) parseMessageListResponse(resp *http.Response) ([]message, error) {
+	defer resp.Body.Close()
+	decoder := json.NewDecoder(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		var errResp errorResp
+		if err := decoder.Decode(&errResp); err != nil {
+			return nil, err
+		}
+		if errResp.Err.Code == 401 {
+			return nil, plugins.ErrTokenExpired
+		}
+		return nil, &errResp
+	}
+
+	var messages messageList
+	if err := decoder.Decode(&messages); err != nil {
+		return nil, err
+	}
+
+	filteredMsg := p.messageListFilter(messages.Messages)
+
+	return filteredMsg, nil
+}
+
+// messageListFilter returns a subset of unread messages where the subset
+// depends on not being in reportedIds. Before returning, reportedIds is
+// updated with the new list of unread messages.
+func (p *GmailPlugin) messageListFilter(messages []message) []message {
+	sort.Sort(byId(messages))
+	var reportMsg []message
+	var ids = make(reportedIdMap)
+
+	for _, msg := range messages {
+		if !p.reported(msg.Id) {
+			reportMsg = append(reportMsg, msg)
+		}
+		ids[msg.Id] = time.Now()
+	}
+	p.reportedIds = ids
+	p.reportedIds.persist(p.accountId)
+	return reportMsg
+}
+
+func (p *GmailPlugin) parseMessageResponse(resp *http.Response) (message, error) {
+	defer resp.Body.Close()
+	decoder := json.NewDecoder(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		var errResp errorResp
+		if err := decoder.Decode(&errResp); err != nil {
+			return message{}, err
+		}
+		return message{}, &errResp
+	}
+
+	var msg message
+	if err := decoder.Decode(&msg); err != nil {
+		return message{}, err
+	}
+
+	return msg, nil
+}
+
+func (p *GmailPlugin) requestMessage(id, accessToken string) (*http.Response, error) {
+	u, err := baseUrl.Parse("messages/" + id)
+	if err != nil {
+		return nil, err
+	}
+
+	query := u.Query()
+	// only request specific fields
+	query.Add("fields", "snippet,threadId,id,payload/headers")
+	// get the full message to get From and Subject from headers
+	query.Add("format", "full")
+	u.RawQuery = query.Encode()
+
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+accessToken)
+
+	return http.DefaultClient.Do(req)
+}
+
+func (p *GmailPlugin) requestMessageList(accessToken string) (*http.Response, error) {
+	u, err := baseUrl.Parse("messages")
+	if err != nil {
+		return nil, err
+	}
+
+	query := u.Query()
+
+	// get all unread inbox emails received after
+	// the last time we checked. If this is the first
+	// time we check, get unread emails after timeDelta
+	query.Add("q", fmt.Sprintf("is:unread in:inbox newer_than:%s", relativeTimeDelta))
+	u.RawQuery = query.Encode()
+
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+accessToken)
+
+	return http.DefaultClient.Do(req)
+}