Merge lp:~ursinha/shadow-database/add-milestone-distro-distroseries into lp:shadow-database

[Ursula Junque]

Implements add_milestone, add_distro, add_distroseries and also handles when an user is gone or merged in launchpad, that was causing the script to fail and abort.

=== modified file 'ShadowDatabase/lp.py'
--- ShadowDatabase/lp.py	2013-01-16 21:43:19 +0000
+++ ShadowDatabase/lp.py	2013-01-16 21:43:19 +0000
@@ -77,11 +77,21 @@
                 try:
                     search_json = self.get_json_from_link(search_link)
                     break
-                except (NotFound, HTTPError, ClientError, ServerError):
+                except ClientError, e:
+                    response_status = e.response["status"]
+                    if response_status == '410':
+                        self.logger.error("Object %s is gone." % search_link)
+                        raise
+                    if response_status == '404':
+                        self.logger.error("Object %s not found." % search_link)
+                        raise
+                    continue
+                except (NotFound, HTTPError, ServerError), e:
                     continue
             pass
         except Exception, e:
             self.logger.error("Unexpected exception: %s" % str(e))
+            self.logger.exception(e)
             pass
 
         return search_json

=== modified file 'ShadowDatabase/models.py'
--- ShadowDatabase/models.py	2013-01-16 21:43:19 +0000
+++ ShadowDatabase/models.py	2013-01-16 21:43:19 +0000
@@ -8,6 +8,7 @@
 from storm.locals import Int
 from storm.expr import SQL
 from datetime import datetime
+from pytz import UTC
 import dateutil.parser
 
 
@@ -29,6 +30,7 @@
     series_target = IntCol()
     target = IntCol()
     release = IntCol()
+    self_link = StringCol()
 
 
 class Person(object):
@@ -46,6 +48,29 @@
     private = BoolCol(notNull=True, default=False,)
     irc_nicknames = StringCol()
     is_team = BoolCol(notNull=True, default=False,)
+    is_gone = BoolCol(notNull=True, default=False,)
+
+    def __init__(self):
+        pass
+
+
+class MissingPerson(Person):
+
+    def __init__(self, person_link):
+        super(MissingPerson, self).__init__()
+        # The only thing we have is the name, because of the link.
+        self.name = self.display_name = "".join(person_link.split("~")[1:])
+        self.date_created = datetime.utcnow().replace(tzinfo=UTC)
+        self.hide_email_addresses = True
+        self.private = False
+        self.is_team = False
+        self.is_valid = False
+        if not self.is_merged:
+            self.is_gone = True
+
+    @property
+    def is_merged(self):
+        return "merged" in self.name
 
 
 class Bug(object):
@@ -125,6 +150,7 @@
     web_link = StringCol(notNull=True,)
     name = StringCol(notNull=True,)
     display_name = StringCol(notNull=True,)
+    target_type = StringCol(notNull=True,)
 
 
 class Bugtracker(object):
@@ -235,13 +261,13 @@
         self.date_left_closed = parseDate(json['date_left_closed'])
         self.date_left_new = parseDate(json['date_left_new'])
         self.date_triaged = parseDate(json['date_triaged'])
-        # self.milestone = 0  # FIXME
+        self.milestone = sql.add_milestone(json['milestone_link'])
         self.owner = sql.add_person(json['owner_link'])
         self.status = sql.lp.StatusMap[json['status']]
         self.importance = (sql.lp.ImportanceMap[json['importance']])
         self.target = sql.add_bug_target(json['target_link'])  # FIXME
         self.bug_target_name = json['bug_target_name']
-        # self.bug_watch = 0  # FIXME
+        # self.bug_watch = sql.add_bug_watch(json['bug_watch_link'])
 
     def update(self, db):
         # FIXME: there has to be a better way.
@@ -284,10 +310,10 @@
     display_name = StringCol(notNull=True,)
     summary = StringCol(notNull=True,)
     bug_supervisor = IntCol()
-    security_contact = IntCol()
     driver = IntCol()
     current_series = IntCol()
     date_created = UtcDateTimeCol(notNull=True,)
+    self_link = StringCol(notNull=True,)
 
 
 class Sourcepackage(object):
@@ -316,15 +342,15 @@
     sourcepackage = IntCol()
 
 
-class Distroseries(object):
+class DistroSeries(object):
     __storm_table__ = "distroseries"
     id = Int(primary=True,)
     owner = IntCol()
     registrant = IntCol()
     status = StringCol()
     name = StringCol()
-    display_name = StringCol()
-    full_series_name = StringCol()
+    displayname = StringCol()
+    fullseriesname = StringCol()
     summary = StringCol()
     supported = BoolCol(notNull=True, default=False,)
     title = StringCol()
@@ -333,7 +359,8 @@
     component_names = StringCol()
     distribution = IntCol()
     date_created = UtcDateTimeCol(notNull=True,)
-    date_released = UtcDateTimeCol()
+    datereleased = UtcDateTimeCol()
+    self_link = StringCol(notNull=True,)
 
 
 class Distrosourcepackage(object):

=== modified file 'ShadowDatabase/sql.py'
--- ShadowDatabase/sql.py	2013-01-16 21:43:19 +0000
+++ ShadowDatabase/sql.py	2013-01-16 21:43:19 +0000
@@ -15,6 +15,8 @@
 import json
 
 from datetime import datetime
+from lazr.restfulclient.errors import ClientError
+from pytz import UTC
 from ShadowDatabase.lp import LP
 from ShadowDatabase.models import *
 from storm.expr import *
@@ -154,6 +156,8 @@
         bug_target.web_link = bug_target_json['web_link']
         bug_target.name = bug_target_json['name']
         bug_target.display_name = bug_target_json['display_name']
+        target_type = bug_target_json['resource_type_link'].split("#")[1]
+        bug_target.target_type = target_type
 
         # Add the object.
         self.store.add(bug_target)
@@ -171,12 +175,142 @@
         ret_str = ",".join(list(set(ret)))
         return ",".join(ret)
 
+    def add_bug_watch(self, bug_watch_link):
+        # TODO: implement this
+        pass
+
+    def add_milestone(self, milestone_link):
+        if milestone_link is None:
+            return None
+
+        # Get milestone id and check if it exists:
+        result = self.store.find(Milestone,
+                                 Milestone.self_link == milestone_link).one()
+        if result:
+            return result.id
+
+        milestone_json = self.lp.check_and_fetch_json(milestone_link)
+        if milestone_json is None:
+            return 0
+
+        milestone = Milestone()
+        milestone.name = milestone_json["name"]
+        milestone.code_name = milestone_json["code_name"]
+        milestone.title = milestone_json["title"]
+        # milestone.release = self.add_release(milestone_json["release_link"])
+        milestone.is_active = milestone_json["is_active"]
+        milestone.summary = milestone_json["summary"]
+        milestone.target = self.add_distro(milestone_json["target_link"])
+        milestone.series_target = self.add_distroseries(
+            milestone_json["series_target_link"])
+        milestone.self_link = milestone_json["self_link"]
+        milestone.official_bug_tags = ",".join(
+            milestone_json["official_bug_tags"])
+        try:
+            date_targeted = milestone.date_targeted = self.parseDate(
+                milestone_json["date_targeted"])
+        except ValueError:
+            date_targeted = self.parseDate(
+                milestone_json["date_targeted"]).replace(tzinfo=UTC)
+        milestone.date_targeted = date_targeted
+
+        # Add the object.
+        self.store.add(milestone)
+        self.store.flush()
+        self.logger.debug("MILESTONE: %s ... added." % str(milestone.name))
+        return milestone.id
+
+    def add_distro(self, distro_link):
+        if distro_link is None:
+            return None
+
+        result = self.store.find(Distribution,
+                                 Distribution.self_link == distro_link).one()
+
+        if result:
+            return result.id
+
+        distro_json = self.lp.check_and_fetch_json(distro_link)
+        if distro_json is None:
+            return 0
+
+        distro = Distribution()
+        distro.name = distro_json["name"]
+        distro.title = distro_json["title"]
+        distro.description = distro_json["description"]
+        distro.owner = self.add_person(distro_json["owner_link"])
+        distro.display_name = distro_json["display_name"]
+        distro.summary = distro_json["summary"]
+        distro.bug_supervisor = self.add_person(
+            distro_json["bug_supervisor_link"])
+        distro.driver = self.add_person(distro_json["driver_link"])
+        distro.date_created = self.parseDate(distro_json["date_created"])
+        distro.self_link = distro_json["self_link"]
+
+        # Add the object.
+        self.store.add(distro)
+        self.store.flush()
+        # Have to add the distro after adding the distroseries to avoid
+        # circular dependency.
+        distro.current_series = self.add_distroseries(
+            distro_json["current_series_link"])
+        self.store.commit()
+
+        self.logger.debug("DISTRO: %s ... added." % str(distro.name))
+        return distro.id
+
+    def add_distroseries(self, distroseries_link):
+        if distroseries_link is None:
+            return None
+
+        result = self.store.find(
+            DistroSeries, DistroSeries.self_link == distroseries_link).one()
+
+        if result:
+            return result.id
+
+        distroseries_json = self.lp.check_and_fetch_json(distroseries_link)
+        if distroseries_json is None:
+            return 0
+
+        distroseries = DistroSeries()
+        distroseries.name = distroseries_json["name"]
+        distroseries.owner = self.add_person(distroseries_json["owner_link"])
+        distroseries.registrant = self.add_person(
+            distroseries_json["registrant_link"])
+        distroseries.status = distroseries_json["status"]
+        distroseries.displayname = distroseries_json["displayname"]
+        distroseries.fullseriesname = distroseries_json["fullseriesname"]
+        distroseries.summary = distroseries_json["summary"]
+        distroseries.supported = distroseries_json["supported"]
+        distroseries.title = distroseries_json["title"]
+        distroseries.version = distroseries_json["version"]
+        distroseries.suite_names = ",".join(distroseries_json["suite_names"])
+        distroseries.component_names = ",".join(
+            distroseries_json["component_names"])
+        distroseries.distribution = self.add_distro(
+            distroseries_json["distribution_link"])
+        distroseries.date_created = self.parseDate(
+            distroseries_json["date_created"])
+        distroseries.date_released = self.parseDate(
+            distroseries_json["datereleased"])
+        distroseries.self_link = distroseries_json["self_link"]
+
+        # Add the object.
+        self.store.add(distroseries)
+        self.store.flush()
+        self.logger.debug("DISTROSERIES: %s ... added." % str(
+            distroseries.name))
+        return distroseries.id
+
     def add_person(self, person_link):
         if person_link is None:
             return None
 
         # Quickly get the username by taking the end of the URL.
         person_name = person_link.split('~')[1]
+        person_is_gone = False
+        person_is_merged = False
         result = self.store.find(Person, Person.name == person_name).one()
         if result:
             # self.logger.debug("PERSON: %s ... skipping (exists)." %
@@ -186,26 +320,44 @@
             return result.id
 
         # if the person wasn't found lets add it
-        person_json = self.lp.check_and_fetch_json(person_link)
-        if person_json is None:
+        person_json = None
+        try:
+            person_json = self.lp.check_and_fetch_json(person_link)
+        except ClientError, e:
+            # If user is no longer available, was deactivated or deleted
+            # because it is spam, we have to add it anyway, marking on the
+            # table it's gone.
+            response_status = e.response["status"]
+            if response_status == '410':
+                # Oops, user is no more
+                person_is_gone = True
+            if response_status == '404':
+                # User not found (was merged or hidden -- spammer)
+                if "merged" in person_name:
+                    person_is_merged = True
+
+        if person_is_gone or person_is_merged:
+            person = MissingPerson(person_link)
+            self.logger.debug("PERSON: %s ... is gone." % str(person.name))
+        elif person_json is None:
             return 0
-
-        person = Person()
-        person.display_name = person_json['display_name']
-        if 'description' in person_json.keys():
-            person.description = person_json['description']
-        person.name = person_json['name']
-        person.date_created = self.parseDate(person_json['date_created'])
-        person.hide_email_addresses = person_json['hide_email_addresses']
-        person.preferred_mail_address_id = 0  # FIXME
-        person.is_valid = person_json['is_valid']
-        person.private = person_json['private']
-        person.irc_nicknames = self.parse_irc_nicknames_collection(
-            person_json['irc_nicknames_collection_link'])
-        person.is_team = person_json['is_team']
-        person.karma = person_json['karma']
-        if person.is_team:
-            person.owner = self.add_person(person_json['team_owner_link'])
+        else:
+            person = Person()
+            person.display_name = person_json['display_name']
+            if 'description' in person_json.keys():
+                person.description = person_json['description']
+            person.name = person_json['name']
+            person.date_created = self.parseDate(person_json['date_created'])
+            person.hide_email_addresses = person_json['hide_email_addresses']
+            person.preferred_mail_address_id = 0  # FIXME
+            person.is_valid = person_json['is_valid']
+            person.private = person_json['private']
+            person.irc_nicknames = self.parse_irc_nicknames_collection(
+                person_json['irc_nicknames_collection_link'])
+            person.is_team = person_json['is_team']
+            person.karma = person_json['karma']
+            if person.is_team:
+                person.owner = self.add_person(person_json['team_owner_link'])
 
         # Add the object.
         self.store.add(person)

=== modified file 'database/tables.sql'
--- database/tables.sql	2012-10-17 18:52:52 +0000
+++ database/tables.sql	2013-01-16 21:43:19 +0000
@@ -9,6 +9,7 @@
     series_target integer,
     target integer,
     release integer,
+    self_link text NOT NULL,
     PRIMARY KEY (id)
 );
 
@@ -24,6 +25,7 @@
     hide_email_addresses boolean DEFAULT false NOT NULL,
     preferred_mail_address integer,
     is_valid boolean,
+    is_gone boolean DEFAULT false NOT NULL,
     private boolean DEFAULT false NOT NULL,
     irc_nicknames text,
     is_team boolean DEFAULT false NOT NULL,
@@ -64,6 +66,7 @@
     web_link text NOT NULL,
     name text NOT NULL,
     display_name text NOT NULL,
+    target_type text NOT NULL,
     PRIMARY KEY (id)
 );
 
@@ -124,7 +127,7 @@
     web_link text,
     PRIMARY KEY (id),
     FOREIGN KEY (bug) REFERENCES bug(id),
-    FOREIGN KEY (message) REFERENCES bugmessage(id),
+    FOREIGN KEY (message) REFERENCES bugmessage(id)
 );
 
 CREATE TABLE bugstatus (
@@ -238,14 +241,13 @@
     display_name text NOT NULL,
     summary text NOT NULL,
     bug_supervisor integer,
-    security_contact integer,
     driver integer,
     current_series integer,
     date_created timestamp without time zone NOT NULL,
+    self_link text NOT NULL,
     PRIMARY KEY (id),
     FOREIGN KEY (owner) REFERENCES person(id),
     FOREIGN KEY (bug_supervisor) REFERENCES person(id),
-    FOREIGN KEY (security_contact) REFERENCES person(id),
     FOREIGN KEY (driver) REFERENCES person(id)
 );
 
@@ -285,8 +287,8 @@
     registrant integer,
     status text,
     name text,
-    display_name text,
-    full_series_name text,
+    displayname text,
+    fullseriesname text,
     summary text,
     supported boolean DEFAULT false NOT NULL,
     title text,
@@ -295,7 +297,8 @@
     component_names text,
     distribution integer,
     date_created timestamp without time zone NOT NULL,
-    date_released timestamp without time zone,
+    datereleased timestamp without time zone,
+    self_link text NOT NULL,
     PRIMARY KEY (id),
     FOREIGN KEY (distribution) REFERENCES distribution(id),
     FOREIGN KEY (owner) REFERENCES person(id),