Merge lp:~dooferlad/linaro-fetch-image/indexer-update-2012-10 into lp:linaro-fetch-image

[James Tunnicliffe]

Updates indexing to work with latest layout.
Adds android information to database, but doesn't build images yet.

[lp:~dooferlad/linaro-fetch-image/indexer-update-2012-10]

17. By James Tunnicliffe on 2012-10-10

Fixed that I had disabled some bits of the indexer while debugging.

18. By James Tunnicliffe on 2012-10-10

Index all Ubuntu releases
Store Ubuntu release name in column upstream_release (release tables only)
New column not used yet.

[Milo Casagrande (milo)]

Hi James,

changes look good to me.
+1

Note: there are some PEP8 warnings, should we fix them? I can manage to do that as maintenance task in case...

review: Approve

=== modified file 'linaro-image-indexer'
--- linaro-image-indexer	2012-03-06 13:37:37 +0000
+++ linaro-image-indexer	2012-10-10 14:13:21 +0000
@@ -27,13 +27,17 @@
 import bz2
 import linaro_fetch_image.fetch_image
 
-RELEASES_WWW_DOCUMENT_ROOT = "/srv/releases.linaro.org/www"
 RELEASE_URL = "http://releases.linaro.org/"
-OLD_RELEASES_WWW_DOCUMENT_ROOT = "/srv/releases.linaro.org/www/platform"
 OLD_RELEASE_URL = "http://releases.linaro.org/platform/"
-SNAPSHOTS_WWW_DOCUMENT_ROOT = "/srv/snapshots.linaro.org/www/"
 SNAPSHOTS_URL = "http://snapshots.linaro.org/"
 
+BASE = "/srv"
+BASE = "/home/dooferlad/dev/fetch_image/layout/srv"
+RELEASES_WWW_DOCUMENT_ROOT = os.path.join(BASE, "releases.linaro.org/www")
+SNAPSHOTS_WWW_DOCUMENT_ROOT = os.path.join(BASE, "snapshots.linaro.org/www")
+OLD_RELEASES_WWW_DOCUMENT_ROOT = os.path.join(BASE,
+                                   "releases.linaro.org/www/platform")
+
 class ServerIndexer():
     """Create a database of files on the linaro image servers for use by image
        creation tools."""
@@ -108,8 +112,7 @@
                                         relative_location, not_match):
                     not_match_ok = False
 
-                if( not (to_match_ok and not_match_ok)
-                   or not re.search("\.gz$", file)):
+                if(not (to_match_ok and not_match_ok)):
                     continue  # URL doesn't match the validator. Ignore.
 
                 logging.getLogger("linaro_fetch_image").info(url)
@@ -174,63 +177,115 @@
     logger.addHandler(ch)
 
     #linaro-n/ubuntu-desktop/11.09
-    crawler.add_directory_parse_list(OLD_RELEASES_WWW_DOCUMENT_ROOT,
-                                     OLD_RELEASE_URL,
-                                     ([], ["platform/", "old/", "hwpack",
-                                           "alpha", "beta", "final", "leb",
-                                           "leb", "release-candidate"]),
-                                     ["platform", "image", "build=final"],
-                                     "release_binaries",
-                                     ["", "image", "platform"])
+    crawler.add_directory_parse_list(
+        OLD_RELEASES_WWW_DOCUMENT_ROOT,
+        OLD_RELEASE_URL,
+        (["\.gz$"],
+         ["platform/", "old/", "hwpack", "alpha", "beta", "final", "leb",
+          "release-candidate"]),
+        ["platform", "image", "build=final", "upstream_release=unknown"],
+        "release_binaries",
+        ["", "image", "platform"])
 
     #linaro-n/hwpacks/11.09
-    crawler.add_directory_parse_list(OLD_RELEASES_WWW_DOCUMENT_ROOT,
-                                     OLD_RELEASE_URL,
-                                     (["/hwpacks/"],
-                                      ["alpha", "beta", "final", "leb",
-                                       "release-candidate"]),
-                                     ["platform", "hardware", "build=final"],
-                                     "release_hwpacks",
-                                     ["", "", "platform",
-                                      ("hardware", r"hwpack_linaro-(.*?)_")])
+    crawler.add_directory_parse_list(
+        OLD_RELEASES_WWW_DOCUMENT_ROOT,
+        OLD_RELEASE_URL,
+        (["/hwpacks/", "\.gz$"],
+         ["alpha", "beta", "final", "leb", "release-candidate"]),
+        ["platform", "hardware", "build=final", "upstream_release=unknown"],
+        "release_hwpacks",
+        ["", "", "platform",
+        ("hardware", r"hwpack_linaro-(.*?)_")])
 
     # 11.10/ubuntu/oneiric-images/ubuntu-desktop/
     # NOT images/...
-    crawler.add_directory_parse_list(RELEASES_WWW_DOCUMENT_ROOT,
-                                     RELEASE_URL,
-                                     (["\d+\.\d+", "ubuntu", "oneiric-images"],
-                                      ["latest/", "platform/", "old/",
-                                       "hwpack", "^images/"]),
-                                     ["platform", "image", "build=final"],
-                                     "release_binaries",
-                                     ["platform", "", "", "image"])
+    # 12.09/ubuntu/precise-images/ubuntu-desktop/
+    #   linaro-precise-ubuntu-desktop-20120923-436.tar.gz
+    crawler.add_directory_parse_list(
+        RELEASES_WWW_DOCUMENT_ROOT,
+        RELEASE_URL,
+        (["\d+\.\d+", "ubuntu", "\w+-images", "\.gz$"],
+         ["latest/", "platform/", "old/", "hwpack", "^images/"]),
+        ["platform", "image", "build=final", "upstream_release"],
+        "release_binaries",
+        ["platform", "", ("upstream_release", "(\w+)-images"), "image"])
 
     # 11.10/ubuntu/oneiric-hwpacks/
-    crawler.add_directory_parse_list(RELEASES_WWW_DOCUMENT_ROOT,
-                                     RELEASE_URL,
-                                     (["\d+\.\d+", "ubuntu", "oneiric-hwpacks"],
-                                      ["latest/", "platform/", "old/",
-                                       "^images/"]),
-                                     ["platform", "hardware", "build=final"],
-                                     "release_hwpacks",
-                                     ["platform", "", "",
-                                      ("hardware", r"hwpack_linaro-(.*?)_")])
-
-    #oneiric/linaro-o-alip/20111026/0/images/tar/
-    crawler.add_directory_parse_list(SNAPSHOTS_WWW_DOCUMENT_ROOT,
-                                     SNAPSHOTS_URL,
-                                     (["^oneiric/"], ["/hwpack"]),
-                                     ["platform", "image", "date", "build"],
-                                     "snapshot_binaries",
-                                     ["platform", "image", "date", "build"])
-
-    #oneiric/lt-panda-oneiric/20111026/0/images/hwpack/
-    crawler.add_directory_parse_list(SNAPSHOTS_WWW_DOCUMENT_ROOT,
-                                     SNAPSHOTS_URL,
-                                     (["^oneiric/", "/hwpack"], []),
-                                     ["platform", "hardware", "date", "build"],
-                                     "snapshot_hwpacks",
-                                     ["platform", "hardware", "date", "build"])
+    # 12.09/ubuntu/precise-hwpacks
+    crawler.add_directory_parse_list(
+        RELEASES_WWW_DOCUMENT_ROOT,
+        RELEASE_URL,
+        (["\d+\.\d+", "ubuntu", "-hwpacks", "\.gz$"],
+         ["latest/", "platform/", "old/", "^images/"]),
+        ["platform", "hardware", "build=final", "upstream_release"],
+        "release_hwpacks",
+        ["platform", "", ("upstream_release", "(\w+)-hwpacks"),
+        ("hardware", r"hwpack_linaro-(.*?)_")])
+
+    # quantal/images/nano/127/linaro-quantal-nano-20120916-127.tar.gz
+    crawler.add_directory_parse_list(
+        SNAPSHOTS_WWW_DOCUMENT_ROOT,  # base_dir
+        SNAPSHOTS_URL,  # base_url
+        # url_validator (match, don't match)
+        (["^quantal/", "\.gz$"], ["/hwpack", "pre-built"]),
+        ["platform", "image", "date", "build"],  # db_columns
+        "snapshot_binaries",  # table
+        ["linaro-"
+         "(?P<platform>\w+)-"
+         "(?P<image>[\w-]+?)[-_]+"
+         "(?P<date>\d+)\D"
+         "(?P<build>\d+)"
+         ".*?\.tar\.gz"])  # url_chunks
+
+    # quantal/hwpacks/lt-panda/215/hwpack_linaro-lt-panda_20120927-215_armhf_supported.tar.gz
+    crawler.add_directory_parse_list(
+        SNAPSHOTS_WWW_DOCUMENT_ROOT,  # base_dir
+        SNAPSHOTS_URL,  # base_url
+        # url_validator (match, don't match)
+        (["^quantal/", "/hwpack", "\.gz$"], []),
+        ["platform=quantal", "hardware", "date", "build"],  # db_columns
+        "snapshot_hwpacks",  # table
+        ["hwpack_linaro-"
+         "(?P<hardware>.+?)_"
+         "(?P<date>.+?)-"
+         "(?P<build>\d+)_"
+         ".*\.tar\.gz"])  # url_chunks
+
+    # www/12.05/android/images/panda-ics-gcc47-tilt-tracking-blob/
+    #   system.tar.bz2
+    crawler.add_directory_parse_list(
+        RELEASES_WWW_DOCUMENT_ROOT,
+        RELEASE_URL,
+        # url_validator (match, don't match)
+        (["^\d+\.\d+/", "android", "images", "system.tar.bz2$"], []),
+        ["release", "hardware"],  # db_columns
+        "android_release",  # table
+        ["release", "", "", "hardware"])  # url_chunks
+
+    # vexpress-rtsm-mp-jb-gcc47-armlt-stable-open/12/target/product/
+    #   vexpress/system.tar.bz2
+    # NOT images/...
+    crawler.add_directory_parse_list(
+        RELEASES_WWW_DOCUMENT_ROOT,
+        RELEASE_URL,
+        # url_validator (match, don't match)
+        (["^/android/~linaro-android/", "system.tar.bz2$"], []),
+        ["build", "tag", "hardware"],  # db_columns
+        "android_snapshots",  # table
+        ["tag", "build", "", "", "hardware"])  # url_chunks
+
+    #linaro-n/android/11.04/panda/system.tar.bz2
+    crawler.add_directory_parse_list(
+        OLD_RELEASES_WWW_DOCUMENT_ROOT,
+        OLD_RELEASE_URL,
+        # url_validator (match, don't match)
+        (["system\.tar\.bz2$"],
+         ["platform/", "old/", "hwpack", "alpha", "beta", "final", "leb",
+          "release-candidate"]),
+        ["release", "hardware"],  # db_columns
+        "android_release",  # table
+        ["", "", "release", "hardware"])  # url_chunks
 
     crawler.crawl()
     crawler.clean_removed_urls_from_db()

=== modified file 'linaro_fetch_image/fetch_image.py'
--- linaro_fetch_image/fetch_image.py	2012-06-23 02:56:23 +0000
+++ linaro_fetch_image/fetch_image.py	2012-10-10 14:13:21 +0000
@@ -963,25 +963,31 @@
         # url_chunks now contains all parts of the url, split on /,
         # not including the base URL
 
+        # Create a dictionary of column_name, value pairs to insert
+        to_insert = {}
+        # While doing this, create a list of columns without any extra
+        # information used by fixed value columns
+        db_columns = []
+        for name in self.url_parse[index]["db_columns"]:
+            name_search = re.search("(\w+)=(.*)", name)
+            if name_search:
+                to_insert[name_search.group(1)] = name_search.group(2)
+                db_columns.append(name_search.group(1))
+            else:
+                to_insert[name] = None
+                db_columns.append(name)
+
         # We now construct an SQL command to insert the index data into the
         # database using the information we have.
-
         sqlcmd = "INSERT INTO " + table + " ("
         length = 0
-        for name in (self.url_parse[index]["url_chunks"] + ["url"]):
+        for name in (db_columns + ["url"]):
             if name != "":
                 if isinstance(name, tuple):
                     name = name[0]
                 sqlcmd += name + ", "
                 length += 1
 
-        # Handle fixed value columns
-        for name in self.url_parse[index]["db_columns"]:
-            name_search = re.search("(\w+)=(.*)", name)
-            if name_search:
-                sqlcmd += name_search.group(1) + ", "
-                length += 1
-
         sqlcmd = sqlcmd.rstrip(", ")  # get rid of unwanted space & comma
         sqlcmd += ") VALUES ("
 
@@ -993,31 +999,45 @@
         # Get the parameters from the URL to record in the SQL database
         sqlparams = []
         chunk_index = 0
-        for name in self.url_parse[index]["url_chunks"]:
-            # If this part of the URL isn't a parameter, don't insert it
-            if name != "":
-                # If the entry is a tuple, it indicates it is of the form
-                # name, regexp
-                if isinstance(name, tuple):
-                    # use stored regexp to extract data for the database
-                    match = re.search(name[1], url_chunks[chunk_index])
-                    assert match, ("Unable to match regexp to string ",
-                                  + url_chunks[chunk_index] + " " + name[1])
-                    sqlparams.append(match.group(1))
-
-                else:
-                    sqlparams.append(url_chunks[chunk_index])
-
-            chunk_index += 1
-
+
+        # Two options for parsing a URL.
+        # 1. Chunks (split up directories)
+        # 2. 1 regexp
+        # We parse as a regexp if there is only one URL chunk listed (we
+        # assume 1 chunk is unusual!)
+        if len(self.url_parse[index]["url_chunks"]) == 1:
+            url_search = re.search(self.url_parse[index]["url_chunks"][0],
+                                    url)
+            if not url_search:
+                print self.url_parse[index]["url_chunks"][0]
+                print url
+                print table
+                exit(1)
+            for column in db_columns:
+                if not to_insert[column]:
+                    to_insert[column] = url_search.group(column)
+        else:
+            for name in self.url_parse[index]["url_chunks"]:
+                # If this part of the URL isn't a parameter, don't insert it
+                if name != "":
+                    # If the entry is a tuple, it indicates it is of the form
+                    # name, regexp
+                    if isinstance(name, tuple):
+                        # use stored regexp to extract data for the database
+                        match = re.search(name[1], url_chunks[chunk_index])
+                        assert match, ("Unable to match regexp to string ",
+                            + url_chunks[chunk_index] + " " + name[1])
+                        to_insert[name[0]] = match.group(1)
+                    else:
+                        to_insert[name] = url_chunks[chunk_index]
+
+                chunk_index += 1
+
+        # Put data into array for inserting into SQL command
+        for column in db_columns:
+            sqlparams.append(to_insert[column])
         sqlparams.append(url)
 
-        # Handle fixed value columns
-        for name in self.url_parse[index]["db_columns"]:
-            name_search = re.search("(\w+)=(.*)", name)
-            if name_search:
-                sqlparams.append(name_search.group(2))
-
         logging.info("{0}: {1}".format(sqlcmd, sqlparams))
         self.c.execute(sqlcmd, tuple(sqlparams))