Merge lp:~deeptik/linaro-fetch-image/release-matrix into lp:linaro-fetch-image

[Deepti B. Kalakeri]

Some useful command to get release list matrix, also modified l-i-f CLI to run l-m-c not just one but all the matching available platforms.

Next step is to be able to specify a list of platforms that we need to be able to build at one go.

[James Tunnicliffe (dooferlad)]

Hi,

Thanks for the patch. There are a couple of minor problems and I have one larger suggestion, but nothing that will take long to fix. First, this section with dates is unneeded:

+ # We only need to look up a single snapshot date. We just use the
+ # latest in the database (we could use today and search from it, but
+ # the database is just one that is checked in, so it could be old
+ # and db.get_next_prev_day_with_builds may give up before finding it).
+ date = db.execute_return_list(
+ "SELECT MAX(date) FROM snapshot_binaries")[0][0]
+ d = re.search("(\d{4})(\d{2})(\d{2})", date)
+ date = (d.group(1) + "-" + d.group(2) + "-" + d.group(3))

The date variable was only used for snapshot lookup, not for release lookup.

+ # - Run the function under test! -

This comment is now wrong

+ image_url, hwpack_url = (
+ db.get_image_and_hwpack_urls(
+ self.settings))
+ temp_list.append(self.settings['platform'])
+ img_list[self.settings['image']].append(self.settings['platform'])
+ release_list[self.settings['hardware']] = img_list

Some of these lines are >80 characters wide. Since the function has a lot of indentation I suggest you move this section into its own function to make life easier:

+ # -- Check build which matches these parameters
+ #(builds that don't match are excluded in UI)--
+ if( len(db.execute_return_list(
+ 'select * from release_hwpacks '
+ 'where platform == ? '
+ 'and hardware == ? '
+ 'and build == ?',
+ (self.settings['platform'],
+ self.settings['hwpack'],
+ self.settings['build'])))
+ and len(db.execute_return_list(
+ 'select * from release_binaries '
+ 'where platform == ? '
+ 'and image == ? '
+ 'and build == ?',
+ (self.settings['platform'],
+ self.settings['image'],
+ self.settings['build'])))):
+
+ # - Run the function under test! -
+ image_url, hwpack_url = (
+ db.get_image_and_hwpack_urls(
+ self.settings))
+ temp_list.append(self.settings['platform'])...

review: Needs Fixing

[lp:~deeptik/linaro-fetch-image/release-matrix]

23. By Deepti B. Kalakeri on 2012-10-12

Adding --hwpack-force-yes command line option to l-f-i

24. By Deepti B. Kalakeri on 2012-10-12

Factoring into small functions, removing unneeded code

=== modified file 'linaro-fetch-image'
--- linaro-fetch-image	2012-10-10 15:42:01 +0000
+++ linaro-fetch-image	2012-10-12 10:32:19 +0000
@@ -24,6 +24,7 @@
 import os
 import linaro_fetch_image.fetch_image as fetch_image
 import logging
+import pprint
 
 
 def main():
@@ -38,8 +39,12 @@
     force_download = ("--force-download" in sys.argv[1:]
                       or "-d" in sys.argv[1:])
 
+    list_matrix = ("--list-matrix" in sys.argv[1:]
+                      or "-l" in sys.argv[1:])
+
     if clean_cache:
         file_handler.clean_cache()
+    
 
     # If the settings file and server index need updating, grab them
     file_handler.update_files_from_server(force_download)
@@ -50,32 +55,57 @@
     db = fetch_image.DB(file_handler.index_file)
     config.finish_config(db)
 
+    # Get a list of release builds that are available for testing 
+    release_list = config.get_release_list(db, config.settings)
+    if list_matrix:
+        pp = pprint.PrettyPrinter(indent=4)
+        pp.pprint(release_list)
+        exit(0)
+
     # Using the settings that the YAML defines as what we need for a build,
     # generate a command line parser and parse the command line
     config.parse_args(sys.argv[1:])
-
     if config.args['platform'] == "snapshot":
         config.args['release_or_snapshot'] = "snapshot"
     else:
         config.args['release_or_snapshot'] = "release"
 
+
+    # Get release builds for specific  image type, for example nano
+    platform_list = config.get_release_list_specific_to_image(release_list, 
+                                                           config.args)
     # Using the config we have, look up URLs to download data from in the
     # server index
-
-    image_url, hwpack_url = db.get_image_and_hwpack_urls(config.args)
-
-    if(image_url and hwpack_url):
-        if config.args['exit']:
-            exit(0)
-        tools_dir = os.path.dirname(__file__)
-        if tools_dir == '':
-            tools_dir = None
-
-        file_handler.create_media(image_url, hwpack_url,
-                                  config.args, tools_dir)
+    if config.args['platform'] == 'all':
+        for platform in platform_list:
+            config.args['platform'] = platform
+            image_url, hwpack_url = db.get_image_and_hwpack_urls(config.args)
+            if(image_url and hwpack_url):
+                if config.args['exit']:
+                    exit(0)
+                tools_dir = os.path.dirname(__file__)
+                if tools_dir == '':
+                    tools_dir = None
+                file_handler.create_media(image_url, hwpack_url,
+                                          config.args, tools_dir)
+            else:
+                logging.error("Unable to find files that match the '%s' " \
+                               "release", platform)
     else:
-        logging.error(
-            "Unable to find files that match the parameters specified")
+        image_url, hwpack_url = db.get_image_and_hwpack_urls(config.args)
+
+        if(image_url and hwpack_url):
+            if config.args['exit']:
+                exit(0)
+            tools_dir = os.path.dirname(__file__)
+            if tools_dir == '':
+                tools_dir = None
+
+            file_handler.create_media(image_url, hwpack_url,
+                                      config.args, tools_dir)
+        else:
+            logging.error(
+                "Unable to find files that match the parameters specified")
         exit(1)
 
 if __name__ == '__main__':

=== modified file 'linaro_fetch_image/fetch_image.py'
--- linaro_fetch_image/fetch_image.py	2012-10-10 15:42:01 +0000
+++ linaro_fetch_image/fetch_image.py	2012-10-12 10:32:19 +0000
@@ -583,7 +583,7 @@
         if run_in_gui:
             args.append("--nocheck-mmc")
 
-        if hwpack_verified:
+        if hwpack_verified or settings['hwpack_force_yes']:
             # We verify the hwpack before calling linaro-media-create because
             # these tools run linaro-media-create as root and to get the GPG
             # signature verification to work, root would need to have GPG set
@@ -642,7 +642,6 @@
                                              settings,
                                              tools_dir,
                                              hwpack_verified)
-
         self.create_process = subprocess.Popen(lmc_command)
         self.create_process.wait()
 
@@ -883,6 +882,12 @@
         parser.add_argument("-d", "--force-download",
                             help="Force re-downloading of cached files",
                             action='store_true')
+        parser.add_argument("-l", "--list-matrix",  action='store_true', 
+                            help="List available board/image/platform release"\
+                                 "matrix")
+        parser.add_argument("-y", "--hwpack-force-yes",
+                            help="Force install packages without verification",
+                            action='store_true')
         parser.add_argument(
                     "-t", "--image-file",
                     help="Where to write image file to (use this XOR mmc)")
@@ -944,6 +949,85 @@
         for image in sorted(self.settings['choice']['image']):
             print image,
 
+    def get_release_list(self, db, config):
+        release_list = {}
+        self.settings = config
+
+        #--- Now test release build lookup ---
+        self.settings['release_or_snapshot'] = "release"
+        # -- Select hardware --
+        for self.settings['hardware'] in (
+                                   self.settings['choice']['hardware'].keys()):
+            compatable_hwpacks = (
+                  self.settings['choice']['hwpack'][self.settings['hardware']])
+            # -- Iterate through hardware packs --
+            for self.settings['hwpack'] in compatable_hwpacks:
+                # If hardware pack is available...
+                if(self.settings['hwpack']
+                    in db.get_hwpacks('release_hwpacks')):
+                    
+                    # -- Iterate through images
+                    os_list = db.get_os_list_from('release_binaries')
+                    img_list = {}
+                    for self.settings['image'] in os_list:
+                        if re.search('old', self.settings['image']):
+                            # Directories with old in the name are of no
+                            # interest to us
+                            continue
+                        img_list[self.settings['image']] = [] 
+                        release_list[self.settings['hardware']] = img_list
+                        for platform, ignore in (
+                                  self.settings['choice']['platform'].items()):
+                            self.settings['platform'] = platform
+                            # -- Iterate through available builds --
+                            builds = db.get_builds(
+                                                    self.settings['platform'],
+                                                    self.settings['image'])
+                            for build in builds:
+                                self.settings['build'] = build
+                                self.update_release_list(db, release_list, img_list)
+
+
+        return release_list
+
+    def update_release_list(self, db, release_list, img_list):
+         temp_list = []
+         # -- Check build which matches these parameters
+         #(builds that don't match are excluded)--
+         if( len(db.execute_return_list(
+                     'select * from release_hwpacks '
+                     'where platform == ? '
+                     'and hardware == ? '
+                     'and build == ?',
+                     (self.settings['platform'],
+                      self.settings['hwpack'],
+                      self.settings['build'])))
+            and len(db.execute_return_list(
+                     'select * from release_binaries '
+                     'where platform == ? '
+                     'and image == ? '
+                     'and build == ?',
+                     (self.settings['platform'],
+                      self.settings['image'],
+                      self.settings['build'])))):
+
+             image_url, hwpack_url = (
+                      db.get_image_and_hwpack_urls(
+                               self.settings))
+             temp_list.append(self.settings['platform']) 
+             img_list[self.settings['image']].append(self.settings['platform'])
+             release_list[self.settings['hardware']] = img_list
+
+    def get_release_list_specific_to_image(self, release_list, config_args):
+        hardware = config_args['hardware']
+        image = config_args['image']
+        if hardware in release_list.keys():
+            if image in release_list[hardware].keys():
+                return release_list[hardware][image]
+            else:
+                return None
+          
+        return None
 
 class DB():
     """Interacts with the database storing URLs of files that are available
@@ -1674,3 +1758,4 @@
                 logging.error(msg)
 
         return(image_url[0], hwpack_url[0])
+