Merge lp:~smoser/maas-images/trunk.fix-powerpc-mining into lp:maas-images

[Scott Moser]

fix mining of d-i kernels for powerpc architecture

Cloud images are built for powerpc architecture, and in that build process
on xenial and yakkety, the build runs maas-images to scrape available d-i
kernel data.

That process was failing as the directory layout for powerpc was slightly
different than other arch. MAAS does not produce powerpc images, but at this
point, the only thing preventing them would be this code. Might as well fix it so that the failure during build does not have to be ignored.

This fixes those issues and adds the paths and expected data to the
file_item_data.json that represents expected data for possible paths.

[Scott Moser (smoser)]

The failure could be reproduced by running:
  ./bin/netboot-mirror --dry-run -vvv --arches=powerpc out.d

Now that command will succeed.

[lp:~smoser/maas-images/trunk.fix-powerpc-mining]

307. By Scott Moser on 2016-06-07

document FLAVOR_COLLISIONS dictionary a bit.

[Andres Rodriguez (andreserl)]

lgtm!

review: Approve

=== modified file 'meph2/netinst.py'
--- meph2/netinst.py	2016-05-11 18:41:21 +0000
+++ meph2/netinst.py	2016-06-07 17:31:34 +0000
@@ -64,13 +64,23 @@
 GPG_KEYRING = "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
 CONTENT_ID = "com.ubuntu.installer:released:netboot"
 
+# see get_kfile_key for how a file item key is generated.
+# flavor can be long, but only 3 chars are used for it in the keyname.
+# This dictionary maintains a static list of flavors that match the first
+# 3 chars and thus need other representation.
 FLAVOR_COLLISIONS = {
+    "e500": "e50",
+    "e500mc": "e5m",
+    "generic-lpae": "glp",
     "omap4": "om4",
-    "generic-lpae": "glp",
+    "powerpc": "ppc",
+    "powerpc64": "pp6",
 }
 
 KERNEL_FLAVORS = (
     'armadaxp',
+    'e500',
+    'e500mc',
     'generic',
     'generic-lpae',
     'highbank',
@@ -78,6 +88,8 @@
     'non-pae',
     'omap',
     'omap4',
+    'powerpc',
+    'powerpc64',
 )
 
 DTB_TO_FORMAT = {
@@ -101,6 +113,8 @@
     "dtb": re.compile(r".dtb$").search,
 }
 
+IGNORED_INITRD_FLAVORS =  ('xen', 'cdrom', 'gtk', 'hd-media')
+
 # #
 # # Under a path like: MIRROR/precise-updates/main/installer-i386/
 # #  we find a listing of directories like:
@@ -300,7 +314,7 @@
 
     # paths with 'xen' or 'cdrom' would be other initrd-flavors
     # essentially just blacklist paths with these toks
-    other_iflavor_toks = ('xen', 'cdrom', 'gtk')
+    other_iflavor_toks = IGNORED_INITRD_FLAVORS
     for other in other_iflavor_toks:
         if other + "/" in path:
             return None

=== modified file 'tests/unittests/file_item_data.json'
--- tests/unittests/file_item_data.json	2016-05-11 18:41:21 +0000
+++ tests/unittests/file_item_data.json	2016-06-07 17:31:34 +0000
@@ -133,6 +133,35 @@
   "kernel-flavor": "generic",
   "kernel-release": "BASE"
  },
+ "e500/netboot/initrd.gz": {
+  "ftype": "initrd",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "e500",
+  "kernel-release": "BASE"
+ },
+ "e500/netboot/vmlinux": {
+  "ftype": "kernel",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "e500",
+  "kernel-release": "BASE"
+ },
+ "e500mc/netboot/initrd.gz": {
+  "ftype": "initrd",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "e500mc",
+  "kernel-release": "BASE"
+ },
+ "e500mc/netboot/vmlinux": {
+  "ftype": "kernel",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "e500mc",
+  "kernel-release": "BASE"
+ },
+ "e500/cdrom/vmlinux": null,
  "generic-lpae/netboot/initrd.gz": {
   "ftype": "initrd",
   "image-format": "default",
@@ -635,6 +664,35 @@
   "kernel-flavor": "omap4",
   "kernel-release": "BASE"
  },
+ "powerpc/hd-media/vmlinux": null,
+ "powerpc/netboot/initrd.gz": {
+  "ftype": "initrd",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "powerpc",
+  "kernel-release": "BASE"
+ },
+ "powerpc/netboot/vmlinux": {
+  "ftype": "kernel",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "powerpc",
+  "kernel-release": "BASE"
+ },
+ "powerpc64/netboot/initrd.gz": {
+  "ftype": "initrd",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "powerpc64",
+  "kernel-release": "BASE"
+ },
+ "powerpc64/netboot/vmlinux": {
+  "ftype": "kernel",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "powerpc64",
+  "kernel-release": "BASE"
+ },
  "quantal-netboot-xen/netboot/xen/initrd.gz": null,
  "quantal-netboot-xen/netboot/xen/vmlinuz": null,
  "quantal-netboot-xen/netboot/xen/xm-debian.cfg": null,
@@ -1372,19 +1430,47 @@
   "kernel-release": "wily"
  },
  "wily-netboot/xen/xm-debian.cfg": null,
- "xenial-generic/netboot/initrd.gz": {
-  "ftype": "initrd",
-  "image-format": "default",
-  "initrd-flavor": "netboot",
-  "kernel-flavor": "generic",
-  "kernel-release": "xenial"
- },
- "xenial-generic/netboot/initrd.gz": {
-  "ftype": "initrd",
-  "image-format": "default",
-  "initrd-flavor": "netboot",
-  "kernel-flavor": "generic",
-  "kernel-release": "xenial"
- },
- "xenial-netboot/ubuntu-installer/amd64/pxelinux.0": null
+ "xenial-e500mc/netboot/initrd.gz": {
+  "ftype": "initrd",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "e500mc",
+  "kernel-release": "xenial"
+ },
+ "xenial-e500mc/netboot/vmlinux": {
+  "ftype": "kernel",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "e500mc",
+  "kernel-release": "xenial"
+ },
+ "xenial-generic/netboot/initrd.gz": {
+  "ftype": "initrd",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "generic",
+  "kernel-release": "xenial"
+ },
+ "xenial-generic/netboot/initrd.gz": {
+  "ftype": "initrd",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "generic",
+  "kernel-release": "xenial"
+ },
+ "xenial-netboot/ubuntu-installer/amd64/pxelinux.0": null,
+ "xenial-powerpc64/netboot/initrd.gz": {
+  "ftype": "initrd",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "powerpc64",
+  "kernel-release": "xenial"
+ },
+ "xenial-powerpc64/netboot/vmlinux": {
+  "ftype": "kernel",
+  "image-format": "default",
+  "initrd-flavor": "netboot",
+  "kernel-flavor": "powerpc64",
+  "kernel-release": "xenial"
+ }
 }