Merge ~gavin.lin/cc-lab-manager:platform-config into cc-lab-manager:master

[Gavin Lin]

Support generating customized agent config from platform name or device_id.
Fix typo for maas provision type.

[Kevin Yeh (kevinyeh)]

LGTM +1

review: Approve

diff --git a/cc_lab_manager/gen_config/gen_agent_tf_config.py b/cc_lab_manager/gen_config/gen_agent_tf_config.py
index 1292ce6..d72b0bf 100644
--- a/cc_lab_manager/gen_config/gen_agent_tf_config.py
+++ b/cc_lab_manager/gen_config/gen_agent_tf_config.py
@@ -4,8 +4,8 @@ import yaml
 import argparse
 import re
 
-from cc_lab_manager.cc_lab_manager_maptable import reboot_script_maptable, provision_type_maptable
-from cc_lab_manager.cc_lab_manager_maptable import customized_agent_config_maptable
+from cc_lab_manager.cc_lab_manager_maptable import lab_netboot_servers, provision_type_maptable, reboot_script_maptable
+from cc_lab_manager.device_config_maptable import agent_config_platform_maptable, agent_config_device_maptable
 
 agent_config = {"device_ip": None,
         "secure_id": None,
@@ -94,6 +94,8 @@ def read_data_from_db(cursor):
 
 
 def device_id_to_agent_name(device_id):
+    # Juju does not accept only numbers in any section divided by hyphen,
+    # so generate agent name to comply that rule.
     id_items = device_id.split('-')
     if re.match('^[0-9]*$', id_items[0]):
         print('Unsupported device id:', device_id)
@@ -110,6 +112,19 @@ def device_id_to_agent_name(device_id):
         return '-'.join(output)
 
 
+def device_id_to_platform_name(device_id):
+    # Platform name: Device_ID without the tailing increments or CID.
+    id_items = device_id.split('-')
+    if len(id_items) < 2:
+        return device_id
+    else:
+        if re.match('^[0-9]{3}$', id_items[-1]) or re.match('^c[0-9]{4,}$', id_items[-1]):
+            id_items.pop()
+            return '-'.join(id_items)
+        else:
+            return device_id
+
+
 def create_agent_config_dir(db_machine_list, cfg_path):
     
     for machine in db_machine_list:
@@ -122,64 +137,87 @@ def create_agent_config_dir(db_machine_list, cfg_path):
         except:
             print("dir exist for {}".format(agent_name))
 
+
+def insert_data_from_maptable(update_conf, device_data, maptable_name, device_key):
+    maptable = globals()[maptable_name]
+    if device_key in maptable:
+        for key in maptable[device_key]:
+            if isinstance(maptable[device_key][key], list):
+                update_conf[key] = []
+                for line in maptable[device_key][key]:
+                    try:
+                        update_conf[key].append(line.format(**device_data))
+                    except KeyError:
+                        update_conf[key].append(line)
+            elif isinstance(maptable[device_key][key], (bool, int, float, complex)):
+                update_conf[key] = maptable[device_key][key]
+            else:
+                try:
+                    update_conf[key] = maptable[device_key][key].format(**device_data)
+                except KeyError:
+                    update_conf[key] = maptable[device_key][key]
+    return update_conf
+
+
 def generate_agent_config(db_machine_list, cfg_path):
     
     for machine in db_machine_list:
         if machine['lab'] == '' or machine['Customized_agent_config'] == 'TRUE':
             continue
 
+        # Copy data from sqlite3.Row object to a dict so we can add more keys later
+        device_data = dict(machine)
+
         try:
-            provision = provision_type_maptable[machine["Provision"]]
+            provision = provision_type_maptable[device_data["Provision"]]
         except:
-            print("Unknown provision type for {}".format(machine['cid']))
+            print("Unknown provision type for {}".format(device_data['Provision']))
             continue
 
+        # Update default configs apply to all devices.
         agent_conf = agent_config.copy()
-        agent_name = device_id_to_agent_name(machine["Device_ID"])
-
-        agent_conf["device_ip"] = machine["IP"]
-        agent_conf["secure_id"] = machine["SecureID"]
+        agent_name = device_id_to_agent_name(device_data["Device_ID"])
+        agent_conf["device_ip"] = device_data["IP"]
+        agent_conf["secure_id"] = device_data["SecureID"]
         agent_conf["agent_name"] = agent_name
-        agent_conf["env"]["HEXR_DEVICE_SECURE_ID"] = machine["SecureID"]
-        agent_conf["env"]["DEVICE_IP"] = machine["IP"]
+        agent_conf["env"]["HEXR_DEVICE_SECURE_ID"] = device_data["SecureID"]
+        agent_conf["env"]["DEVICE_IP"] = device_data["IP"]
 
-        if provision == 'maas':
-            agent_conf["node_id"] = machine["MAAS_Node_ID"]
-            agent_conf["node_name"] = machine["Device_ID"]
+        # Update default configs for some provision types.
+        if provision == 'maas2':
+            agent_conf["node_id"] = device_data["MAAS_Node_ID"]
+            agent_conf["node_name"] = device_data["Device_ID"]
 
-        if machine["Provision"] == 'cm3' or machine["Provision"] == 'muxpi' \
-        or machine["Provision"] == 'dragonboard':
-            agent_conf["control_host"] = machine["Controller_IP"]
-            agent_conf['serial_host'] = machine["Controller_IP"]
+        if provision == 'netboot':
+            device_data['Netboot_IP'] = lab_netboot_servers[device_data['Lab'].lower()]
+
+        if device_data["Provision"] == 'cm3' or device_data["Provision"] == 'muxpi' \
+        or device_data["Provision"] == 'dragonboard':
+            agent_conf["control_host"] = device_data["Controller_IP"]
+            agent_conf['serial_host'] = device_data["Controller_IP"]
             agent_conf['serial_port'] = '3000'
 
-        if machine["Provision"] == 'cm3' or machine["Provision"] == 'muxpi':
+        if device_data["Provision"] == 'cm3' or device_data["Provision"] == 'muxpi':
             agent_conf['test_device'] = '/dev/sda'
             agent_conf['snappy_writable_partition'] = '/dev/sda2'
-        elif machine["Provision"] == 'dragonboard':
+        elif device_data["Provision"] == 'dragonboard':
             agent_conf['test_device'] = '/dev/mmcblk1'
             agent_conf['snappy_writable_partition'] = '/dev/mmcblk1p9'
             agent_conf['user_assertion'] = '/home/ubuntu/plars/auto-import.assert'
             agent_conf['model_assertion'] = '/home/ubuntu/plars/dragon.model'
 
+        # Update reboot script for known type.
         agent_conf["reboot_script"] = []
-        power_template = {'PDU_IP' : machine["PDU_IP"],
-                          'PDU_Outlet' : machine["PDU_Outlet"]}
-        try:
-            reboot_script_template = reboot_script_maptable[machine["Power"]]
-            for line in reboot_script_template:
-                agent_conf["reboot_script"].append(line.format(**machine))
-        except:
-            print('No power template')
+        if device_data["Power"] in reboot_script_maptable:
+            for line in reboot_script_maptable[device_data["Power"]]:
+                agent_conf["reboot_script"].append(line.format(**device_data))
+
+        # Insert or overwrite platform specific config from maptable if any.
+        platform_name = device_id_to_platform_name(device_data['Device_ID'])
+        agent_conf = insert_data_from_maptable(agent_conf, device_data, 'agent_config_platform_maptable', platform_name)
 
-        if machine["Device_ID"] in customized_agent_config_maptable:
-            for key in customized_agent_config_maptable[machine["Device_ID"]]:
-                if isinstance(customized_agent_config_maptable[machine["Device_ID"]][key], list): 
-                    agent_conf[key] = []
-                    for line in customized_agent_config_maptable[machine["Device_ID"]][key]:
-                        agent_conf[key].append(line.format(**machine))
-                else:
-                    agent_conf[key] = customized_agent_config_maptable[machine["Device_ID"]][key]
+        # Insert or overwrite device specific config from maptable if any.
+        agent_conf = insert_data_from_maptable(agent_conf, device_data, 'agent_config_device_maptable', device_data["Device_ID"])
 
         empty_keys = []
         for key in agent_conf:
@@ -211,15 +249,13 @@ def generate_tf_config(db_machine_list, cfg_path):
         tf_conf["results_basedir"] = tf_conf["results_basedir"].format(agent_name)
         tf_conf["provision_type"] = provision
 
-        # Default queues: CID, Device_ID, and Device_ID without tailing CID or increment
+        # Default queues: CID, Device_ID, and platform name(Device_ID without tailing CID or increment)
         tf_conf['job_queues'] = []
         tf_conf['job_queues'].append(machine['CID'])
         tf_conf['job_queues'].append(machine['Device_ID'])
-        id_items = machine['Device_ID'].split('-')
-        if len(id_items) > 1:
-            if re.match('^[0-9]{3}$', id_items[-1]) or re.match('^c[0-9]{4,}$', id_items[-1]):
-                id_items.pop()
-                tf_conf['job_queues'].append('-'.join(id_items))
+        platform_name = device_id_to_platform_name(machine['Device_ID'])
+        if platform_name != machine['Device_ID']:
+            tf_conf['job_queues'].append(platform_name)
 
         # Add customized queues if any
         if machine['TF_Queue'] != '':