Merge lp:~danilo/linaro-license-protection/sanitized-tree into lp:~linaro-automation/linaro-license-protection/trunk

[Данило Шеган]

Use the functionality of SnapshotsPublisher to create a sanitized copy of
an entire tree. We rely on shutil.copy2 and shutil.copystat to ensure
appropriate flags are transferred as well.

We ignore errors and simply log them. This will result in them coming to our inboxes via linaro-infrastructure-errors so we can insure they are not a problem (and we'll need to update the script then, but hey).

The script simply prints out the destination directory when done (all other output goes to stderr). The idea is to use this and store it in an environment variable and then pass it to rsync to copy from mombin (production) to kahaku (staging).

I am still not sure if it'd be better to leave the temp dir creation to the script caller as well. Makes for slightly more complicated wrapper script, but probably not by much. What do you think?

This also extracts some of the common bits (like supporting multiple -v options to increase verbosity of the logger output, and configuration of the logger) out into a separate class and shares it with update-deployment script (the changes in update-deployment are very minor: mostly reindentation and additions of "self." where appropriate).

[Stevan Radaković (stevanr)]

This looks good. Temp directory creation should be left here, IMO.
It might make sense to change my validation script to extend the linaroscript class as well..
Approve +1.

review: Approve

=== added file 'scripts/linaroscript.py'
--- scripts/linaroscript.py	1970-01-01 00:00:00 +0000
+++ scripts/linaroscript.py	2012-08-22 23:43:25 +0000
@@ -0,0 +1,63 @@
+# Copyright 2012 Linaro.
+
+"""Helper class for creating new scripts.
+
+It pre-defines a logger using the script_name (passed through the constructor)
+and allows different verbosity levels.
+
+Overload the work() method to define the main work to be done by the script.
+
+You can use the logger by accessing instance.logger attribute.
+You can use the parser (eg. to add another argument) by accessing
+instance.argument_parser attribute.
+
+Parsed arguments are available to your work() method in instance.arguments.
+"""
+
+import argparse
+import logging
+
+
+class LinaroScript(object):
+    def __init__(self, script_name, description=None):
+        self.script_name = script_name
+        self.description = description
+        self.argument_parser = argparse.ArgumentParser(
+            description=self.description)
+        self.setup_parser()
+
+    def work(self):
+        """The main body of the script.  Overload when subclassing."""
+        raise NotImplementedError
+
+    def run(self):
+        self.arguments = self.argument_parser.parse_args()
+        logging_level = self.get_logging_level_from_verbosity(
+            self.arguments.verbose)
+        self.logger = logging.getLogger(self.script_name)
+        self.logger.setLevel(logging_level)
+        formatter = logging.Formatter(
+            fmt='%(asctime)s %(levelname)s: %(message)s')
+        handler = logging.StreamHandler()
+        handler.setFormatter(formatter)
+        self.logger.addHandler(handler)
+
+        self.work()
+
+    def setup_parser(self):
+        self.argument_parser.add_argument(
+            "-v", "--verbose", action='count',
+            help=("Increase the output verbosity. "
+                  "Can be used multiple times"))
+
+    def get_logging_level_from_verbosity(self, verbosity):
+        """Return a logging level based on the number of -v arguments."""
+        if verbosity == 0:
+            logging_level = logging.ERROR
+        elif verbosity == 1:
+            logging_level = logging.INFO
+        elif verbosity >= 2:
+            logging_level = logging.DEBUG
+        else:
+            logging_level = logging.ERROR
+        return logging_level
 
=== added file 'scripts/make-sanitized-tree-copy.py'
--- scripts/make-sanitized-tree-copy.py	1970-01-01 00:00:00 +0000
+++ scripts/make-sanitized-tree-copy.py	2012-08-22 23:43:25 +0000
@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# Create a copy of a directory structure while preserving as much metadata
+# as possible and sanitizing any sensitive data.
+
+# Everything, unless whitelisted, is truncated and the contents are replaced
+# with the base file name itself.
+
+import os
+import shutil
+import tempfile
+
+from linaroscript import LinaroScript
+from publish_to_snapshots import SnapshotsPublisher
+
+
+class MakeSanitizedTreeCopyScript(LinaroScript):
+
+    def setup_parser(self):
+        super(MakeSanitizedTreeCopyScript, self).setup_parser()
+        self.argument_parser.add_argument(
+            'directory', metavar='DIR', type=str,
+            help="Directory to create a sanitized deep copy of.")
+
+    @staticmethod
+    def filter_accepted_files(directory, list_of_files):
+        accepted_files = []
+        for filename in list_of_files:
+            full_path = os.path.join(directory, filename)
+            if SnapshotsPublisher.is_accepted_for_staging(full_path):
+                accepted_files.append(filename)
+        return accepted_files
+
+    def copy_sanitized_tree(self, source, target):
+        """Copies the tree from `source` to `target` while sanitizing it.
+
+        Performs a recursive copy trying to preserve as many file
+        attributes as possible.
+        """
+        assert os.path.isdir(source) and os.path.isdir(target), (
+            "Both source (%s) and target (%s) must be directories." % (
+                source, target))
+        self.logger.debug("copy_sanitized_tree('%s', '%s')", source, target)
+        filenames = os.listdir(source)
+        for filename in filenames:
+            self.logger.debug("Copying '%s'...", filename)
+            source_file = os.path.join(source, filename)
+            target_file = os.path.join(target, filename)
+            try:
+                if os.path.isdir(source_file):
+                    self.logger.debug("Making directory '%s'" % target_file)
+                    os.makedirs(target_file)
+                    self.copy_sanitized_tree(source_file, target_file)
+                elif SnapshotsPublisher.is_accepted_for_staging(source_file):
+                    self.logger.debug(
+                        "Copying '%s' to '%s' with no sanitization...",
+                        source_file, target_file)
+                    shutil.copy2(source_file, target_file)
+                else:
+                    self.logger.debug(
+                        "Creating sanitized file '%s'", target_file)
+                    # This creates an target file.
+                    open(target_file, "w").close()
+                    shutil.copystat(source_file, target_file)
+                    SnapshotsPublisher.sanitize_file(target_file)
+            except (IOError, os.error) as why:
+                self.logger.error(
+                    "While copying '%s' to '%s' we hit:\n\t%s",
+                    source_file, target_file, str(why))
+
+        try:
+            shutil.copystat(source, target)
+        except OSError as why:
+            self.logger.error(
+                "While copying '%s' to '%s' we hit:\n\t%s",
+                source, target, str(why))
+
+    def work(self):
+        source_directory = self.arguments.directory
+        self.logger.info("Copying and sanitizing '%s'...", source_directory)
+        target_directory = tempfile.mkdtemp()
+        self.logger.info("Temporary directory: '%s'", target_directory)
+
+        self.copy_sanitized_tree(
+            self.arguments.directory, target_directory)
+
+        print target_directory
+
+if __name__ == '__main__':
+    script = MakeSanitizedTreeCopyScript(
+        'make-sanitized-tree-copy',
+        description=(
+            "Makes a copy of a directory tree in a temporary location "
+            "and sanitize file that can contain potentially restricted "
+            "content. "
+            "Returns the path of a newly created temporary directory."))
+    script.run()
 
=== modified file 'scripts/update-deployment.py'
--- scripts/update-deployment.py	2012-08-16 13:09:23 +0000
+++ scripts/update-deployment.py	2012-08-22 23:43:25 +0000
@@ -34,13 +34,13 @@
 
 """
 
-import argparse
 import bzrlib.branch
 import bzrlib.workingtree
-import logging
 import os
 import subprocess
 
+from linaroscript import LinaroScript
+
 code_base = '/srv/shared-branches'
 branch_name = 'linaro-license-protection'
 configs_branch_name = 'linaro-license-protection-config'
@@ -60,92 +60,83 @@
 configs_root = os.path.join(code_base, configs_branch_name)
 
 
-def refresh_branch(branch_dir):
-    """Refreshes a branch checked-out to a branch_dir."""
-
-    code_branch = bzrlib.branch.Branch.open(branch_dir)
-    parent_branch = bzrlib.branch.Branch.open(
-        code_branch.get_parent())
-    result = code_branch.pull(source=parent_branch)
-    if result.old_revno != result.new_revno:
-        logger.info("Updated %s from %d to %d.",
-                    branch_dir, result.old_revno, result.new_revno)
-    else:
-        logger.info("No changes to pull from %s.", code_branch.get_parent())
-    logger.debug("Updating working tree in %s.", branch_dir)
-    update_tree(branch_dir)
-    return code_branch
-
-
-def update_tree(working_tree_dir):
-    """Does a checkout update."""
-    code_tree = bzrlib.workingtree.WorkingTree.open(working_tree_dir)
-    code_tree.update()
-
-
-def update_installation(config, installation_root):
-    """Updates a single installation code and databases.
-
-    It expects code and config branches to be simple checkouts (working trees)
-    so it only does an "update" on them.
-
-    Afterwards, it runs "syncdb" and "collectstatic" steps.
-    """
-    refresh_branch(os.path.join(installation_root, branch_name))
-    refresh_branch(os.path.join(installation_root, "configs"))
-    os.environ["PYTHONPATH"] = (
-        ":".join(
-            [installation_root,
-             os.path.join(installation_root, branch_name),
-             os.path.join(installation_root, "configs", "django"),
-             os.environ.get("PYTHONPATH", "")]))
-
-    logger.info("Updating installation in %s with config %s...",
-                installation_root, config)
-    os.environ["DJANGO_SETTINGS_MODULE"] = config
-    logger.debug("DJANGO_SETTINGS_MODULE=%s",
-                 os.environ.get("DJANGO_SETTINGS_MODULE"))
-
-    logger.debug("Doing 'syncdb'...")
-    logger.debug(subprocess.check_output(
-        ["django-admin", "syncdb", "--noinput"], cwd=code_root))
-
-    logger.debug("Doing 'collectstatic'...")
-    logger.debug(subprocess.check_output(
-        ["django-admin", "collectstatic", "--noinput"],
-        cwd=code_root))
+class UpdateDeploymentScript(LinaroScript):
+
+    def refresh_branch(self, branch_dir):
+        """Refreshes a branch checked-out to a branch_dir."""
+
+        code_branch = bzrlib.branch.Branch.open(branch_dir)
+        parent_branch = bzrlib.branch.Branch.open(
+            code_branch.get_parent())
+        result = code_branch.pull(source=parent_branch)
+        if result.old_revno != result.new_revno:
+            self.logger.info("Updated %s from %d to %d.",
+                             branch_dir, result.old_revno, result.new_revno)
+        else:
+            self.logger.info(
+                "No changes to pull from %s.", code_branch.get_parent())
+        self.logger.debug("Updating working tree in %s.", branch_dir)
+        self.update_tree(branch_dir)
+        return code_branch
+
+    def update_tree(self, working_tree_dir):
+        """Does a checkout update."""
+        code_tree = bzrlib.workingtree.WorkingTree.open(working_tree_dir)
+        code_tree.update()
+
+    def update_installation(self, config, installation_root):
+        """Updates a single installation code and databases.
+
+        It expects code and config branches to be simple checkouts
+        (working trees) so it only does an "update" on them.
+
+        Afterwards, it runs "syncdb" and "collectstatic" steps.
+        """
+        self.refresh_branch(os.path.join(installation_root, branch_name))
+        self.refresh_branch(os.path.join(installation_root, "configs"))
+        os.environ["PYTHONPATH"] = (
+            ":".join(
+                [installation_root,
+                 os.path.join(installation_root, branch_name),
+                 os.path.join(installation_root, "configs", "django"),
+                 os.environ.get("PYTHONPATH", "")]))
+
+        self.logger.info("Updating installation in %s with config %s...",
+                         installation_root, config)
+        os.environ["DJANGO_SETTINGS_MODULE"] = config
+        self.logger.debug("DJANGO_SETTINGS_MODULE=%s",
+                          os.environ.get("DJANGO_SETTINGS_MODULE"))
+
+        self.logger.debug("Doing 'syncdb'...")
+        self.logger.debug(subprocess.check_output(
+                ["django-admin", "syncdb", "--noinput"], cwd=code_root))
+
+        self.logger.debug("Doing 'collectstatic'...")
+        self.logger.debug(subprocess.check_output(
+                ["django-admin", "collectstatic", "--noinput"],
+                cwd=code_root))
+
+    def setup_parser(self):
+        super(UpdateDeploymentScript, self).setup_parser()
+        self.argument_parser.add_argument(
+            'configs', metavar='CONFIG', nargs='+',
+            choices=configs_to_use.keys(),
+            help=("Django configuration module to use. One of " +
+                  ', '.join(configs_to_use.keys())))
+
+    def work(self):
+        # Refresh code in shared-branches.
+        self.refresh_branch(code_root)
+        self.refresh_branch(configs_root)
+
+        # We update installations for all the configs we've got.
+        for config in self.arguments.configs:
+            self.update_installation(config, configs_to_use[config])
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
+    script = UpdateDeploymentScript(
+        'update-deployment',
         description=(
             "Update staging deployment of lp:linaro-license-protection."))
-    parser.add_argument(
-        'configs', metavar='CONFIG', nargs='+', choices=configs_to_use.keys(),
-        help=("Django configuration module to use. One of " +
-              ', '.join(configs_to_use.keys())))
-    parser.add_argument("-v", "--verbose", action='count',
-                        help=("Increase the output verbosity. "
-                              "Can be used multiple times"))
-    args = parser.parse_args()
-
-    logging_level = logging.ERROR
-    if args.verbose == 0:
-        logging_level = logging.ERROR
-    elif args.verbose == 1:
-        logging_level = logging.INFO
-    elif args.verbose >= 2:
-        logging_level = logging.DEBUG
-
-    logger = logging.getLogger('update-staging')
-    logging.basicConfig(
-        format='%(asctime)s %(levelname)s: %(message)s',
-        level=logging_level)
-
-    # Refresh code in shared-branches.
-    refresh_branch(code_root)
-    refresh_branch(configs_root)
-
-    # We update installations for all the configs we've got.
-    for config in args.configs:
-        update_installation(config, configs_to_use[config])
+    script.run()