Merge lp:~dooferlad/linaro-license-protection/api-updates into lp:~linaro-automation/linaro-license-protection/trunk

[James Tunnicliffe]

Minor update to the API so <server>/api/ls/<path to file> works. Previously a listing would only work for single directory.

Updated download.py. Now it is a full, interactive download script:

download.py <URL of directory>
 - download all files in directory

download <URL of file>
 - download single file

Both of the above require the user to accept each license once. Once a license has been accepted the digest is stored so they don't have to re-accept an unchanged license.

Have split up download.py so the guts of the work is done in a single function, but set up and some messing around with URLs is moved out into a class. This improves readability. Since it is designed as executable documentation, it seems reasonable to take the cruft out of the main function.

[Milo Casagrande (milo)]

Hey James,

thanks for working on this, it looks good to go.
For the download.py file, as long as it is a demo, it looks OK too, otherwise there might be a couple of changes to apply. I hope we can provide a good CLI-client for this in the future.

review: Approve

=== modified file 'license_protected_downloads/tests/test_views.py'
--- license_protected_downloads/tests/test_views.py	2013-03-12 15:03:35 +0000
+++ license_protected_downloads/tests/test_views.py	2013-03-18 18:32:28 +0000
@@ -290,6 +290,30 @@
 
             self.assertEqual(mtime, file_info["mtime"])
 
+    def test_api_get_listing_single_file(self):
+        url = "/api/ls/build-info/snowball-blob.txt"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+
+        data = json.loads(response.content)["files"]
+
+        # Should be a listing for a single file
+        self.assertEqual(len(data), 1)
+
+        # For each file listed, check some key attributes
+        for file_info in data:
+            file_path = os.path.join(TESTSERVER_ROOT,
+                                     file_info["url"].lstrip("/"))
+            if file_info["type"] == "folder":
+                self.assertTrue(os.path.isdir(file_path))
+            else:
+                self.assertTrue(os.path.isfile(file_path))
+
+            mtime = datetime.fromtimestamp(
+                os.path.getmtime(file_path)).strftime('%d-%b-%Y %H:%M')
+
+            self.assertEqual(mtime, file_info["mtime"])
+
     def test_api_get_listing_404(self):
         url = "/api/ls/buld-info"
         response = self.client.get(url)

=== modified file 'license_protected_downloads/views.py'
--- license_protected_downloads/views.py	2013-03-11 15:31:31 +0000
+++ license_protected_downloads/views.py	2013-03-18 18:32:28 +0000
@@ -68,23 +68,23 @@
     files.sort()
     listing = []
 
-    for file in files:
-        if _hidden_file(file):
+    for file_name in files:
+        if _hidden_file(file_name):
             continue
 
-        name = file
-        file = os.path.join(path, file)
+        name = file_name
+        file_name = os.path.join(path, file_name)
 
-        if os.path.exists(file):
+        if os.path.exists(file_name):
             mtime = datetime.fromtimestamp(
-                os.path.getmtime(file)).strftime('%d-%b-%Y %H:%M')
+                os.path.getmtime(file_name)).strftime('%d-%b-%Y %H:%M')
         else:
             # If the file we are looking at doesn't exist (broken symlink for
             # example), it doesn't have a mtime.
             mtime = 0
 
         target_type = "other"
-        if os.path.isdir(file):
+        if os.path.isdir(file_name):
             target_type = "folder"
         else:
             type_tuple = guess_type(name)
@@ -92,8 +92,8 @@
                 if type_tuple[0].split('/')[0] == "text":
                     target_type = "text"
 
-        if os.path.exists(file):
-            size = os.path.getsize(file)
+        if os.path.exists(file_name):
+            size = os.path.getsize(file_name)
         else:
             # If the file we are looking at doesn't exist (broken symlink for
             # example), it doesn't have a size
@@ -512,11 +512,22 @@
     target_type = result[0]
     path = result[1]
 
-    if target_type == "dir":
+    if target_type:
+        if target_type == "file":
+            file_url = url
+            if file_url[0] != "/":
+                file_url = "/" + file_url
+            path = os.path.dirname(path)
+            url = os.path.dirname(url)
+
         listing = dir_list(url, path)
 
         clean_listing = []
         for entry in listing:
+            if target_type == "file" and file_url != entry["url"]:
+                # If we are getting a listing for a single file, skip the rest
+                continue
+
             if len(entry["license_list"]) == 0:
                 entry["license_list"] = ["Open"]
 
@@ -545,7 +556,7 @@
 
     if target_type == "dir":
         data = json.dumps({"licenses":
-                           ["File not found."]})
+                           ["ERROR: License only shown for a single file."]})
     else:
         license_digest_list = is_protected(path)
         license_list = License.objects.all_with_hashes(license_digest_list)

=== modified file 'scripts/download.py'
--- scripts/download.py	2013-03-12 14:59:02 +0000
+++ scripts/download.py	2013-03-18 18:32:28 +0000
@@ -6,84 +6,108 @@
 import urllib2
 import os
 from html2text import html2text
-
-# Example of how to use the API to download all files in a directory. This is
-# written as one procedural script without functions
-directory_url = "http://localhost:8001/build-info"
-
-# Generate the URL that will return the license information. This is the URL
-# of the file with /api/license prepended to the path.
-
-# Unfortunately urlsplit returns an immutable object. Convert it to an array
-# so we can modify the path section (index 2)
-parsed_url = [c for c in urlparse.urlsplit(directory_url)]
-url_path_section = parsed_url[2]
-
-parsed_url[2] = "/api/ls" + url_path_section
-listing_url = urlparse.urlunsplit(parsed_url)
-
-u = urllib2.urlopen(listing_url)
-data = json.loads(u.read())["files"]
-
-for file_info in data:
-    if file_info["type"] == "folder":
-        # Skip folders...
-        continue
-
-    parsed_url[2] = "/api/license" + file_info["url"]
-    license_url = urlparse.urlunsplit(parsed_url)
-
-    parsed_url[2] = file_info["url"]
-    file_url = urlparse.urlunsplit(parsed_url)
-
-    # Get the licenses. They are returned as a JSON document in the form:
-    # {"licenses":
-    #  [{"text": "<license text>", "digest": "<digest of license>"},
-    #   {"text": "<license text>", "digest": "<digest of license>"},
-    #   ...
-    # ]}
-    # Each license has a digest associated with it.
-    u = urllib2.urlopen(license_url)
-    data = json.loads(u.read())["licenses"]
-
-    if data[0] == "Open":
-        headers = {}
+import sys
+import xdg.BaseDirectory as xdgBaseDir
+
+
+def download(api_urls, accepted_licenses):
+    """Example of how to use the API to download a/all files in a directory."""
+
+    # Get listing for file(s) pointed to by URL we were given
+    request = urllib2.urlopen(api_urls.ls())
+    listing = json.loads(request.read())["files"]
+
+    for file_info in listing:
+        if file_info["type"] == "folder":
+            # Skip folders...
+            continue
+
+        # Get the licenses. They are returned as a JSON document in the form:
+        # {"licenses":
+        #  [{"text": "<license text>", "digest": "<digest of license>"},
+        #   {"text": "<license text>", "digest": "<digest of license>"},
+        #   ...
+        # ]}
+        # Each license has a digest associated with it.
+        request = urllib2.urlopen(api_urls.license(file_info["url"]))
+        licenses = json.loads(request.read())["licenses"]
+
+        if licenses[0] == "Open":
+            headers = {}
+        else:
+            # Present each license to the user...
+            for lic in licenses:
+                if lic["digest"] not in accepted_licenses:
+                    # Licenses are stored as HTML. Convert them to markdown
+                    # (text) and print it to the terminal.
+                    print html2text(lic["text"])
+
+                    # Ask the user if they accept the license. If they don't we
+                    # terminate the script.
+                    user_response = raw_input(
+                                        "Do you accept this license? (y/N)")
+                    if user_response != "y":
+                        exit(1)
+
+                    # Remember this license acceptance for another download.
+                    accepted_licenses.append(lic["digest"])
+
+            # To accept a license, place the digest in the LICENSE_ACCEPTED
+            # header. For multiple licenses, they are stored space separated.
+            digests = [lic["digest"] for lic in licenses]
+            headers = {"LICENSE_ACCEPTED": " ".join(digests)}
+
+        # Once the header has been generated, just download the file.
+        req = urllib2.urlopen(urllib2.Request(api_urls.file(file_info["url"]),
+                                              headers=headers))
+        with open(os.path.basename(file_info["url"]), 'wb') as fp:
+            shutil.copyfileobj(req, fp)
+
+
+class ApiUrls():
+    """Since we want to manipulate URLS, but urlsplit returns an immutable
+    object this is a convenience object to perform the manipulations for us"""
+    def __init__(self, input_url):
+        self.parsed_url = [c for c in urlparse.urlsplit(input_url)]
+        self.path = self.parsed_url[2]
+
+    def ls(self, path=None):
+        if not path:
+            path = self.path
+        self.parsed_url[2] = "/api/ls" + path
+        return urlparse.urlunsplit(self.parsed_url)
+
+    def license(self, path):
+        self.parsed_url[2] = "/api/license" + path
+        return urlparse.urlunsplit(self.parsed_url)
+
+    def file(self, path):
+        self.parsed_url[2] = path
+        return urlparse.urlunsplit(self.parsed_url)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+    # Check that a URL has been supplied.
+        print >> sys.stderr, "Usage: download.py <URL>"
+        exit(1)
+
+    accepted_licenses_path = os.path.join(xdgBaseDir.xdg_data_home,
+                                          "linaro",
+                                          "accepted_licenses")
+
+    # Later we ask the user to accept each license in turn. Store which
+    # licenses are accepted so the user only has to accept them once.
+    if os.path.isfile(accepted_licenses_path):
+        with open(accepted_licenses_path) as accepted_licenses_file:
+            accepted_licenses = accepted_licenses_file.read().split()
     else:
-        # If this were a command line client designed to ask the user to accept
-        # each license, you could use this code to ask the user to accept each
-        # license in turn. In this example we store which licenses are accepted
-        # so the user only has to accept them once.
-        if os.path.isfile("accepted_licenses"):
-            with open("accepted_licenses") as accepted_licenses_file:
-                accepted_licenses = accepted_licenses_file.read().split()
-        else:
-            accepted_licenses = []
-
-        # Present each license to the user...
-        for d in data:
-            if d["digest"] not in accepted_licenses:
-                # Licenses are stored as HTML. Convert them to markdown (text)
-                # and print it to the terminal.
-                print html2text(d["text"])
-
-                # Ask the user if they accept the license. If they don't we
-                # terminate the script.
-                user_response = raw_input("Do you accept this license? (y/N)")
-                if user_response != "y":
-                    exit(1)
-
-                accepted_licenses.append(d["digest"])
-
-        # Store the licenses that the user accepted
-        with open("accepted_licenses", "w") as accepted_licenses_file:
-            accepted_licenses_file.write(" ".join(accepted_licenses))
-
-        # To accept a license, place the digest in the LICENSE_ACCEPTED header.
-        # For multiple licenses, they are stored space separated.
-        digests = [d["digest"] for d in data]
-        headers = {"LICENSE_ACCEPTED": " ".join(digests)}
-
-    # Once the header has been generated, just download the file.
-    req = urllib2.urlopen(urllib2.Request(file_url, headers=headers))
-    with open(os.path.basename(parsed_url[2]), 'wb') as fp:
-        shutil.copyfileobj(req, fp)
+        accepted_licenses = []
+
+    api_urls = ApiUrls(sys.argv[1])
+
+    download(api_urls, accepted_licenses)
+
+    # Store the licenses that the user accepted
+    with open(accepted_licenses_path, "w") as accepted_licenses_file:
+        accepted_licenses_file.write(" ".join(accepted_licenses))