Hi,

I have stuff running on port 8080, I modified
test_click_through_license.py to get a free port from the OS:

=== modified file 'testing/test_click_through_license.py'
--- testing/test_click_through_license.py	2012-01-13 08:02:11 +0000
+++ testing/test_click_through_license.py	2012-01-13 11:04:58 +0000
@@ -2,14 +2,13 @@

 import re
 import os
-import sys
 import shutil
 import shlex
 import subprocess
+import socket

 from testtools import TestCase
 from testtools.matchers import Mismatch
-from testtools.matchers import MatchesAny
 from filefetcher import LicenseProtectedFileFetcher

 fetcher = LicenseProtectedFileFetcher()
@@ -19,7 +18,7 @@
 local_rewrite = 'RewriteCond %{REMOTE_ADDR} 127.0.0.1 [OR]'

 host = 'http://127.0.0.1'
-port = '8080'
+port = '0'  # 0 == Pick a free port.
 samsung_license_path = '/licenses/samsung-v2.html'
 ste_license_path = '/licenses/ste.html'
 samsung_test_file = '/android/~linaro-android/staging-origen/test.txt'
@@ -55,7 +54,12 @@
     @classmethod
     def setUpClass(cls):
         global host
-        if port != '80':
+        global port
+        if port == '0':
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.bind(('127.0.0.1', 0))
+            port = str(s.getsockname()[1])
+            s.close()
             host = host + ':' + port
         shutil.copy("%s/apache2.conf.tmpl" % srvroot, "%s/apache2.conf" %
                 srvroot)

I think you missed checking in a file: licenses/nolicense.html - after
the above modification I have failing tests because of this.

I also deleted a few unused libraries. If you use the pep8 tool you
will find a string of issues that it has found with style. If you use
gedit you can install pep8 and pylint they will nag you about all
sorts of things :-) If you use gedit and install pylint, activate all
the plugins that ship with it and you will find that when you save a
python file you get the pep8 output as a sidebar, which is quite
useful. We also have a license for pycharm, which is a nice Python
IDE:

https://wiki.linaro.org/Internal/Licenses/PyCharm

This has some stuff built in that makes hacking easier, but doesn't do
quite as much style nagging.

Nice start though - thanks for testing this.

James

On 12 January 2012 14:39, Georgy Redkozubov
<email address hidden> wrote:
> Georgy Redkozubov has proposed merging lp:~gesha/linaro-license-protection/clickthrough-testing into lp:linaro-license-protection.
>
> Requested reviews:
>  Linaro Infrastructure (linaro-infrastructure)
>
> For more details, see:
> https://code.launchpad.net/~gesha/linaro-license-protection/clickthrough-testing/+merge/88387
>
> Added tests for click-through license feature using Testtools and Testrepository.
> --
> https://code.launchpad.net/~gesha/linaro-license-protection/clickthrough-testing/+merge/88387
> Your team Linaro Infrastructure is requested to review the proposed merge of lp:~gesha/linaro-license-protection/clickthrough-testing into lp:linaro-license-protection.
>
> === modified file '.htaccess'
> --- .htaccess   2011-12-19 16:57:20 +0000
> +++ .htaccess   2012-01-12 14:37:26 +0000
> @@ -6,6 +6,8 @@
>  ## Let internal hosts through always.
>  # android-build.linaro.org.
>  RewriteCond %{REMOTE_ADDR} 50.17.250.69 [OR]
> +# ci.linaro.org.
> +RewriteCond %{REMOTE_ADDR} 50.17.200.206 [OR]
>  # validation.linaro.org.
>  RewriteCond %{REMOTE_ADDR} 213.123.120.124
>  RewriteRule .* - [L]
>
> === added file '.testr.conf'
> --- .testr.conf 1970-01-01 00:00:00 +0000
> +++ .testr.conf 2012-01-12 14:37:26 +0000
> @@ -0,0 +1,3 @@
> +[DEFAULT]
> +test_command=python -m subunit.run $IDLIST
> +test_id_list_default=testing.test_suite
>
> === added directory 'testing'
> === added file 'testing/__init__.py'
> --- testing/__init__.py 1970-01-01 00:00:00 +0000
> +++ testing/__init__.py 2012-01-12 14:37:26 +0000
> @@ -0,0 +1,11 @@
> +import os
> +import unittest
> +
> +def test_suite():
> +    module_names = [
> +        'testing.test_click_through_license',
> +        ]
> +    loader = unittest.TestLoader()
> +    suite = loader.loadTestsFromNames(module_names)
> +    return suite
> +
>
> === added file 'testing/apache2.conf.tmpl'
> --- testing/apache2.conf.tmpl   1970-01-01 00:00:00 +0000
> +++ testing/apache2.conf.tmpl   2012-01-12 14:37:26 +0000
> @@ -0,0 +1,104 @@
> +ServerRoot ""
> +
> +LockFile click_through_license.lock
> +PidFile click_through_license.pid
> +
> +Timeout 300
> +
> +KeepAlive On
> +
> +KeepAliveTimeout 5
> +
> +Listen
> +
> +ServerAdmin <email address hidden>
> +
> +#ServerName www.example.com:80
> +
> +DocumentRoot ""
> +
> +AccessFileName .htaccess
> +
> +LoadModule authz_host_module /usr/lib/apache2/modules/mod_authz_host.so
> +LoadModule cache_module /usr/lib/apache2/modules/mod_cache.so
> +LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
> +LoadModule autoindex_module /usr/lib/apache2/modules/mod_autoindex.so
> +LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
> +
> +User daemon
> +Group daemon
> +
> +<IfModule mpm_prefork_module>
> +    StartServers          5
> +    MinSpareServers       5
> +    MaxSpareServers      10
> +    MaxClients          150
> +    MaxRequestsPerChild   0
> +</IfModule>
> +
> +<IfModule mpm_worker_module>
> +    StartServers          2
> +    MinSpareThreads      25
> +    MaxSpareThreads      75.
> +    ThreadLimit          64
> +    ThreadsPerChild      25
> +    MaxClients          150
> +    MaxRequestsPerChild   0
> +</IfModule>
> +
> +<IfModule mpm_event_module>
> +    StartServers          2
> +    MinSpareThreads      25
> +    MaxSpareThreads      75.
> +    ThreadLimit          64
> +    ThreadsPerChild      25
> +    MaxClients          150
> +    MaxRequestsPerChild   0
> +</IfModule>
> +
> +<Directory />
> +    Options +FollowSymLinks
> +    AllowOverride All
> +    Order deny,allow
> +    Deny from all
> +</Directory>
> +
> +<Directory "">
> +    Options Indexes +FollowSymLinks MultiViews
> +    AllowOverride All
> +    Order allow,deny
> +    Allow from all
> +</Directory>
> +
> +<IfModule dir_module>
> +    DirectoryIndex index.html
> +</IfModule>
> +
> +<FilesMatch "^\.ht">
> +    Order allow,deny
> +    Deny from all
> +    Satisfy All
> +</FilesMatch>
> +
> +ErrorLog "click_through_license_error.log"
> +
> +LogLevel warn
> +
> +<IfModule log_config_module>
> +    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
> +    LogFormat "%h %l %u %t \"%r\" %>s %b" common
> +
> +    <IfModule logio_module>
> +      LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
> +    </IfModule>
> +
> +    CustomLog "click_through_license_access.log" common
> +
> +</IfModule>
> +
> +DefaultType text/plain
> +
> +IndexOptions FancyIndexing VersionSort HTMLTable NameWidth=* DescriptionWidth=* Charset=UTF-8
> +ReadmeName README.html
> +HeaderName HEADER.html
> +IndexIgnore .??* *~ *# RCS CVS *,v *,t.
>
> === added file 'testing/filefetcher.py'
> --- testing/filefetcher.py      1970-01-01 00:00:00 +0000
> +++ testing/filefetcher.py      2012-01-12 14:37:26 +0000
> @@ -0,0 +1,128 @@
> +#!/usr/bin/env python
> +
> +# Changes required to address EULA for the origen hwpacks
> +
> +import argparse
> +import os
> +import pycurl
> +import re
> +import urlparse
> +
> +class LicenseProtectedFileFetcher:
> +    """Fetch a file from the web that may be protected by a license redirect
> +
> +    This is designed to run on snapshots.linaro.org. License HTML file are in
> +    the form:
> +
> +    <vendor>.html has a link to <vendor>-accept.html
> +
> +    If self.get is pointed at a file that has to go through one of these
> +    licenses, it should be able to automatically accept the license and
> +    download the file.
> +
> +    Once a license has been accepted, it will be used for all following
> +    downloads.
> +
> +    If self.close() is called before the object is deleted, cURL will store
> +    the license accept cookie to cookies.txt, so it can be used for later
> +    downloads.
> +
> +    """
> +    def __init__(self):
> +        """Set up cURL"""
> +        self.curl = pycurl.Curl()
> +        self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
> +        self.curl.setopt(pycurl.WRITEFUNCTION, self._write_body)
> +        self.curl.setopt(pycurl.HEADERFUNCTION, self._write_header)
> +        self.curl.setopt(pycurl.COOKIEFILE, "cookies.txt")
> +        self.curl.setopt(pycurl.COOKIEJAR, "cookies.txt")
> +
> +    def _get(self, url):
> +        """Clear out header and body storage, fetch URL, filling them in."""
> +        self.curl.setopt(pycurl.URL, url)
> +
> +        self.body = ""
> +        self.header = ""
> +
> +        self.curl.perform()
> +
> +    def get(self, url, ignore_license=False, accept_license=True):
> +        """Fetch the requested URL, ignoring license at all or
> +        accepting or declining licenses, returns file body.
> +
> +        Fetches the file at url. If a redirect is encountered, it is
> +        expected to be to a license that has an accept or decline link.
> +        Follow that link, then download original file or nolicense notice.
> +
> +        """
> +        self._get(url)
> +
> +        if ignore_license:
> +            return self.body
> +
> +        location = self._get_location()
> +        if location:
> +            # Off to the races - we have been redirected.
> +            # Expect to find a link to self.location with -accepted or
> +            # -declined inserted before the .html,
> +            # i.e. ste.html -> ste-accepted.html
> +
> +            # Get the file from the URL (full path)
> +            file = urlparse.urlparse(location).path
> +
> +            # Get the file without the rest of the path
> +            file = os.path.split(file)[-1]
> +
> +            # Look for a link with accepted.html or declined.html
> +            # in the page name. Follow it.
> +            new_file = None
> +            for line in self.body.splitlines():
> +                if accept_license:
> +                    link_search = re.search("""href=.*?["'](.*?-accepted.html)""",
> +                                        line)
> +                else:
> +                    link_search = re.search("""href=.*?["'](.*?-declined.html)""",
> +                                        line)
> +                if link_search:
> +                    # Have found license decline URL!
> +                    new_file = link_search.group(1)
> +
> +            if new_file:
> +                # accept or decline the license...
> +                next_url = re.sub(file, new_file, location)
> +                self._get(next_url)
> +
> +                # The above get *should* take us to the file requested via
> +                # a redirect. If we manually need to follow that redirect,
> +                # do that now.
> +
> +                if accept_license and self._get_location():
> +                    # If we haven't been redirected to our original file,
> +                    # we should be able to just download it now.
> +                    self._get(url)
> +
> +        return self.body
> +
> +    def _search_header(self, field):
> +        """Search header for the supplied field, return field / None"""
> +        for line in self.header.splitlines():
> +            search = re.search(field + ":\s+(.*?)$", line)
> +            if search:
> +                return search.group(1)
> +        return None
> +
> +    def _get_location(self):
> +        """Return content of Location field in header / None"""
> +        return self._search_header("Location")
> +
> +    def _write_body(self, buf):
> +        """Used by curl as a sink for body content"""
> +        self.body += buf
> +
> +    def _write_header(self, buf):
> +        """Used by curl as a sink for header content"""
> +        self.header += buf
> +
> +    def close(self):
> +        """Wrapper to close curl - this will allow curl to write out cookies"""
> +        self.curl.close()
>
> === added file 'testing/mime.types'
> --- testing/mime.types  1970-01-01 00:00:00 +0000
> +++ testing/mime.types  2012-01-12 14:37:26 +0000
> @@ -0,0 +1,6 @@
> +text/css                                       css
> +text/directory
> +text/html                                      html htm shtml
> +text/plain                                     asc txt text pot brf
> +text/uri-list
> +
>
> === added file 'testing/test_click_through_license.py'
> --- testing/test_click_through_license.py       1970-01-01 00:00:00 +0000
> +++ testing/test_click_through_license.py       2012-01-12 14:37:26 +0000
> @@ -0,0 +1,220 @@
> +#!/usr/bin/env python
> +
> +import re
> +import os
> +import sys
> +import shutil
> +import shlex
> +import subprocess
> +
> +from testtools import TestCase
> +from testtools.matchers import Mismatch
> +from testtools.matchers import MatchesAny
> +from filefetcher import LicenseProtectedFileFetcher
> +
> +fetcher = LicenseProtectedFileFetcher()
> +cwd = os.getcwd()
> +docroot = cwd
> +srvroot = os.path.abspath(os.path.join(*([cwd] + ['testing'])))
> +local_rewrite = 'RewriteCond %{REMOTE_ADDR} 127.0.0.1 [OR]'
> +
> +# Now not all tests pass in usermode, .htaccess file needs to be fixed
> +# To successfully run all tests use
> +#host = 'http://127.0.0.1'
> +# and run tests with 'sudo'.
> +
> +host = 'http://127.0.0.1'
> +port = '8080'
> +samsung_license_path = '/licenses/samsung-v2.html'
> +ste_license_path = '/licenses/ste.html'
> +samsung_test_file = '/android/~linaro-android/staging-origen/test.txt'
> +ste_test_file = '/android/~linaro-android/staging-snowball/test.txt'
> +not_protected_test_file = '/android/~linaro-android/staging-panda/test.txt'
> +
> +
> +class Contains(object):
> +    '''Match if a string contains substring'''
> +    def __init__(self, substr):
> +        self.substr = substr
> +
> +    def __str__(self):
> +        return 'Contains(%s)' % (self.substr,)
> +
> +    def match(self, actual):
> +        for line in actual.splitlines():
> +            res = re.search(self.substr, line)
> +            if res:
> +                return None
> +        return Mismatch("Initial string doesn't contain substring (%s)" %
> +                self.substr)
> +
> +class CommandNotFoundException(Exception):
> +    ''' Unable to find command '''
> +
> +class NonZeroReturnValueException(Exception):
> +    ''' Command exited with nonzero return value '''
> +
> +class TestLicense(TestCase):
> +    '''Tests for accessing files and directories with license protection'''
> +
> +    @classmethod
> +    def setUpClass(cls):
> +        global host
> +        if port != '80':
> +            host = host + ':' + port
> +        shutil.copy("%s/apache2.conf.tmpl" % srvroot, "%s/apache2.conf" %
> +                srvroot)
> +        shutil.copy("%s/.htaccess" % docroot, "%s/dothtaccess" % docroot)
> +        subprocess.Popen(['sed', '-i' , 's/ServerRoot \"\"/ServerRoot \"%s\"/' % srvroot.replace('/', '\/'),
> +            '%s/apache2.conf' % srvroot], stdout=open('/dev/null', 'w'),
> +            stderr=subprocess.STDOUT).wait()
> +        subprocess.Popen(['sed', '-i' , 's/DocumentRoot \"\"/DocumentRoot \"%s\"/' % docroot.replace('/', '\/'),
> +            '%s/apache2.conf' % srvroot], stdout=open('/dev/null', 'w'),
> +            stderr=subprocess.STDOUT).wait()
> +        subprocess.Popen(['sed', '-i' , 's/Directory \"\"/Directory \"%s\"/' % docroot.replace('/', '\/'),
> +            '%s/apache2.conf' % srvroot], stdout=open('/dev/null', 'w'),
> +            stderr=subprocess.STDOUT).wait()
> +        subprocess.Popen(['sed', '-i' , 's/Listen/Listen %s/' % port,
> +            '%s/apache2.conf' % srvroot], stdout=open('/dev/null', 'w'),
> +            stderr=subprocess.STDOUT).wait()
> +        if subprocess.Popen(['which', 'apache2'], stdout=open('/dev/null', 'w'),
> +                stderr=subprocess.STDOUT).wait():
> +            raise CommandNotFoundException("apache2 command not found. Please install apache2 web server "
> +                    "and rerun tests.")
> +        args = shlex.split('apache2 -d %s -f apache2.conf -k start' % srvroot)
> +        rc = subprocess.Popen(args, stdout=open('/dev/null', 'w'),
> +                stderr=subprocess.STDOUT).wait()
> +        if rc:
> +            raise NonZeroReturnValueException("apache2 server exited with error"
> +                    " %s" % rc)
> +
> +    @classmethod
> +    def tearDownClass(cls):
> +        if os.path.exists("%s/cookies.txt" % docroot):
> +            os.unlink("%s/cookies.txt" % docroot)
> +        args = shlex.split('apache2 -d %s -f apache2.conf -k stop' % srvroot)
> +        subprocess.Popen(args, stdout=open('/dev/null', 'w'),
> +                stderr=subprocess.STDOUT).wait()
> +        if os.path.exists("%s/apache2.conf" % srvroot):
> +            os.unlink("%s/apache2.conf" % srvroot)
> +        if os.path.exists("%s/click_through_license_access.log" % srvroot):
> +            os.unlink("%s/click_through_license_access.log" % srvroot)
> +        if os.path.exists("%s/click_through_license_error.log" % srvroot):
> +            os.unlink("%s/click_through_license_error.log" % srvroot)
> +        os.rename("%s/dothtaccess" % docroot, "%s/.htaccess" % docroot)
> +
> +    def setUp(self):
> +        super(TestLicense, self).setUp()
> +        global fetcher
> +        fetcher = LicenseProtectedFileFetcher()
> +
> +    def tearDown(self):
> +        super(TestLicense, self).tearDown()
> +        if isinstance(fetcher, LicenseProtectedFileFetcher):
> +            fetcher.close()
> +        if os.path.exists("%s/cookies.txt" % docroot):
> +            os.unlink("%s/cookies.txt" % docroot)
> +
> +    def test_licensefile_directly_samsung(self):
> +        search = "Index of /"
> +        testfile = fetcher.get(host + samsung_license_path)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_licensefile_directly_ste(self):
> +        search = "Index of /"
> +        testfile = fetcher.get(host + ste_license_path)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_redirect_to_license_samsung(self):
> +        search = "SAMSUNG DEVELOPMENT TOOL KIT END-USER LICENSE AGREEMENT"
> +        testfile = fetcher.get(host + samsung_test_file, ignore_license=True)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_redirect_to_license_ste(self):
> +        search = "LIMITED LICENSE AGREEMENT FOR APPLICATION DEVELOPERS"
> +        testfile = fetcher.get(host + ste_test_file, ignore_license=True)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_decline_license_samsung(self):
> +        search = "License has not been accepted"
> +        testfile = fetcher.get(host + samsung_test_file, accept_license=False)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_decline_license_ste(self):
> +        search = "License has not been accepted"
> +        testfile = fetcher.get(host + ste_test_file, accept_license=False)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_non_protected_dirs(self):
> +        search = "This is always available."
> +        testfile = fetcher.get(host + not_protected_test_file)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_accept_license_samsung_file(self):
> +        if port != '80':
> +            self.skipTest("Needs .htaccess fixing")
> +        search = "This is a protected with click-through Samsung license."
> +        testfile = fetcher.get(host + samsung_test_file)
> +        fetcher.close()
> +        if os.path.exists("%s/cookies.txt" % docroot):
> +            os.rename("%s/cookies.txt" % docroot, "%s/cookies.samsung" % docroot)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_accept_license_samsung_dir(self):
> +        if port != '80':
> +            self.skipTest("Needs .htaccess fixing")
> +        search = "Index of /android/~linaro-android/staging-origen"
> +        testfile = fetcher.get(host + os.path.dirname(samsung_test_file))
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_accept_license_ste_file(self):
> +        if port != '80':
> +            self.skipTest("Needs .htaccess fixing")
> +        search = "This is a protected with click-through ST-E license."
> +        testfile = fetcher.get(host + ste_test_file)
> +        fetcher.close()
> +        if os.path.exists("%s/cookies.txt" % docroot):
> +            os.rename("%s/cookies.txt" % docroot, "%s/cookies.ste" % docroot)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_accept_license_ste_dir(self):
> +        if port != '80':
> +            self.skipTest("Needs .htaccess fixing")
> +        search = "Index of /android/~linaro-android/staging-snowball"
> +        testfile = fetcher.get(host + os.path.dirname(ste_test_file))
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_license_accepted_samsung(self):
> +        if port != '80':
> +            self.skipTest("Needs .htaccess fixing")
> +        search = "This is a protected with click-through Samsung license."
> +        os.rename("%s/cookies.samsung" % docroot, "%s/cookies.txt" % docroot)
> +        testfile = fetcher.get(host + samsung_test_file, ignore_license=True)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_license_accepted_ste(self):
> +        if port != '80':
> +            self.skipTest("Needs .htaccess fixing")
> +        search = "This is a protected with click-through ST-E license."
> +        os.rename("%s/cookies.ste" % docroot, "%s/cookies.txt" % docroot)
> +        testfile = fetcher.get(host + ste_test_file, ignore_license=True)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_internal_host_samsung(self):
> +        search = "This is a protected with click-through Samsung license."
> +        subprocess.Popen(['sed', '-i' , '/RewriteEngine On/ a %s' % local_rewrite,
> +            '%s/.htaccess' % docroot], stdout=open('/dev/null', 'w'),
> +            stderr=subprocess.STDOUT).wait()
> +        testfile = fetcher.get(host + samsung_test_file, ignore_license=True)
> +        shutil.copy("%s/dothtaccess" % docroot, "%s/.htaccess" % docroot)
> +        self.assertThat(testfile, Contains(search))
> +
> +    def test_internal_host_ste(self):
> +        search = "This is a protected with click-through ST-E license."
> +        subprocess.Popen(['sed', '-i' , '/RewriteEngine On/ a %s' % local_rewrite,
> +            '%s/.htaccess' % docroot], stdout=open('/dev/null', 'w'),
> +            stderr=subprocess.STDOUT).wait()
> +        testfile = fetcher.get(host + ste_test_file, ignore_license=True)
> +        shutil.copy("%s/dothtaccess" % docroot, "%s/.htaccess" % docroot)
> +        self.assertThat(testfile, Contains(search))
> +
>
>



-- 
James Tunnicliffe