Merge ~nacc/git-ubuntu:script-add-options into git-ubuntu:master

[Nish Aravamudan]

General fixes/cleanups again.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:0b10138cd2029a9727ead8407d1bfd302ad5c186
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/332/
Executed test runs:
    SUCCESS: VM Setup
    SUCCESS: Build
    SUCCESS: Unit Tests
    SUCCESS: Integration Tests
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/332/rebuild

review: Approve (continuous-integration)

[Nish Aravamudan (nacc)]

Self-approving.

review: Approve

[Nish Aravamudan (nacc)]

Self-approving.

review: Approve

diff --git a/bin/import-source-packages.py b/bin/import-source-packages.py
index 5c52948..aa51c63 100755
--- a/bin/import-source-packages.py
+++ b/bin/import-source-packages.py
@@ -150,6 +150,7 @@ def import_new_published_sources(
     ret = scriptutils.pool_map_import_srcpkg(
         num_workers=num_workers,
         dry_run=dry_run,
+        reimport=False,
         pkgnames=filtered_pkgnames,
     )
 
diff --git a/bin/source-package-walker.py b/bin/source-package-walker.py
index 28e2425..0395b18 100755
--- a/bin/source-package-walker.py
+++ b/bin/source-package-walker.py
@@ -43,6 +43,7 @@ def import_all_published_sources(
     phasing_main,
     phasing_universe,
     dry_run,
+    reimport,
 ):
     """Import all publishes satisfying a {white,black}list and phasing
 
@@ -53,6 +54,7 @@ def import_all_published_sources(
     phasing_main - a integer percentage of all packages in main to import
     phasing_universe - a integer percentage of all packages in universe to import
     dry_run - a boolean to indicate a dry-run operation
+    reimport - a boolean to indicate source packages should be reimported
 
     Returns:
     A tuple of two lists, the first containing the names of all
@@ -88,7 +90,6 @@ def import_all_published_sources(
                         url + compression
                     ) as source_url_file:
                         with opener(source_url_file, mode='r') as sources:
-                            print(url + compression)
                             for src in Sources.iter_paragraphs(
                                 sources,
                                 use_apt_pkg=False,
@@ -124,6 +125,7 @@ def import_all_published_sources(
     return scriptutils.pool_map_import_srcpkg(
         num_workers=num_workers,
         dry_run=dry_run,
+        reimport=reimport,
         pkgnames=pkgnames,
     )
 
@@ -135,6 +137,7 @@ def main(
     phasing_universe=scriptutils.DEFAULTS.phasing_universe,
     dry_run=scriptutils.DEFAULTS.dry_run,
     use_whitelist=scriptutils.DEFAULTS.use_whitelist,
+    reimport=scriptutils.DEFAULTS.reimport,
 ):
     """Main entry point to the script
 
@@ -151,6 +154,7 @@ def main(
     dry_run - a boolean to indicate a dry-run operation
     use_whitelist - a boolean to control whether the whitelist data is
     used
+    reimport - a boolean to indicate source packages should be reimported
 
     use_whitelist exists because during the rampup of imports, we want
     to import the whitelist packages and the phased packages. But after
@@ -189,6 +193,7 @@ def main(
         phasing_main,
         phasing_universe,
         dry_run,
+        reimport,
     )
     print(
         "Imported %d source packages:\n%s" % (
@@ -220,7 +225,7 @@ def cli_main():
         action='store_false',
         dest='use_whitelist',
         help="Do not process packages in the whitelist",
-        default=not scriptutils.DEFAULTS.use_whitelist,
+        default=scriptutils.DEFAULTS.use_whitelist,
     )
     parser.add_argument(
         '--whitelist',
@@ -252,6 +257,12 @@ def cli_main():
         help="Simulate operation but do not actually do anything",
         default=scriptutils.DEFAULTS.dry_run,
     )
+    parser.add_argument(
+        '--reimport',
+        action='store_true',
+        help="Reimport source packages",
+        default=scriptutils.DEFAULTS.reimport,
+    )
 
     args = parser.parse_args()
 
@@ -263,6 +274,7 @@ def cli_main():
         phasing_universe=args.phasing_universe,
         dry_run=args.dry_run,
         use_whitelist=args.use_whitelist,
+        reimport=args.reimport,
     )
 
 if __name__ == '__main__':
diff --git a/gitubuntu/scriptutils.py b/gitubuntu/scriptutils.py
index 58843a2..6806bbf 100644
--- a/gitubuntu/scriptutils.py
+++ b/gitubuntu/scriptutils.py
@@ -35,6 +35,7 @@ Defaults = namedtuple(
         'phasing_main',
         'dry_run',
         'use_whitelist',
+        'reimport',
     ],
 )
 
@@ -52,6 +53,7 @@ DEFAULTS = Defaults(
     phasing_main=1,
     dry_run=False,
     use_whitelist=True,
+    reimport=False,
 )
 
 
@@ -101,12 +103,13 @@ def should_import_srcpkg(
     return False
 
 
-def import_srcpkg(pkgname, dry_run):
+def import_srcpkg(pkgname, dry_run, reimport):
     """Invoke git ubuntu import on @pkgname
 
     Arguments:
     pkgname - string name of a source package
     dry_run - a boolean to indicate a dry-run operation
+    reimport - a boolean to indicate a reimport operation
 
     Returns:
     A tuple of a string and a boolean, where the boolean is the success
@@ -123,8 +126,10 @@ def import_srcpkg(pkgname, dry_run):
             'import',
             '-l',
             'usd-importer-bot',
-            pkgname,
         ]
+        if reimport:
+           cmd.append('--reimport')
+        cmd.append(pkgname)
         try:
             print(' '.join(cmd))
             if not dry_run:
@@ -163,6 +168,7 @@ def setup_git_config(
 def pool_map_import_srcpkg(
     num_workers,
     dry_run,
+    reimport,
     pkgnames,
 ):
     """Use a multiprocessing.Pool to parallel import source packages
@@ -170,12 +176,14 @@ def pool_map_import_srcpkg(
     Arguments:
     num_workers - integer number of worker processes to use
     dry_run - a boolean to indicate a dry-run operation
+    reimport- a boolean to indicate a reimport operation
     pkgnames - a list of string names of source packages
     """
     map_args = [
         functools.partial(
             import_srcpkg,
             dry_run=dry_run,
+            reimport=reimport,
         ),
         pkgnames,
     ]
diff --git a/gitubuntu/scriptutils_test.py b/gitubuntu/scriptutils_test.py
new file mode 100755
index 0000000..63d72ff
--- /dev/null
+++ b/gitubuntu/scriptutils_test.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+import pytest
+
+import gitubuntu.scriptutils as target
+
[e-mail]('pkgname, component, whitelist, blacklist, phasing_main, phasing_universe, expected', [
+    ('jimtcl', 'universe', [], [], 1, 0, False),
+    ('jimtcl', 'universe', [], [], 1, 100, True),
+    ('jimtcl', 'universe', ['jimtcl',], [], 1, 0, True),
+    ('linux', 'main', [], ['linux',], 100, 0, False),
+    ('jimtcl', 'main', ['php7.2', 'apache2', 'bind9'], [], 0, 0, False),
+    ('php7.2', 'main', ['php7.2', 'apache2', 'bind9'], [], 0, 0, True),
+])
+def test_should_import_srcpkg(
+    pkgname,
+    component,
+    whitelist,
+    blacklist,
+    phasing_main,
+    phasing_universe,
+    expected,
+):
+    assert target.should_import_srcpkg(
+        pkgname=pkgname,
+        component=component,
+        whitelist=whitelist,
+        blacklist=blacklist,
+        phasing_main=phasing_main,
+        phasing_universe=phasing_universe,
+    ) == expected
diff --git a/snap/wrappers/git-ubuntu-self-test b/snap/wrappers/git-ubuntu-self-test
index 04789b4..f57cde7 100755
--- a/snap/wrappers/git-ubuntu-self-test
+++ b/snap/wrappers/git-ubuntu-self-test
@@ -8,7 +8,7 @@ if [ $PYLINT_RC == 0 ]; then
 fi
 
 TMPFILE=$(mktemp)
-COVERAGE_FILE=$TMPFILE python3 -m pytest -p no:cacheprovider --cov=gitubuntu $SNAP/lib/python3.6/site-packages/gitubuntu/*
+COVERAGE_FILE=$TMPFILE python3 -m pytest -p no:cacheprovider --cov=$SNAP/lib/python3.6/site-packages/gitubuntu $SNAP/lib/python3.6/site-packages/gitubuntu/*
 PYTEST_RC=$?
 unlink $TMPFILE