Merge ~racb/git-ubuntu:prepare-upload-adjustments into git-ubuntu:master

Proposed by Robie Basak
Status: Superseded
Proposed branch: ~racb/git-ubuntu:prepare-upload-adjustments
Merge into: git-ubuntu:master
Diff against target: 19208 lines (+18908/-0) (has conflicts)
50 files modified
doc/README.md (+117/-0)
doc/SPECIFICATION (+167/-0)
doc/release-process.md (+264/-0)
gitubuntu/changelog_date_overrides.txt (+19/-0)
gitubuntu/changelog_tests/maintainer_name_inner_space (+8/-0)
gitubuntu/changelog_tests/maintainer_name_leading_space (+8/-0)
gitubuntu/changelog_tests/maintainer_name_trailing_space (+8/-0)
gitubuntu/changelog_tests/test_date_1 (+8/-0)
gitubuntu/changelog_tests/test_date_2 (+8/-0)
gitubuntu/changelog_tests/test_distribution (+8/-0)
gitubuntu/changelog_tests/test_distribution_source_1 (+8/-0)
gitubuntu/changelog_tests/test_distribution_source_2 (+8/-0)
gitubuntu/changelog_tests/test_distribution_source_3 (+8/-0)
gitubuntu/changelog_tests/test_distribution_source_4 (+8/-0)
gitubuntu/changelog_tests/test_maintainer_1 (+8/-0)
gitubuntu/changelog_tests/test_maintainer_2 (+8/-0)
gitubuntu/changelog_tests/test_maintainer_3 (+8/-0)
gitubuntu/changelog_tests/test_versions_1 (+8/-0)
gitubuntu/changelog_tests/test_versions_2 (+14/-0)
gitubuntu/changelog_tests/test_versions_3 (+26/-0)
gitubuntu/clone.py (+178/-0)
gitubuntu/git_repository.py (+3026/-0)
gitubuntu/git_repository_test.py (+1191/-0)
gitubuntu/importer.py (+2703/-0)
gitubuntu/importer_service.py (+916/-0)
gitubuntu/importer_service_broker.py (+178/-0)
gitubuntu/importer_service_poller.py (+239/-0)
gitubuntu/importer_service_poller_test.py (+66/-0)
gitubuntu/importer_service_worker.py (+311/-0)
gitubuntu/importer_test.py (+2288/-0)
gitubuntu/prepare_upload.py (+215/-0)
gitubuntu/prepare_upload_test.py (+268/-0)
gitubuntu/repo_builder.py (+450/-0)
gitubuntu/scriptutils.py (+226/-0)
gitubuntu/source-package-allowlist.txt (+2881/-0)
gitubuntu/source-package-denylist.txt (+56/-0)
gitubuntu/source_builder.py (+344/-0)
gitubuntu/source_information.py (+785/-0)
gitubuntu/source_information_test.py (+503/-0)
gitubuntu/submit.py (+252/-0)
man/man1/git-ubuntu-clone.1 (+68/-0)
man/man1/git-ubuntu-export-orig.1 (+63/-0)
man/man1/git-ubuntu-import.1 (+224/-0)
man/man1/git-ubuntu-merge.1 (+134/-0)
man/man1/git-ubuntu-queue.1 (+96/-0)
man/man1/git-ubuntu-remote.1 (+86/-0)
man/man1/git-ubuntu-submit.1 (+97/-0)
man/man1/git-ubuntu-tag.1 (+88/-0)
man/man1/git-ubuntu.1 (+217/-0)
setup.py (+40/-0)
Conflict in doc/README.md
Conflict in doc/SPECIFICATION
Conflict in doc/release-process.md
Conflict in gitubuntu/changelog_date_overrides.txt
Conflict in gitubuntu/changelog_tests/maintainer_name_inner_space
Conflict in gitubuntu/changelog_tests/maintainer_name_leading_space
Conflict in gitubuntu/changelog_tests/maintainer_name_trailing_space
Conflict in gitubuntu/changelog_tests/test_date_1
Conflict in gitubuntu/changelog_tests/test_date_2
Conflict in gitubuntu/changelog_tests/test_distribution
Conflict in gitubuntu/changelog_tests/test_distribution_source_1
Conflict in gitubuntu/changelog_tests/test_distribution_source_2
Conflict in gitubuntu/changelog_tests/test_distribution_source_3
Conflict in gitubuntu/changelog_tests/test_distribution_source_4
Conflict in gitubuntu/changelog_tests/test_maintainer_1
Conflict in gitubuntu/changelog_tests/test_maintainer_2
Conflict in gitubuntu/changelog_tests/test_maintainer_3
Conflict in gitubuntu/changelog_tests/test_versions_1
Conflict in gitubuntu/changelog_tests/test_versions_2
Conflict in gitubuntu/changelog_tests/test_versions_3
Conflict in gitubuntu/clone.py
Conflict in gitubuntu/git_repository.py
Conflict in gitubuntu/git_repository_test.py
Conflict in gitubuntu/importer.py
Conflict in gitubuntu/importer_service.py
Conflict in gitubuntu/importer_service_broker.py
Conflict in gitubuntu/importer_service_poller.py
Conflict in gitubuntu/importer_service_poller_test.py
Conflict in gitubuntu/importer_service_worker.py
Conflict in gitubuntu/importer_test.py
Conflict in gitubuntu/prepare_upload.py
Conflict in gitubuntu/prepare_upload_test.py
Conflict in gitubuntu/repo_builder.py
Conflict in gitubuntu/scriptutils.py
Conflict in gitubuntu/source-package-allowlist.txt
Conflict in gitubuntu/source-package-blacklist.txt
Conflict in gitubuntu/source-package-denylist.txt
Conflict in gitubuntu/source-package-whitelist.txt
Conflict in gitubuntu/source_builder.py
Conflict in gitubuntu/source_information.py
Conflict in gitubuntu/source_information_test.py
Conflict in gitubuntu/submit.py
Conflict in man/man1/git-ubuntu-clone.1
Conflict in man/man1/git-ubuntu-export-orig.1
Conflict in man/man1/git-ubuntu-import.1
Conflict in man/man1/git-ubuntu-merge.1
Conflict in man/man1/git-ubuntu-queue.1
Conflict in man/man1/git-ubuntu-remote.1
Conflict in man/man1/git-ubuntu-submit.1
Conflict in man/man1/git-ubuntu-tag.1
Conflict in man/man1/git-ubuntu.1
Conflict in setup.py
Reviewer Review Type Date Requested Status
Athos Ribeiro (community) Approve
Server Team CI bot continuous-integration Approve
git-ubuntu developers Pending
Review via email: mp+413881@code.launchpad.net

This proposal has been superseded by a proposal from 2023-05-25.

Commit message

Make Jenkins happy

To post a comment you must log in.
Revision history for this message
Server Team CI bot (server-team-bot) wrote :

PASSED: Continuous integration, rev:7f25e4872b2d134925d3c6768be7ece34e24f112
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/63/
Executed test runs:
    SUCCESS: VM Setup
    SUCCESS: Build
    SUCCESS: VM Reset
    SUCCESS: Unit Tests
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/63//rebuild

review: Approve (continuous-integration)
Revision history for this message
Athos Ribeiro (athos-ribeiro) wrote :

LGTM!

At first I wondered if it would be a good idea to verify the contents of the "headers" dict in "cli_printargs", but then I realized it is already being done in "push" (even though that is being done through calls to "assert" and therefore we do rely on __debug__ being set to True).

review: Approve
Revision history for this message
Robie Basak (racb) wrote :

> At first I wondered if it would be a good idea to verify the contents of the "headers" dict in "cli_printargs", but then I realized it is already being done in "push"

Right - it's more tedious to test from something closer to the CLI interface, so I tend to test the inner bits more directly.

> (even though that is being done through calls to "assert" and therefore we do rely on __debug__ being set to True).

I'm not sure we're on the same page here. What I mean above is that I'm testing the contents of the "headers" dict in prepare_upload_test.py using tests that test the behaviour of the prepare_upload.py::push(). assert statements from there are the usual pattern for pytest-based test suites and test suites are expected to always run with asserts enabled.

There are separate assert statements in the code itself in prepare_upload.py::push(), but these are there to state (and runtime verify when asserts are enabled) invariants that would help fail earlier and more helpfully if there is a bug somewhere. But I intend to test every actual case from the test suite in prepare_upload_test.py. If there's something you spotted that you think isn't being tested from there, I'd like to add it!

Revision history for this message
Athos Ribeiro (athos-ribeiro) wrote :

> I'm not sure we're on the same page here. What I mean above is that I'm testing the contents of the "headers" dict in prepare_upload_test.py using tests that test the behaviour of the prepare_upload.py::push(). assert statements from there are the usual pattern for pytest-based test suites and test suites are expected to always run with asserts enabled.

I was referring to the assert calls in `gitubuntu/prepare_upload.py`, as you mentioned later in your reply. I was just wondering why they are not explicitly raising exceptions instead. The comments in that file do justify them though:

> # However, we don't know of any actual case when this might happen, so
  # these are assertions rather than fully UX-compliant error paths.

> If there's something you spotted that you think isn't being tested from there, I'd like to add it!

The only thing that came to mind was the regular expression to parse the git URL. Although it is based on the one present in `https://git.launchpad.net/launchpad/tree/lib/lp/app/validators/name.py`, it is slightly different since it allows the username to start with one of the allowed special characters. Parametrizing that test to include other valid username examples could prevent mistakes in future changes to that regex. However, this looks simple enough and perhaps the effort is just not worth here.

This LGTM :)

1f99957... by Lena Voytek

Updates for inclusive naming

Edit filenames, variables, and comments to match inclusive naming
standards. The user experience will remain the same with this
update. However, when allowing and denying specific packages,
additions must be placed in source-package-allowlist.txt and
source-package-denylist.txt instead of source-package-whitelist.txt
and source-package-blacklist.txt.

Signed-off-by: Lena Voytek <email address hidden>

ef3b3a9... by Robie Basak

Merge remote-tracking branch 'lvoytek/master'

57e5776... by Robie Basak

Update maintainer email address

<email address hidden> no longer exists. Use
<email address hidden> instead.

002e452... by Robie Basak

Update email address to request an import

This is for the message printed when a repository is not found on "git
ubuntu clone".

Since <email address hidden> no longer exists, we'll use
<email address hidden> instead.

2f0e855... by Robie Basak

Update email address default used in tests

Since <email address hidden> no longer exists, we'll use
<email address hidden> instead. This shouldn't affect
production code since the repo_builder and source_builder modules are
only used in tests.

5aad111... by Robie Basak

Update default bot account name

usd-importer-bot is now renamed to git-ubuntu-bot as part of catching up
the project rename to git-ubuntu.

5b0868f... by Sergio Durigan Junior

Accept ref names containing plus sign

Currently, if the ref name contains a plus sign, git ubuntu will fail
due to the following assertion error:

Traceback (most recent call last):
  File "/snap/git-ubuntu/891/usr/bin/git-ubuntu", line 11, in <module>
    load_entry_point('gitubuntu==1.0', 'console_scripts', 'git-ubuntu')()
  File "/snap/git-ubuntu/891/usr/lib/python3/dist-packages/gitubuntu/__main__.py", line 270, in main
    sys.exit(args.func(args))
  File "/snap/git-ubuntu/891/usr/lib/python3/dist-packages/gitubuntu/prepare_upload.py", line 170, in cli_printargs
    headers = push(
  File "/snap/git-ubuntu/891/usr/lib/python3/dist-packages/gitubuntu/prepare_upload.py", line 118, in push
    assert gitubuntu.importer.VCS_GIT_REF_VALIDATION.fullmatch(ref.name)
AssertionError

However, branch names (which compose ref names) are allowed to contain
the plus sign. This commit expands the VCS_GIT_REF_VALIDATION regexp
to accept that.

FWIW, I triggered this assertion when I named my branch after the
Debian release I was merging (for the net-snmp package):

  merge-5.9.1+dfsg-4-kinetic

f5dc43c... by Robie Basak

Clean up get_head_versions()

This method had no tests, and returning a pygit2.Branch object made it
harder to supply test data to other functions that accept data in the
structure returned by this method.

In practice, callers only need the version, commit time and commit hash
of each branch head, so return only exactly this, and adjust all
callers. This should not change any behaviour.

We also adjust and fill out the docstring.

A unit test will follow in a subsequent change. It can't be added here
without fixing a bug first.

b2f98c1... by Robie Basak

GitUbuntuSourceInformation: dependency injection

Add dependency injection to GitUbuntuSourceInformation. This allows the
creation of this object in tests such that we can mock a
launchpadlib.launchpad.Launchpad object.

a11bfa8... by Robie Basak

launchpad_versions_published_after: refactor call

We don't need to set args and then call with **args, given that is all
we do with it. Instead, just call self.archive.getPublishedSources()
with keyword arguments directly.

9addaa0... by Robie Basak

launchpad_versions_published_after: drop return

This return statement is redundant since it's at the end of the method
anyway.

535dca8... by Robie Basak

Use date_created to determine head versions

It's incorrect to use date_published to determine head versions that
will be used to match against Launchpad publications, since we use
date_created at commit creation time. We should be using date_created
consistently instead.

Not doing so means that we often (always?) fail to find a matching
Launchpad publication that is already imported and end up redundantly
reimporting everything from the beginning of time. This is terrible for
performance.

More details on date_created vs. date_published here:
https://irclogs.ubuntu.com/2020/04/16/%23launchpad.html#t13:45

This change is difficult to test right here. Further refactoring follows
in subsequent changes and a test is added later.

LP: #1979650

0b8cc0b... by Robie Basak

Rewrite launchpad_versions_published_after

The logic in this method can be simplified significantly with a rewrite.
To mitigate any regression, a parametrized unit test is added with the
expected behaviour thought out from the importer spec.

There is still an inefficiency present here. In theory we could skip
importing pocket copies if the branch corresponding to a pocket is
up-to-date. However, currently the algorithm only matches against the
exact date_created attribute of the Launchpad publication object against
which a particular version was first imported. To ensure that branches
are updated if any new pocket copies have occurred, we must "replay"
them all through the importer. Therefore there is potential here for a
future performance improvement.

284beb9... by Robie Basak

Add unit test for get_head_info()

Now that get_head_info() returns what we expect, we can add a unit test
for it now.

2133870... by Robie Basak

Refactor _head_version_is_equal

Add a docstring, refactor the code to make it more readable, and rename
the method to match its definition better.

This should not result in any functional change.

06f0eca... by Robie Basak

More project renames

This is a followup to commits 57e5776, 002e452, 2f0e855 and 5aad111 with
further cleanups around the project rename from usd-importer to
git-ubuntu and the move to the <email address hidden>
mailing list. Thanks to Bryce for spotting some of the remaining pieces.

Where code is no longer used at all, or docs are completely out-of-date,
I've removed it instead of renaming the relevant bits. I've not worried
too much about fixing docs that I've touched if there's some value to
them staying, as that's a bit of a rabbit hole and I'd prefer to make
incremental progress.

0e3ca5f... by Robie Basak

submit: default to ~canonical-server-reporter

On the Canonical Server Team, we have been using ~canonical-server in
its own review slot for the sole purpose of gathering all reviews we're
interested in tracking together on this team's +activereviews page.

A problem with this is that we all belong to this team, so when a person
does a review, they sometimes accidentally "grab" that slot, so it
appears as their name rather than the team's, and thus disappears from
the report.

One way around this is to use a separate team that none of us actually
belong to. This way we can't "grab" that slot.

This changes the default team to this new ~canonical-server-reporter
team to help those who use the "git ubuntu submit" command.

4d497a7... by Robie Basak

Add comment on missing observability

This should help locate the older emailing code should it be needed in
the future.

c8216a7... by Robie Basak

Add test to accept refs that contain '+'

ab3351a... by Robie Basak

Improve documentation on validation constants

This should hopefully do a better job of signposting anyone who wants to
change the constants to better understand the implications of doing so.

7c524e6... by Robie Basak

importer: flip sense of push arguments

Instead of passing around an inverse boolean "no_push" argument, pass
the more natural "push" instead.

This should make no functional change, but prepares us to invert the CLI
argument since after that future change passing around an inverse
boolean will make even less sense.

7f3a586... by Robie Basak

importer: flip CLI argument default to not push

git ubuntu import has two uses: 1) it's run by the importer service
workers, which should push by default; 2) it's run by users, for whom it
doesn't generally make sense to push by default.

Since it's easy for importer service workers to specify an option by
automation, we flip the default of the CLI to explicitly require --push
if you want to push. This makes it safer and easier to explain to users
how to use the import command locally.

The importer service worker then adds --push unconditionally.

This makes the "implied no push" behaviour of certain options redundant,
so those are removed.

0901740... by Robie Basak

scriptutils: remove pool_map_import_srcpkg()

This function is no longer used from anywhere.

4a01d97... by Robie Basak

importer-service-worker: add --no-push argument

Asking the worker for --no-push enables a deeper dry run for performance
testing purposes. Unlike the behaviour of "git ubuntu import", "push
mode" is the default here because normally when one sets up the importer
service, it would be surprising behaviour not to do this, and this is
the only use case for this command.

ab5d7ad... by Robie Basak

Move import_srcpkg() to importer_service_worker.py

This function is only used from here, so there's no need for it to be in
a different module.

8681418... by Robie Basak

_main_with_repo: simplify if statement

Since the normal operation is to push, and straight after that we
"return 0", it's simpler to immediately "return 0" if we don't want to
push. This stops the usual code path being pushed "to the right", and
makes it easier to follow the logic.

This should not result in a functional change.

f9a91d9... by Robie Basak

Add changelog date override for gmsh

36eef3f... by Robie Basak

prepare-upload: add test for ssh:// URL rewrites

According to LP 1942985, this is another case where a rewrite is
expected.

695a4d4... by Robie Basak

prepare-upload: handle ssh:// rewrites

According to LP 1942985, this is another case where a URL rewrite is
expected.

Unmerged commits

695a4d4... by Robie Basak

prepare-upload: handle ssh:// rewrites

According to LP 1942985, this is another case where a URL rewrite is
expected.

36eef3f... by Robie Basak

prepare-upload: add test for ssh:// URL rewrites

According to LP 1942985, this is another case where a rewrite is
expected.

cc48a7f... by Robie Basak

prepare-upload: output invalid option on failure

If "git ubuntu prepare-upload args" fails for whatever reason, we don't
want dpkg-buildpackage or similar to proceed if invoked using
"dpkg-buildpackage $(git ubuntu prepare-upload args)" as this will
silently hide the error. Instead, we can output an invalid option
"--git-ubuntu-prepare-upload-args-failed" which should cause
dpkg-buildpackage to fail, and hopefully lead the user to find the cause
in stderr from the failure in our command.

This change implments this new behaviour.

LP: #1942865

d8750f1... by Robie Basak

prepare-upload: test for invalid option on failure

If "git ubuntu prepare-upload args" fails for whatever reason, we don't
want dpkg-buildpackage or similar to proceed if invoked using
"dpkg-buildpackage $(git ubuntu prepare-upload args)" as this will
silently hide the error. Instead, we can output an invalid option
"--git-ubuntu-prepare-upload-args-failed" which should cause
dpkg-buildpackage to fail, and hopefully lead the user to find the cause
in stderr from the failure in our command.

This change adds the test for this behaviour, prior to implementation.

3c5c8c7... by Robie Basak

prepare-upload: rewrite LP git+ssh:// URLs

Automatically supply the corresponding https:// LP URL for the rich
history changes file headers if a git+ssh:// LP URL is used.

LP: #1942985

76824a3... by Robie Basak

prepare-upload: refactor header data handling

Explicitly pull out the three header data items into their own named
variables to avoid confusion.

dfa0998... by Robie Basak

prepare-upload: add test for git+ssh:// rewrite

Identified in LP: #1942985: if a user has a git+ssh:// LP URL, we should
automatically rewrite it to the https:// one.

This is the test for this, which is expected to fail because it isn't
fixed yet.

8fb8e5a... by Robie Basak

Fix typos in test docstrings

ab5d7ad... by Robie Basak

Move import_srcpkg() to importer_service_worker.py

This function is only used from here, so there's no need for it to be in
a different module.

4a01d97... by Robie Basak

importer-service-worker: add --no-push argument

Asking the worker for --no-push enables a deeper dry run for performance
testing purposes. Unlike the behaviour of "git ubuntu import", "push
mode" is the default here because normally when one sets up the importer
service, it would be surprising behaviour not to do this, and this is
the only use case for this command.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/doc/README.md b/doc/README.md
2new file mode 100644
3index 0000000..ff10689
4--- /dev/null
5+++ b/doc/README.md
6@@ -0,0 +1,117 @@
7+<<<<<<< doc/README.md
8+=======
9+## Running the git-ubuntu importer ##
10+This just covers how to run [`git ubuntu import`](https://code.launchpad.net/git-ubuntu).
11+
12+## Getting via snap ##
13+The preferred installation method is to install via snap:
14+
15+1. install the snap
16+
17+ $ snap install --classic git-ubuntu
18+
19+## [Alternate:] Getting via git ##
20+Less well tested, but in theory this should work as well.
21+
22+1. Get `git-ubuntu` from git
23+
24+ $ git clone git://git.launchpad.net/git-ubuntu git-ubuntu
25+or
26+
27+ $ git clone https://git.launchpad.net/git-ubuntu git-ubuntu
28+or
29+
30+ $ git clone ssh://git.launchpad.net/git-ubuntu git-ubuntu
31+
32+
33+2. Put it in your PATH
34+
35+ $ PATH="$PWD/git-ubuntu/bin:$PATH"
36+
37+3. Get necessary dependencies
38+
39+ $ sudo apt update -qy
40+
41+ $ deps="dpkg-dev git-buildpackage python3-argcomplete \
42+ python3-lazr.restfulclient python3-debian python3-distro-info \
43+ python3-launchpadlib python3-pygit2 python3-ubuntutools \
44+ python3-cachetools python3-pkg-resources python3-pytest \
45+ python3-petname quilt"
46+
47+ $ sudo apt install -qy ${deps}
48+
49+
50+## Running ##
51+
52+ * For local usage
53+
54+ `git ubuntu import` will push to launchpad git by default. If you just want to get a git repo locally of a given package, then:
55+
56+ $ mkdir ${HOME}/Imports
57+ $ PKG=uvtool
58+ $ git ubuntu import -v --no-push --directory=${HOME}/Imports/$PKG $PKG
59+
60+ * As member of [git-ubuntu-import](https://launchpad.net/~git-ubuntu-import) for official publishing.
61+
62+ $ PKG=uvtool
63+ $ LP_USER=smoser # your launchpad user name if different from $USER
64+ $ git ubuntu import -v --directory=$PKG --lp-user=$LP_USER $PKG
65+
66+## Shell completion (bash) ##
67+
68+ * `git-ubuntu` will autocomplete by default if global argcomplete has
69+ been enabled
70+ (https://github.com/kislyuk/argcomplete#activating-global-completion)
71+ or specify
72+
73+ eval "$(register-python-argcomplete git-ubuntu)"
74+
75+ The snap version does this by default.
76+
77+ * `git ubuntu` autocompletion is a little more challenging. To enable
78+ it, add the following to your .bashrc or similar:
79+
80+ if [ -f /path/to/git-ubuntu/doc/gitubuntu-completion.sh ]; then
81+ . /path/to/git-ubuntu/doc/gitubuntu-completion.sh
82+ fi
83+
84+ For the snap version, this would look like:
85+
86+ if [ -f /snap/git-ubuntu/current/doc/gitubuntu-completion.sh ]; then
87+ . /snap/git-ubuntu/current/doc/gitubuntu-completion.sh
88+ fi
89+
90+## View Output ##
91+If you did a local checkout with `--directory=./$PKG` then you'll have a git repository in `./$PKG/git`.
92+
93+ $ cd $PKG
94+ $ git branch
95+ ubuntu/saucy
96+ ubuntu/saucy-proposed
97+ ubuntu/trusty
98+ ubuntu/trusty-proposed
99+ ubuntu/utopic
100+ ubuntu/utopic-proposed
101+ ubuntu/vivid
102+ ubuntu/vivid-proposed
103+ ubuntu/wily
104+ ubuntu/xenial
105+ ubuntu/yakkety
106+
107+If you did `--lp-owner=git-ubuntu-import`, then your repo should be
108+listed in web view at [https://code.launchpad.net/~git-ubuntu-import/+git]. And it should be able to be cloned with:
109+
110+ $ git clone https://git.launchpad.net/~git-ubuntu-import/ubuntu/+source/$PKG
111+or
112+
113+ $ git clone lp:~git-ubuntu-import/ubuntu/+source/$PKG
114+or
115+
116+ $ git ubuntu clone $PKG
117+
118+
119+## Links ##
120+ * [GitWorkflow Wiki page](https://wiki.ubuntu.com/UbuntuDevelopment/Merging/GitWorkflow)
121+ * [Launchpad git for git-ubuntu](https://code.launchpad.net/git-ubuntu)
122+ * [Git view of git-ubuntu](https://git.launchpad.net/git-ubuntu)
123+>>>>>>> doc/README.md
124diff --git a/doc/SPECIFICATION b/doc/SPECIFICATION
125new file mode 100644
126index 0000000..21cd184
127--- /dev/null
128+++ b/doc/SPECIFICATION
129@@ -0,0 +1,167 @@
130+<<<<<<< doc/SPECIFICATION
131+=======
132+Specification
133+
134+git URL shortcuts used (add these to ~/.gitconfig or expand them
135+manually yourself):
136+
137+[url "ssh://<LPID>@git.launchpad.net/~<LPID>/ubuntu/+source/"]
138+ insteadof = lpmep:
139+
140+Definitions: "old debian", "old ubuntu", "new debian", "new ubuntu" are
141+as understood. Make sure that "old debian" is really the last common
142+ancestor of "old ubuntu" and "new debian". Determining this is
143+especially prone to error if Ubuntu imported new upstream versions since
144+it diverged from Debian. If this is wrong, then pain will result.
145+
146+By "merge" we always mean an "Ubuntu merge", which is in git terms
147+really a rebase. No actual git merge takes place in this entire
148+workflow.
149+
150+No trees in this workflow ever have quilt patches applied. All commits
151+are with quilt fully popped and no .pc directory. Changes to quilt
152+patches are seen in debian/patches/* only.
153+
154+Common git references expected (T for tag, B for branch):
155+
156+Things that will be imported by a sponsor or the importer (available
157+from: lpusip:<package>; ask a sponsor if missing):
158+
159+* T import/<version> and T upload/<version>
160+ * Logically this is the tree corresponding to a particular tag;
161+ history is secondary.
162+ * The tree is identical to corresponding source package version in the
163+ archive.
164+ * For T import/<version>: imported from the archive and pushed to
165+ ~git-ubuntu-import as an authoritative source.
166+ * For T upload/<version>: pushed to ~ubuntu-server-dev by an uploader
167+ to record exactly what was uploaded.
168+ * Pushing to ~ubuntu-server-dev is restricted to uploaders.
169+ * The parent commit should be the previous version import or upload
170+ tag where available. An orphan commit is acceptable in the
171+ exceptional case that this is not possible.
172+
173+* B ubuntu/devel
174+ * Logically this is our moving reference for what is currently in the
175+ Ubuntu development release.
176+ * In ~ubuntu-server-dev, this must always point to something also
177+ tagged as import/<version> or upload/<version>.
178+ * Pushing to ~ubuntu-server-dev is restricted to uploaders.
179+ * This branch will be rebased to new Debian imports during Ubuntu
180+ "merges" (but tags will be left behind).
181+
182+Things that should be made available to a sponsor when submitting a
183+merge for upload (push to: lpmep:<package>):
184+
185+* T logical/<old ubuntu>
186+ * Logically, this is a patchset
187+ ({import,upload}/<old debian>..logical/<old ubuntu>).
188+ * Breakdown of previous Ubuntu delta.
189+ * Must be based on an official import/<old debian> or upload/<old debian>
190+ tag ("official" means from ~ubuntu-server-dev).
191+ * One commit per logical change over the entire Ubuntu delta.
192+ * Churn squashed.
193+ * No upstream changes (so only changes in debian/*).
194+ * No changes to debian/changelog.
195+ * No "update-maintainer" or "Vcs-*" or other meta changes.
196+ * To get to this, you will probably start from reconstruct/<old ubuntu>,
197+ described below.
198+ * Coherence checks:
199+ - Identical to the corresponding import/<version> except for:
200+ + Meta changes (update-maintainer, Vcs-*) in debian/control.
201+ + Anything not in debian/*, which should be unchanged
202+ (exceptionally this happens when new upstream versions were
203+ imported ahead of Debian).
204+ + debian/changelog, which should be unchanged.
205+ - No line should be touched twice, except where separate logical
206+ changes need to touch the same line.
207+ * Providing this makes it easy for the sponsor to check a proposed
208+ merge:
209+ 1. Check correctness of this tag against the previous Ubuntu delta
210+ (perform the above checks and use "git log -p" to make
211+ sure each logical commit describes only its own changes).
212+ 2. Ensure that every commit here is accounted for in the proposed
213+ merge.
214+
215+* B merge
216+ * Proposed merge for upload.
217+ * Based on import/<new debian> or upload/<new debian>.
218+ * One commit per logical change; no changes to debian/changelog in
219+ those commits.
220+ * One commit for each of merge-changelogs, reconstruct-changelog, any
221+ changelog tweaks and ubuntu-meta (or update-maintainer as you wish).
222+ * debian/changelog should be "released" with the version string
223+ matching the proposed upload version and targeting the correct
224+ pocket.
225+ * Add commits to the end of this branch in response to reviewer
226+ comments.
227+ * If agreed with your sponsor that for the changes requested a new
228+ rebased merge branch will be easier to manage than adding commits to
229+ the end, then do this instead. Rebase the original "merge" branch.
230+ To keep history, if you wish tag the old one "merge.v1". You may
231+ also rebase like this as you wish during preparation before
232+ presenting this branch for review.
233+
234+Things you may want to make available to reviewers so that they can
235+check your process (push to: lpmep:<package>), for which we have
236+standardised names:
237+
238+* T reconstruct/<old ubuntu>
239+ * Logically, this is a patchset
240+ ({import,upload}/<old debian>..reconstruct/<old ubuntu>).
241+ * Based on import/<old debian>. For each Ubuntu upload since then:
242+ * One commit to pull in a new upstream if there is one (rare). This
243+ must not contain any changes to debian/.
244+ * One commit per logical change.
245+ * One commit for changelog.
246+ * One commit for any ubuntu-meta/update-maintainer change (usually
247+ only in merge uploads).
248+ * Drop non-logical commits from this tip and rebase to squash and
249+ split to derive the logical/<old ubuntu> tag.
250+
251+* T merge.v1, merge.v2, etc.
252+ * The old state of each merge branch before you rebased it. Only
253+ useful if you rebased during your merge. If done after your initial
254+ review request, please only do this with agreement of your sponsor,
255+ since it causes your sponsor more review time.
256+
257+Merge proposal to make in Launchpad:
258+
259+lpmep:<package> merge → lpusdp:<package> ubuntu/devel
260+
261+After review:
262+
263+If adding commits in response to reveiwer comments, just push again to
264+lpmep:<package> merge.
265+
266+If (exceptionally) rebasing in response to reviewer comments:
267+ 1. Tag the old branch "merge.v1" (or v2, v3 etc. for future iterations)
268+ 2. Rebase the "merge" branch as required
269+ 3. Push to lpmep:<package>:
270+ a) The new "v" tag from above.
271+ b) The merge branch (force will be required).
272+
273+For "traditional" sponsors:
274+
275+git can easily generate the traditional debdiffs that you normally
276+review. Assuming you have appropriate remote tracking branches:
277+
278+ * For Ubuntu → Ubuntu, "git diff lpusdp/ubuntu/devel sponsoree/merge"
279+ * For Debian → Ubuntu, "git diff lpusdp/debian/sid sponsoree/merge"
280+
281+Or you can ask the sponsoree to generate these for you.
282+
283+To upload a reviewed merge (for the sponsor):
284+
285+(Sponsors: you can just ignore these instructions and upload the
286+traditional way if you like. But sponsorees cannot push to our VCS and
287+you can, so it would be nice if you could push this please, so a future
288+merger doesn't have to reconstruct the lost information).
289+
290+1. Upload using dput as usual.
291+2. Tag the merge branch "upload/<version>" (replace ':' and '~' with '_'
292+ to meet git's naming requirements). A lightweight tag is fine, or
293+ go ahead and annotate if you want to include any extra notes.
294+3. Force push the merge branch to lpusdp:<package> ubuntu/devel.
295+4. Push the "upload/<version>" tag to lpusdp:<package>.
296+>>>>>>> doc/SPECIFICATION
297diff --git a/doc/release-process.md b/doc/release-process.md
298new file mode 100644
299index 0000000..524736e
300--- /dev/null
301+++ b/doc/release-process.md
302@@ -0,0 +1,264 @@
303+<<<<<<< doc/release-process.md
304+=======
305+Release Process
306+===============
307+
308+1. Set the new version number
309+------------------------------
310+
311+See gitubuntu/version.py for the current version number.
312+
313+ $ export LAST_RELEASE=$(cat gitubuntu/version.py | cut -d\' -f2)
314+ $ echo "${LAST_RELEASE}"
315+
316+Git Ubuntu's version numbers follow the common MAJOR.MINOR.PATCH and
317+MAJOR.MINOR.PATCH-rcN patterns, where for this project these are
318+interpreted as follows:
319+
320+ - MAJOR is updated for API breaking changes such as alterations in
321+ importer hash ABI stability. As a special rule, MAJOR=0 indicates
322+ no stability guarantees. Notably, changes in MAJOR version are not
323+ guaranteed to be forward or backward compatible with earlier MAJOR
324+ versions.
325+
326+ - MINOR is incremented for feature-level changes that may alter how
327+ the git ubuntu frontends behave, including breaking changes in how
328+ git ubuntu subcommands and their parameters work. The importer API,
329+ however, is intended to be backward compatible from one MINOR
330+ version to the next, with no breaking changes.
331+
332+ - PATCH is incremented for bug fixes and routine feature additions
333+ that introduce no compatibility issues for either the backend
334+ importer or the frontend client. In particular, new commands and
335+ parameters may be introduced, but existing ones will not be changed
336+ or removed.
337+
338+ - rcN indicates a release candidate, using a sequential numbering for
339+ 'N'.
340+
341+Define the new version for the release:
342+
343+ $ export VERSION="<MAJOR>.<MINOR>.<PATCH>"
344+
345+Or, for a release candidate:
346+
347+ $ export VERSION="<MAJOR>.<MINOR>.<PATCH>-rcN"
348+
349+Set it in the git repo:
350+
351+ $ git checkout -b ${VERSION}-release
352+ $ echo "VERSION = '${VERSION}'" > gitubuntu/version.py
353+ $ git commit gitubuntu/version.py -m "version: bump to ${VERSION}"
354+ $ git tag --annotate -m "${VERSION} Release" ${VERSION}
355+
356+The annotated tag is necessary, because the snap build mechanisms
357+determine the version to set in the snap based on it.
358+
359+
360+2. Draft release announcement
361+------------------------------
362+
363+The release announcement generally summarizes the major changes in the
364+release, and (where possible) identifies the bug fixes included in it.
365+Some examples of past release announcements:
366+
367+ - 0.2.1: https://lists.ubuntu.com/archives/ubuntu-server/2017-September/007594.html
368+ - 0.3.0: https://lists.ubuntu.com/archives/ubuntu-server/2017-October/007598.html
369+ - 0.4.0: https://lists.ubuntu.com/archives/ubuntu-server/2017-October/007605.html
370+ - 0.7.1: https://lists.ubuntu.com/archives/ubuntu-server/2018-March/007667.html
371+
372+The git log can be referred to for changes worth mentioning:
373+
374+ $ git log --stat ${LAST_RELEASE}..
375+
376+If desired, a shortlog can be appended to the release announcement, to
377+itemize all changes:
378+
379+ $ git shortlog ${LAST_RELEASE}...
380+
381+
382+3. Testing
383+-----------
384+
385+First check there are no unexpected test failures in trunk:
386+
387+ $ python3 ./setup.py check
388+ $ python3 ./setup.py build
389+ $ pytest-3 .
390+
391+Optionally, the full test suite can be directly executed, although since
392+it has some rather exacting dependencies, it may not be able to build
393+properly.
394+
395+ $ python3 ./setup.py test
396+
397+Next, push a copy of the branch up to launchpad under your own namespace
398+for Continuous Integration (CI) testing:
399+
400+ $ git push ${LP_USERNAME} ${VERSION}-release
401+
402+Go to the Launchpad page for the branch and create a merge proposal
403+targeted to lp:git-ubuntu, set 'main' as the Target branch and set
404+the Description to say "For CI build only". Review type can be set to
405+'ci'. This will ensure the regular CI runs on it, which exercises the
406+snap build mechanics, but let's the development team know it can be
407+ignored for review purposes. This isn't the snap we'll actually be
408+using, but will produce one we can download and inspect.
409+
410+A snap candidate (not yet uploaded to the store) can be installed
411+locally for testing like this:
412+
413+ $ lxc exec ${CONTAINER} -- rm /tmp/git-ubuntu_0+git.*_amd64.snap
414+ $ lxc file push ./git-ubuntu_0+git.*_amd64.snap ${CONTAINER}/tmp
415+ $ lxc exec ${CONTAINER} -- bash
416+ $ sudo snap install --classic --dangerous /tmp/git-ubuntu_0+git.*_amd64.snap
417+
418+The snap package itself can be locally mounted directly as a filesystem,
419+which can be helpful for evaluating its contents. For example, to look
420+at what Python modules are included:
421+
422+ $ mkdir /tmp/snap
423+ $ sudo mount git-ubuntu_0+git.59a1e51_amd64.snap /tmp/snap/
424+ $ ls /tmp/snap/usr/lib/python3.6/
425+ $ cd ${HOME} && umount /tmp/snap && rmdir /tmp/snap
426+
427+
428+4. Release the new version
429+---------------------------
430+
431+Once everything looks good, merge the change from your local release branch to master:
432+
433+ $ git checkout master
434+ $ git merge --ff-only ${VERSION}-release
435+
436+Make sure everything looks ok. The status should show no uncommitted
437+changes, etc. Verify the log shows the correct tags and that HEAD
438+points to master, etc. Doublecheck that git describe displays
439+${VERSION}:
440+
441+ $ git status
442+ $ git log --oneline --decorate=short
443+ $ git describe
444+
445+If all looks good, now push the annotated tag and code changes to origin:
446+
447+ $ git push origin master ${VERSION}
448+
449+
450+5. Publish Snap
451+---------------
452+
453+Channels used for delivering the snap package are defined as follows:
454+
455+ - EDGE: Tracks the latest code in master to allow testing of
456+ potentially unstable work. This is not recommended for general
457+ usage by end users.
458+
459+ - BETA: Most of the time, this channel will track the same version as
460+ in STABLE, but also delivers release candidates and sometimes may
461+ provide early access to new features or bug fixes. This channel is
462+ recommended particularly for advanced git-ubuntu users who wish to
463+ participate in testing activities. It is also the channel used for
464+ the importer on the server.
465+
466+ - STABLE: This channel tracks the current release used in the
467+ git-ubuntu service itself. This is the recommended channel for
468+ all end users.
469+
470+You will initially publish the package to EDGE only to verify it builds
471+properly.
472+
473+First, trigger a rebuild of the snap in the server team's Jenkins
474+instance. The git push from step #4 will get picked up by the nightly
475+builder, but if you don't wish to wait a day for the build, you can
476+manually trigger it on this page:
477+
478+ https://jenkins.ubuntu.com/server/job/git-ubuntu-ci-nightly/
479+
480+Make sure you're logged into Jenkins, then click
481+
482+ "Build Now"
483+
484+Once this is done, download the snap from Jenkins. It should be listed
485+under Last Successful Artifacts on this page:
486+
487+ https://jenkins.ubuntu.com/server/job/git-ubuntu-ci-nightly/
488+
489+Next, verify you have your snapcraft account configured, logged in, and
490+working locally:
491+
492+ $ snapcraft whoami
493+ $ snap list
494+
495+Finally, upload the snap to EDGE:
496+
497+ $ snapcraft push --release edge ./git-ubuntu_${VERSION}+git<whatever>_amd64.snap
498+
499+The command will block for a few minutes while the store analyzes the
500+snap. Once it is approved, it will become available in the edge channel.
501+
502+For anything but trivial releases, you should then `snap install` the
503+edge version of the package in a test environment to verify it.
504+
505+Once you deem it good to go, use the Snapcraft website
506+(https://snapcraft.io/git-ubuntu/releases) to copy the snap to BETA, and
507+proceed with installing it in production (next step). Solicit broader
508+testing, as appropriate, and then after a sufficient amount of testing
509+time (e.g. a week or so) copy the snap to STABLE.
510+
511+
512+6. Installation to Production
513+-----------------------------
514+
515+See our internal process documentation for details on how to do this.
516+
517+
518+7. Announce Release
519+-------------------
520+
521+Email the (gpg signed) announcement to:
522+
523+ To: ubuntu-devel@lists.ubuntu.com
524+ Cc: ubuntu-distributed-devel@lists.ubuntu.com
525+
526+Upload a copy of the announcement to https://launchpad.net/git-ubuntu/
527+
528+
529+8. Close bugs
530+-------------
531+
532+Close all bugs fixed by this release. Here's an example that can be run
533+from `lp-shell` to close all bug tasks marked "Fix Committed". If you
534+use this, remember to change `VERSION` appropriately:
535+
536+ VERSION = '1.0'
537+ tasks = list(lp.projects['git-ubuntu'].searchTasks(status='Fix Committed'))
538+ bugs = [lp.load(bug_link) for bug_link in set(task.bug_link for task in tasks)]
539+ for bug in bugs:
540+ bug.newMessage(
541+ subject=f'Fix released in git-ubuntu',
542+ content=f'Fix released in git-ubuntu version {VERSION}',
543+ )
544+ for task in tasks:
545+ task.status = 'Fix Released'
546+ task.lp_save()
547+
548+
549+9. Update Trello Card
550+---------------------
551+
552+If a card hasn't been created in the daily-ubuntu-server board for the
553+release task already, add one at this point. Add yourself as a member
554+of the card, and add labels 'git-ubuntu' and 'highlight'. The latter
555+label flags it to be mentioned in the week's Ubuntu Server Developer Summary.
556+
557+
558+10. Discourse Blogging (Optional)
559+--------------------------------
560+
561+If desired, follow up with one or more topics/posts to
562+discourse.ubuntu.com about the major new features included in the
563+release. Discourse posts shouldn't be done just for ordinary bug
564+fixing, and shouldn't simply mirror the release announcement or usage
565+documentation.
566+>>>>>>> doc/release-process.md
567diff --git a/gitubuntu/changelog_date_overrides.txt b/gitubuntu/changelog_date_overrides.txt
568new file mode 100644
569index 0000000..cdf5e30
570--- /dev/null
571+++ b/gitubuntu/changelog_date_overrides.txt
572@@ -0,0 +1,19 @@
573+<<<<<<< gitubuntu/changelog_date_overrides.txt
574+=======
575+# Package versions that have illegal dates in their changelog entries.
576+# In these cases the first seen publication date must be used instead
577+# for the author date of a synthesized commit.
578+#
579+# Note: this file must exactly match the import specification. Before
580+# adding an entry here, adjust the specification first.
581+
582+ghostscript 9.50~dfsg-5ubuntu4
583+gmsh 2.0.7-1.2ubuntu1
584+iscsitarget 0.4.15+svn148-2.1ubuntu1
585+lxqt-config 0.13.0-0ubuntu4
586+mail-spf-perl 2.004-0ubuntu1
587+nut 2.2.0-2
588+prips 0.9.4-3
589+prometheus-alertmanager 0.15.3+ds-3ubuntu1
590+software-properties 0.80
591+>>>>>>> gitubuntu/changelog_date_overrides.txt
592diff --git a/gitubuntu/changelog_tests/maintainer_name_inner_space b/gitubuntu/changelog_tests/maintainer_name_inner_space
593new file mode 100644
594index 0000000..17cc421
595--- /dev/null
596+++ b/gitubuntu/changelog_tests/maintainer_name_inner_space
597@@ -0,0 +1,8 @@
598+<<<<<<< gitubuntu/changelog_tests/maintainer_name_inner_space
599+=======
600+testpkg (1.0) xenial; urgency=medium
601+
602+ * Sample entry.
603+
604+ -- Test Maintainer <test-maintainer@example.com> Thu, 01 Jan 1970 00:00:00 +0000
605+>>>>>>> gitubuntu/changelog_tests/maintainer_name_inner_space
606diff --git a/gitubuntu/changelog_tests/maintainer_name_leading_space b/gitubuntu/changelog_tests/maintainer_name_leading_space
607new file mode 100644
608index 0000000..07e9dd8
609--- /dev/null
610+++ b/gitubuntu/changelog_tests/maintainer_name_leading_space
611@@ -0,0 +1,8 @@
612+<<<<<<< gitubuntu/changelog_tests/maintainer_name_leading_space
613+=======
614+testpkg (1.0) xenial; urgency=medium
615+
616+ * Sample entry.
617+
618+ -- Test Maintainer <test-maintainer@example.com> Thu, 01 Jan 1970 00:00:00 +0000
619+>>>>>>> gitubuntu/changelog_tests/maintainer_name_leading_space
620diff --git a/gitubuntu/changelog_tests/maintainer_name_trailing_space b/gitubuntu/changelog_tests/maintainer_name_trailing_space
621new file mode 100644
622index 0000000..358b4a3
623--- /dev/null
624+++ b/gitubuntu/changelog_tests/maintainer_name_trailing_space
625@@ -0,0 +1,8 @@
626+<<<<<<< gitubuntu/changelog_tests/maintainer_name_trailing_space
627+=======
628+testpkg (1.0) xenial; urgency=medium
629+
630+ * Sample entry.
631+
632+ -- Test Maintainer <test-maintainer@example.com> Thu, 01 Jan 1970 00:00:00 +0000
633+>>>>>>> gitubuntu/changelog_tests/maintainer_name_trailing_space
634diff --git a/gitubuntu/changelog_tests/test_date_1 b/gitubuntu/changelog_tests/test_date_1
635new file mode 100644
636index 0000000..3c54cec
637--- /dev/null
638+++ b/gitubuntu/changelog_tests/test_date_1
639@@ -0,0 +1,8 @@
640+<<<<<<< gitubuntu/changelog_tests/test_date_1
641+=======
642+testpkg (1.0) xenial; urgency=medium
643+
644+ * Sample entry.
645+
646+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
647+>>>>>>> gitubuntu/changelog_tests/test_date_1
648diff --git a/gitubuntu/changelog_tests/test_date_2 b/gitubuntu/changelog_tests/test_date_2
649new file mode 100644
650index 0000000..9defc1f
651--- /dev/null
652+++ b/gitubuntu/changelog_tests/test_date_2
653@@ -0,0 +1,8 @@
654+<<<<<<< gitubuntu/changelog_tests/test_date_2
655+=======
656+testpkg (1.0) xenial; urgency=medium
657+
658+ * Sample entry.
659+
660+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
661+>>>>>>> gitubuntu/changelog_tests/test_date_2
662diff --git a/gitubuntu/changelog_tests/test_distribution b/gitubuntu/changelog_tests/test_distribution
663new file mode 100644
664index 0000000..099b2b2
665--- /dev/null
666+++ b/gitubuntu/changelog_tests/test_distribution
667@@ -0,0 +1,8 @@
668+<<<<<<< gitubuntu/changelog_tests/test_distribution
669+=======
670+testpkg (1.0) xenial; urgency=medium
671+
672+ * Sample entry.
673+
674+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
675+>>>>>>> gitubuntu/changelog_tests/test_distribution
676diff --git a/gitubuntu/changelog_tests/test_distribution_source_1 b/gitubuntu/changelog_tests/test_distribution_source_1
677new file mode 100644
678index 0000000..dbb4587
679--- /dev/null
680+++ b/gitubuntu/changelog_tests/test_distribution_source_1
681@@ -0,0 +1,8 @@
682+<<<<<<< gitubuntu/changelog_tests/test_distribution_source_1
683+=======
684+testpkg (1.0) xenial; urgency=medium
685+
686+ * Sample entry.
687+
688+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
689+>>>>>>> gitubuntu/changelog_tests/test_distribution_source_1
690diff --git a/gitubuntu/changelog_tests/test_distribution_source_2 b/gitubuntu/changelog_tests/test_distribution_source_2
691new file mode 100644
692index 0000000..04abafe
693--- /dev/null
694+++ b/gitubuntu/changelog_tests/test_distribution_source_2
695@@ -0,0 +1,8 @@
696+<<<<<<< gitubuntu/changelog_tests/test_distribution_source_2
697+=======
698+testpkg (1.0) zesty-security; urgency=medium
699+
700+ * Sample entry.
701+
702+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
703+>>>>>>> gitubuntu/changelog_tests/test_distribution_source_2
704diff --git a/gitubuntu/changelog_tests/test_distribution_source_3 b/gitubuntu/changelog_tests/test_distribution_source_3
705new file mode 100644
706index 0000000..f89aa4a
707--- /dev/null
708+++ b/gitubuntu/changelog_tests/test_distribution_source_3
709@@ -0,0 +1,8 @@
710+<<<<<<< gitubuntu/changelog_tests/test_distribution_source_3
711+=======
712+testpkg (1.0) unstable; urgency=medium
713+
714+ * Sample entry.
715+
716+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
717+>>>>>>> gitubuntu/changelog_tests/test_distribution_source_3
718diff --git a/gitubuntu/changelog_tests/test_distribution_source_4 b/gitubuntu/changelog_tests/test_distribution_source_4
719new file mode 100644
720index 0000000..8c8fb68
721--- /dev/null
722+++ b/gitubuntu/changelog_tests/test_distribution_source_4
723@@ -0,0 +1,8 @@
724+<<<<<<< gitubuntu/changelog_tests/test_distribution_source_4
725+=======
726+testpkg (1.0) devel; urgency=medium
727+
728+ * Sample entry.
729+
730+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
731+>>>>>>> gitubuntu/changelog_tests/test_distribution_source_4
732diff --git a/gitubuntu/changelog_tests/test_maintainer_1 b/gitubuntu/changelog_tests/test_maintainer_1
733new file mode 100644
734index 0000000..62a89ff
735--- /dev/null
736+++ b/gitubuntu/changelog_tests/test_maintainer_1
737@@ -0,0 +1,8 @@
738+<<<<<<< gitubuntu/changelog_tests/test_maintainer_1
739+=======
740+testpkg (1.0) xenial; urgency=medium
741+
742+ * Sample entry.
743+
744+ -- Test Maintainer <test-maintainer@donotmail.com> Thu, 01 Jan 1970 00:00:00 +0000
745+>>>>>>> gitubuntu/changelog_tests/test_maintainer_1
746diff --git a/gitubuntu/changelog_tests/test_maintainer_2 b/gitubuntu/changelog_tests/test_maintainer_2
747new file mode 100644
748index 0000000..7b8db29
749--- /dev/null
750+++ b/gitubuntu/changelog_tests/test_maintainer_2
751@@ -0,0 +1,8 @@
752+<<<<<<< gitubuntu/changelog_tests/test_maintainer_2
753+=======
754+testpkg (1.0) xenial; urgency=medium
755+
756+ * Sample entry.
757+
758+ -- <test-maintainer@donotmail.com> Thu, 01 Jan 1970 00:00:00 +0000
759+>>>>>>> gitubuntu/changelog_tests/test_maintainer_2
760diff --git a/gitubuntu/changelog_tests/test_maintainer_3 b/gitubuntu/changelog_tests/test_maintainer_3
761new file mode 100644
762index 0000000..195564a
763--- /dev/null
764+++ b/gitubuntu/changelog_tests/test_maintainer_3
765@@ -0,0 +1,8 @@
766+<<<<<<< gitubuntu/changelog_tests/test_maintainer_3
767+=======
768+testpkg (1.0) xenial; urgency=medium
769+
770+ * Sample entry.
771+
772+ -- <test-maintainer@donotmail.com> Thu, 01 Jan 1970 00:00:00 +0000
773+>>>>>>> gitubuntu/changelog_tests/test_maintainer_3
774diff --git a/gitubuntu/changelog_tests/test_versions_1 b/gitubuntu/changelog_tests/test_versions_1
775new file mode 100644
776index 0000000..ec94bc5
777--- /dev/null
778+++ b/gitubuntu/changelog_tests/test_versions_1
779@@ -0,0 +1,8 @@
780+<<<<<<< gitubuntu/changelog_tests/test_versions_1
781+=======
782+testpkg (1.0) xenial; urgency=medium
783+
784+ * Sample entry.
785+
786+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
787+>>>>>>> gitubuntu/changelog_tests/test_versions_1
788diff --git a/gitubuntu/changelog_tests/test_versions_2 b/gitubuntu/changelog_tests/test_versions_2
789new file mode 100644
790index 0000000..ed6ade9
791--- /dev/null
792+++ b/gitubuntu/changelog_tests/test_versions_2
793@@ -0,0 +1,14 @@
794+<<<<<<< gitubuntu/changelog_tests/test_versions_2
795+=======
796+testpkg (2.0) xenial; urgency=medium
797+
798+ * Sample entry 2.
799+
800+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 27 Aug 2016 12:10:34 -0700
801+
802+testpkg (1.0) xenial; urgency=medium
803+
804+ * Sample entry 1.
805+
806+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 12 May 2016 08:14:34 -0700
807+>>>>>>> gitubuntu/changelog_tests/test_versions_2
808diff --git a/gitubuntu/changelog_tests/test_versions_3 b/gitubuntu/changelog_tests/test_versions_3
809new file mode 100644
810index 0000000..8fbf944
811--- /dev/null
812+++ b/gitubuntu/changelog_tests/test_versions_3
813@@ -0,0 +1,26 @@
814+<<<<<<< gitubuntu/changelog_tests/test_versions_3
815+=======
816+testpkg (4.0) zesty; urgency=medium
817+
818+ * Sample entry 4.
819+
820+ -- Test Maintainer <test-maintainer@donotmail.com> Mon, 03 Apr 2017 18:04:01 -0700
821+
822+testpkg (3.0) yakkety; urgency=medium
823+
824+ * Sample entry 3.
825+
826+ -- Test Maintainer <test-maintainer@donotmail.com> Fri, 10 Nov 2016 03:34:10 -0700
827+
828+testpkg (2.0) xenial; urgency=medium
829+
830+ * Sample entry 2.
831+
832+ -- Test Maintainer <test-maintainer@donotmail.com> Sat, 27 Aug 2016 12:10:55 -0700
833+
834+testpkg (1.0) xenial; urgency=medium
835+
836+ * Sample entry 1.
837+
838+ -- Test Maintainer <test-maintainer@donotmail.com> Thu, 12 May 2016 08:14:34 -0700
839+>>>>>>> gitubuntu/changelog_tests/test_versions_3
840diff --git a/gitubuntu/clone.py b/gitubuntu/clone.py
841new file mode 100644
842index 0000000..aff52be
843--- /dev/null
844+++ b/gitubuntu/clone.py
845@@ -0,0 +1,178 @@
846+<<<<<<< gitubuntu/clone.py
847+=======
848+import argparse
849+import logging
850+import os
851+import re
852+import shutil
853+from subprocess import CalledProcessError
854+import sys
855+from gitubuntu.__main__ import top_level_defaults
856+from gitubuntu.git_repository import (
857+ GitUbuntuRepository,
858+ GitUbuntuRepositoryFetchError,
859+)
860+from gitubuntu.run import decode_binary, run
861+
862+import pkg_resources
863+import pygit2
864+
865+def copy_hooks(src, dst):
866+ try:
867+ os.mkdir(dst)
868+ except FileExistsError:
869+ pass
870+
871+ for hook in os.listdir(src):
872+ shutil.copy2(
873+ os.path.join(src, hook),
874+ dst,
875+ )
876+
877+def main(
878+ package,
879+ directory=None,
880+ lp_user=None,
881+ proto=top_level_defaults.proto,
882+):
883+ """Entry point to clone subcommand
884+
885+ @package: Name of source package
886+ @directory: directory to clone the repository into
887+ @lp_user: user to authenticate to Launchpad as
888+ @proto: string protocol to use (one of 'http', 'https', 'git')
889+
890+ If directory is None, a relative directory with the same name as
891+ package will be used.
892+
893+ If lp_user is None, value of `git config gitubuntu.lpuser` will be
894+ used.
895+
896+ Returns the resulting GitUbuntuRepository object, if successful;
897+ None otherwise.
898+ """
899+ directory = (
900+ os.path.abspath(directory)
901+ if directory
902+ else os.path.join(os.path.abspath(os.getcwd()), package)
903+ )
904+ if os.path.isdir(directory):
905+ logging.error('directory %s exists' % directory)
906+ return None
907+
908+ local_repo = GitUbuntuRepository(
909+ local_dir=directory,
910+ lp_user=lp_user,
911+ fetch_proto=proto,
912+ )
913+
914+ copy_hooks(
915+ pkg_resources.resource_filename(
916+ 'gitubuntu',
917+ 'hooks',
918+ ),
919+ os.path.join(
920+ directory,
921+ os.getenv('GIT_DIR', '.git'),
922+ 'hooks',
923+ ),
924+ )
925+
926+ local_repo.add_base_remotes(package)
927+ try:
928+ local_repo.fetch_base_remotes(verbose=True)
929+ except GitUbuntuRepositoryFetchError:
930+ logging.error("Unable to find an imported repository for %s. "
931+ "Please request an import by e-mailing "
932+ "ubuntu-distributed-devel@lists.ubuntu.com.",
933+ package
934+ )
935+ shutil.rmtree(local_repo.local_dir)
936+ return None
937+
938+ local_repo.add_lpuser_remote(pkgname=package)
939+ logging.debug("added remote '%s' -> %s", local_repo.lp_user,
940+ local_repo.raw_repo.remotes[local_repo.lp_user].url
941+ )
942+ try:
943+ local_repo.fetch_lpuser_remote(verbose=True)
944+ except GitUbuntuRepositoryFetchError:
945+ pass
946+
947+ try:
948+ local_repo.create_tracking_branch(
949+ 'ubuntu/devel',
950+ 'pkg/ubuntu/devel'
951+ )
952+ local_repo.checkout_commitish('ubuntu/devel')
953+ except:
954+ logging.error('Unable to checkout ubuntu/devel, does '
955+ 'pkg/ubuntu/devel branch exist?'
956+ )
957+
958+ local_repo.git_run(['config', 'notes.displayRef', 'refs/notes/changelog'])
959+
960+ if os.path.isfile(os.path.join(directory, '.gitignore')):
961+ logging.warning('A .gitignore file exists in the source '
962+ 'package. This will affect the behavior of git. Consider '
963+ 'backing up the gitignore while working on this package '
964+ 'to ensure all changes are tracked or passing appropriate '
965+ 'flags to git commands (e.g., git status --ignored).'
966+ )
967+
968+ return local_repo
969+
970+def parse_args(subparsers=None, base_subparsers=None):
971+ kwargs = dict(
972+ description='Clone a source package git repository to a directory',
973+ formatter_class=argparse.RawTextHelpFormatter,
974+ epilog='''
975+Example:
976+ * clone to open-iscsi/
977+ %(prog)s open-iscsi
978+ * clone to ubuntu.git
979+ %(prog)s open-iscsi ubuntu.git
980+ * use git rather than https protocol for remotes:
981+ %(prog)s --proto=git open-iscsi
982+'''
983+ )
984+ if base_subparsers:
985+ kwargs['parents'] = base_subparsers
986+ if subparsers:
987+ parser = subparsers.add_parser('clone', **kwargs)
988+ parser.set_defaults(func=cli_main)
989+ else:
990+ parser = argparse.ArgumentParser(**kwargs)
991+ parser.add_argument('package', type=str,
992+ help='Name of source package to clone'
993+ )
994+ parser.add_argument('directory', type=str,
995+ help='Local directory to clone to. If not specified, a '
996+ ' directory with the same name as PACKAGE will be '
997+ 'used',
998+ default=None,
999+ nargs='?'
1000+ )
1001+ parser.add_argument('-l', '--lp-user', type=str, help=argparse.SUPPRESS)
1002+ if not subparsers:
1003+ return parser.parse_args()
1004+ return 'clone - %s' % kwargs['description']
1005+
1006+def cli_main(args):
1007+ try:
1008+ lp_user = args.lp_user
1009+ except AttributeError:
1010+ lp_user = None
1011+
1012+ if main(
1013+ package=args.package,
1014+ directory=args.directory,
1015+ lp_user=lp_user,
1016+ proto=args.proto,
1017+ ) is not None:
1018+ return 0
1019+ return 1
1020+
1021+
1022+# vi: ts=4 expandtab
1023+>>>>>>> gitubuntu/clone.py
1024diff --git a/gitubuntu/git_repository.py b/gitubuntu/git_repository.py
1025new file mode 100644
1026index 0000000..9a81860
1027--- /dev/null
1028+++ b/gitubuntu/git_repository.py
1029@@ -0,0 +1,3026 @@
1030+<<<<<<< gitubuntu/git_repository.py
1031+=======
1032+### XXX: can we reduce number of calls to dpkg-parsechangelog
1033+### XXX: is any of this data in lp already?
1034+
1035+import collections
1036+from contextlib import contextmanager
1037+from copy import copy
1038+import datetime
1039+import enum
1040+from functools import lru_cache
1041+import itertools
1042+import logging
1043+import os
1044+import posixpath
1045+import re
1046+import shutil
1047+import stat
1048+from subprocess import CalledProcessError
1049+import sys
1050+import tempfile
1051+from gitubuntu.__main__ import top_level_defaults
1052+import gitubuntu.build
1053+from gitubuntu.dsc import component_tarball_matches
1054+from gitubuntu.patch_state import PatchState
1055+from gitubuntu.run import (
1056+ decode_binary,
1057+ run,
1058+ runq,
1059+ run_gbp,
1060+ run_quilt,
1061+)
1062+import gitubuntu.spec
1063+from gitubuntu.test_util import get_test_changelog
1064+import gitubuntu.versioning
1065+import debian.changelog
1066+import debian.debian_support
1067+import pygit2
1068+import pytest
1069+
1070+
1071+def _follow_symlinks_to_blob(repo, top_tree_object, search_path,
1072+ _rel_tree=None, _rel_path=''
1073+):
1074+ '''Recursively follow a path down a tree, following symlinks, to find blob
1075+
1076+ repo: pygit2.Repository object
1077+ top_tree: pygit2.Tree object of the top of the tree structure
1078+ search_path: '/'-separated path string of blob to find
1079+ _rel_tree: (internal) which tree to look further into
1080+ _rel_path: (internal) the path we are in so far
1081+ '''
1082+
1083+ NORMAL_BLOB_MODES = set([
1084+ pygit2.GIT_FILEMODE_BLOB,
1085+ pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
1086+ ])
1087+
1088+ _rel_tree = _rel_tree or top_tree_object
1089+ head, tail = posixpath.split(search_path)
1090+
1091+ # A traditional functional split would put a single entry in head with tail
1092+ # empty, but posixpath.split doesn't necessarily do this. Jiggle it round
1093+ # to make it appear to have traditional semantics.
1094+ if not head:
1095+ head = tail
1096+ tail = None
1097+
1098+ entry = _rel_tree[head]
1099+ if entry.type in [pygit2.GIT_OBJ_TREE, 'tree']:
1100+ return _follow_symlinks_to_blob(
1101+ repo=repo,
1102+ top_tree_object=top_tree_object,
1103+ search_path=tail,
1104+ _rel_tree=repo.get(entry.id),
1105+ _rel_path=posixpath.join(_rel_path, head),
1106+ )
1107+ elif entry.type in [pygit2.GIT_OBJ_BLOB, 'blob'] and entry.filemode == pygit2.GIT_FILEMODE_LINK:
1108+ # Found a symlink. Start again from the top with adjustment for symlink
1109+ # following
1110+ target_tail = [decode_binary(repo.get(entry.id).data)]
1111+ if tail is not None:
1112+ target_tail.append(tail)
1113+ search_path = posixpath.normpath(
1114+ posixpath.join(_rel_path, *target_tail)
1115+ )
1116+ return _follow_symlinks_to_blob(
1117+ repo=repo,
1118+ top_tree_object=top_tree_object,
1119+ search_path=search_path,
1120+ )
1121+ elif entry.type in [pygit2.GIT_OBJ_BLOB, 'blob'] and entry.filemode in NORMAL_BLOB_MODES:
1122+ return repo.get(entry.id)
1123+ else:
1124+ # Found some special entry such as a "gitlink" (submodule entry)
1125+ raise ValueError(
1126+ "Found %r filemode %r looking for %r" %
1127+ (entry, entry.filemode, posixpath.join(_rel_path, search_path))
1128+ )
1129+
1130+
1131+def follow_symlinks_to_blob(repo, treeish_object, path):
1132+ return _follow_symlinks_to_blob(
1133+ repo=repo,
1134+ top_tree_object=treeish_object.peel(pygit2.Tree),
1135+ search_path=posixpath.normpath(path),
1136+ )
1137+
1138+
1139+def _derive_git_cli_env(
1140+ pygit2_repo,
1141+ initial_env=None,
1142+ update_env=None,
1143+ work_tree_path=None,
1144+ index_path=None,
1145+):
1146+ """Calculate the environment to be used in a call to the git CLI
1147+
1148+ :param pygit2.Repository pygit2_repo: the repository for which to calculate
1149+ the environment
1150+ :param dict initial_env: the environment to start with
1151+ :param dict update_env: additional environment setings with which to
1152+ override the result
1153+ :param str work_tree_path: in the case of an alternate work tree being
1154+ used, specify this here and GIT_WORK_TREE will be set to it instead of
1155+ the default being taken from the work tree used by pygit2_repo
1156+ :param str index_path: if an alternate index is being used, specify it here
1157+ and GIT_INDEX_FILE will be set accordingly.
1158+ :rtype: dict
1159+ :returns: a dictionary representing the environment with which to call the
1160+ git CLI
1161+
1162+ This function encapsulates the setting of the GIT_DIR, GIT_WORK_TREE and
1163+ GIT_INDEX_FILE environment variables as necessary. The provided
1164+ pygit2.Repository instance is used to determine these values. initial_env,
1165+ if provided, specifies the initial environment to use instead of defaulting
1166+ to the process' current environment. update_env allows extra environment
1167+ variables to be added as well as the override of any variables set by this
1168+ function, including GIT_DIR, GIT_WORK_TREE and GIT_INDEX_FILE.
1169+ """
1170+ if initial_env is None:
1171+ env = os.environ.copy()
1172+ else:
1173+ env = initial_env.copy()
1174+
1175+ env['GIT_DIR'] = pygit2_repo.path
1176+
1177+ if work_tree_path is None:
1178+ env['GIT_WORK_TREE'] = pygit2_repo.workdir
1179+ else:
1180+ env['GIT_WORK_TREE'] = work_tree_path
1181+
1182+ if index_path is not None:
1183+ env['GIT_INDEX_FILE'] = index_path
1184+
1185+ if update_env:
1186+ env.update(update_env)
1187+
1188+ return env
1189+
1190+
1191+def _derive_target_branch_string(remote_branch_objects):
1192+ '''Given a list of branch objects, return the name of the one to use as the target branch
1193+
1194+ Returns either one of the branch objects' names, or the empty string
1195+ to indicate no suitable candidate.
1196+ '''
1197+ if len(remote_branch_objects) == 0:
1198+ logging.error("Unable to automatically determine importer "
1199+ "branch: No candidate branches found."
1200+ )
1201+ return ''
1202+ remote_branch_strings = [
1203+ b.branch_name for b in remote_branch_objects
1204+ ]
1205+ if len(remote_branch_objects) > 1:
1206+ # do the trees of each branch's tip match?
1207+ if len(
1208+ set(b.peel(pygit2.Tree).id for b in remote_branch_objects)
1209+ ) != 1:
1210+ logging.error("Unable to automatically determine importer "
1211+ "branch: Multiple candidate branches found and "
1212+ "their trees do not match: %s. This might be a "
1213+ "bug in `git ubuntu lint`, please report it at "
1214+ "https://bugs.launchpad.net/git-ubuntu. "
1215+ "Please pass --target-branch.",
1216+ ", ".join(remote_branch_strings)
1217+ )
1218+ return ''
1219+ # is ubuntu/devel one of the candidates?
1220+ try:
1221+ return [
1222+ b for b in remote_branch_strings if 'ubuntu/devel' in b
1223+ ].pop()
1224+ except IndexError:
1225+ pass
1226+ # are all candidate branches for the same series?
1227+ pkg_remote_branch_serieses = set(
1228+ # remove the prefix, trim the distribution and
1229+ # extract the series
1230+ b[len('pkg/'):].split('/')[1].split('-')[0] for
1231+ b in remote_branch_strings
1232+ )
1233+ if len(pkg_remote_branch_serieses) != 1:
1234+ logging.error("Unable to automatically determine importer "
1235+ "branch: Multiple candidate branches found and "
1236+ "they do not target the same series: %s. Please pass "
1237+ "--target-branch.", ", ".join(remote_branch_strings)
1238+ )
1239+ return ''
1240+ # is a -devel branch present?
1241+ if not any('-devel' in b for b in remote_branch_strings):
1242+ logging.error("Unable to automatically determine importer "
1243+ "branch: Multiple candidate branches found and "
1244+ "none appear to be a -devel branch: %s. Please "
1245+ "pass --target-branch.", ", ".join(remote_branch_strings)
1246+ )
1247+ return ''
1248+ # if so, favor -devel
1249+ remote_branch_strings = [
1250+ b for b in remote_branch_strings if '-devel' in b
1251+ ]
1252+ return remote_branch_strings.pop()
1253+
1254+def derive_target_branch(repo, commitish_string, namespace='pkg'):
1255+ return _derive_target_branch_string(
1256+ repo.nearest_remote_branches(commitish_string, namespace)
1257+ )
1258+
1259+
1260+def git_run(
1261+ pygit2_repo,
1262+ args,
1263+ initial_env=None,
1264+ update_env=None,
1265+ work_tree_path=None,
1266+ index_path=None,
1267+ **kwargs
1268+):
1269+ """Run the git CLI with the provided arguments
1270+
1271+ :param pygit2.Repository: the repository on which to act
1272+ :param list(str) args: arguments to the git CLI
1273+ :param dict initial_env: the environment to use
1274+ :param dict update_env: additional environment variables and overrides
1275+ :param dict **kwargs: further arguments to pass through to
1276+ gitubuntu.run.run()
1277+ :raises subprocess.CalledProcessError: if git exits non-zero
1278+ :rtype: (str, str)
1279+ :returns: stdout and stderr strings containing the subprocess output
1280+
1281+ If initial_env is not set, it defaults to the current process' environment.
1282+
1283+ The GIT_DIR, GIT_WORK_TREE and GIT_INDEX_FILE environment variables are set
1284+ automatically as necessary based on the repository's existing location and
1285+ settings.
1286+
1287+ If update_env is set, then the environment to be used is updated with env
1288+ before the call to git is made. This can override GIT_DIR,
1289+ GIT_WORK_TREE, GIT_INDEX_FILE and anything else.
1290+ """
1291+ env = _derive_git_cli_env(
1292+ pygit2_repo=pygit2_repo,
1293+ initial_env=initial_env,
1294+ update_env=update_env,
1295+ work_tree_path=work_tree_path,
1296+ index_path=index_path,
1297+ )
1298+ return run(['git'] + list(args), env=env, **kwargs)
1299+
1300+
1301+class RenameableDir:
1302+ """An on-disk directory that can be renamed and traversed recursively.
1303+
1304+ This is a thin wrapper around a filesystem path string (and must be
1305+ instantiated with one). Methods and attributes are modeled around a
1306+ py.path, but we do not use py.path as we don't really need its
1307+ functionality and it would add another dependency. This interface allows
1308+ for filesystem operations to be easily faked with a FakeRenameableDir for
1309+ testing consumers of this class.
1310+
1311+ One wart around renaming and py.path is that once renamed a py.path object
1312+ becomes useless as it no longer validly refers to an on-disk path. Rather
1313+ than supporting a rename method, this wrapper provides a basename
1314+ setter to handle the rename and replacement wrapped string object
1315+ transparently. This moves complexity away from the class consumer, allowing
1316+ the consumer to be tested more easily.
1317+
1318+ Since the underlying purpose of this class is to handle manipulations of a
1319+ directory tree for adjustments needed during import/export, symlink
1320+ handling is effectively "turned off" in the specification of this class.
1321+ Symlinks to directories are not recursed into; they are handled no
1322+ differently to a regular file, in the same manner as lstat(2).
1323+ """
1324+ def __init__(self, path):
1325+ """Create a new RenameableDir instance.
1326+
1327+ :param str path: the on-disk directory to wrap, which must exist. For
1328+ symlinks, it is the symlink itself that must exist; the existence
1329+ of a symlink's target does not matter.
1330+ :raises FileNotFoundError: if the path supplied does not exist.
1331+ """
1332+ # Ignore the return value of os.lstat(); this call is used to raise
1333+ # FileNotFoundError if the path does not exist (as required in the spec
1334+ # specified by the docstring), or succeed otherwise. The
1335+ # call for os.path.lexists() would use the same underlying system call
1336+ # anyway, so this is equivalent and this way we end up with a full
1337+ # FileNotFoundError exception created for us with all the correct
1338+ # parameters.
1339+ os.lstat(path)
1340+
1341+ self._path = path
1342+
1343+ @property
1344+ def basename(self):
1345+ """The name of the directory itself."""
1346+ return os.path.basename(self._path)
1347+
1348+ @basename.setter
1349+ def basename(self, new_basename):
1350+ """Rename this directory."""
1351+ renamed_path = os.path.join(os.path.dirname(self._path), new_basename)
1352+ os.rename(self._path, renamed_path)
1353+ self._path = renamed_path
1354+
1355+ def listdir(self, fil=lambda x: True):
1356+ """Return subdirectory objects.
1357+
1358+ :param fil: a function that, given a basename, returns a boolean
1359+ indicating whether or not the corresponding object should be
1360+ returned in the results.
1361+ """
1362+ return [
1363+ RenameableDir(os.path.join(self._path, p))
1364+ for p in os.listdir(self._path)
1365+ if fil(p)
1366+ ]
1367+
1368+ @property
1369+ def recursive(self):
1370+ """Indicate if this object can contain subdirectory objects.
1371+
1372+ An object representing a file will return False. An object representing
1373+ a directory will return True, even if it is empty.
1374+
1375+ Symlinks return False even if they point to a directory. Broken
1376+ symlinks also always return False.
1377+
1378+ :rtype: bool
1379+ """
1380+ st = os.stat(self._path, follow_symlinks=False)
1381+ return stat.S_ISDIR(st.st_mode)
1382+
1383+ def __str__(self):
1384+ return str(self._path)
1385+
1386+ def __repr__(self):
1387+ return 'RenameableDir(%r)' % str(self)
1388+
1389+ def __hash__(self):
1390+ # https://stackoverflow.com/q/2909106/478206
1391+ return hash((
1392+ type(self),
1393+ self._path
1394+ ))
1395+
1396+ def __eq__(self, other):
1397+ return hash(self) == hash(other)
1398+
1399+
1400+class FakeRenameableDir:
1401+ """A fake RenameDir that retains its structure in memory.
1402+
1403+ This is useful for testing consumers of a RenameableDir.
1404+
1405+ In addition, renames are recorded and those records passed up to parent
1406+ FakeRenameableDir objects so that the order of renames that occur can be
1407+ checked later.
1408+ """
1409+ def __init__(self, basename, subdirs):
1410+ """Create a new RenameableDir instance.
1411+
1412+ :param str basename: the basename of this instance.
1413+ :param subdirs: FakeRenameableDir objects contained within this one.
1414+ For non-recursive objects (such as those intended to represent
1415+ files), use None.
1416+ :type subdirs: list(FakeRenameableDir)
1417+ """
1418+ self._basename = basename
1419+ self._subdirs = subdirs
1420+
1421+ if self._subdirs:
1422+ for subdir in self._subdirs:
1423+ subdir._parent = self
1424+
1425+ self._parent = None
1426+ self._rename_record = []
1427+
1428+ @property
1429+ def basename(self):
1430+ return self._basename
1431+
1432+ @basename.setter
1433+ def basename(self, new_basename):
1434+ self._record_rename(self)
1435+ self._basename = new_basename
1436+
1437+ def _record_rename(self, obj):
1438+ self._rename_record.append(obj)
1439+ if self._parent:
1440+ self._parent._record_rename(obj)
1441+
1442+ def listdir(self, fil=lambda x: True):
1443+ return (subdir for subdir in self._subdirs if fil(subdir.basename))
1444+
1445+ @property
1446+ def recursive(self):
1447+ return self._subdirs is not None
1448+
1449+ def __hash__(self):
1450+ # https://stackoverflow.com/q/2909106/478206
1451+ return hash((
1452+ type(self),
1453+ self.basename,
1454+ None if self._subdirs is None else tuple(self._subdirs),
1455+ ))
1456+
1457+ def __eq__(self, other):
1458+ return hash(self) == hash(other)
1459+
1460+ def __repr__(self):
1461+ return 'FakeRenameableDir(%r, %r)' % (self.basename, self._subdirs)
1462+
1463+
1464+_dot_git_match = re.compile(r'^\.+git$').search
1465+_EscapeDirection = enum.Enum('EscapeDirection', ['ESCAPE', 'UNESCAPE'])
1466+
1467+
1468+def _escape_unescape_dot_git(path, direction):
1469+ """Escape or unescape .git entries in a directory recursively.
1470+
1471+ :param RenameableDir path: top of directory tree to escape or unescape.
1472+ :param _EscapeDirection direction: whether to escape or unescape.
1473+
1474+ Escaping rules:
1475+ .git -> ..git
1476+ ..git -> ...git
1477+ ...git -> ....git
1478+ etc.
1479+
1480+ All these escaping rules apply all of the time, regardless of whether
1481+ or not .git exists. Only names matching '.git' with zero or more '.'
1482+ prepended are touched.
1483+
1484+ This allows any directory tree to be losslessly stored in git, since git
1485+ does not permit entries named '.git'.
1486+
1487+ Unescaping is the inverse of escaping. Before unescaping, an entry called
1488+ '.git' must not exist. If it does, RuntimeError is raised, and the
1489+ directory is left in an undefined (probably partially unescaped) state.
1490+ """
1491+ # When escaping, we have to rename ..git to ...git before renaming .git to
1492+ # ..git in order to make room, and the reverse for unescaping. If we do the
1493+ # renames ordered by length of name, we can meet this requirement.
1494+ # Escaping: order by longest first; unescaping: order by shortest first.
1495+ sorted_subpaths_to_rename = sorted(
1496+ path.listdir(fil=_dot_git_match),
1497+ key=lambda p: len(p.basename),
1498+ reverse=direction is _EscapeDirection.ESCAPE,
1499+ )
1500+ for entry in sorted_subpaths_to_rename:
1501+ if direction is _EscapeDirection.ESCAPE:
1502+ # Add a leading '.'
1503+ entry.basename = '.' + entry.basename
1504+ else:
1505+ assert direction is _EscapeDirection.UNESCAPE
1506+ if entry.basename == '.git':
1507+ raise RuntimeError(
1508+ "%s exists but is invalid when unescaping" % entry,
1509+ )
1510+ # Drop the leading '.'
1511+ assert entry.basename[0] == '.'
1512+ entry.basename = entry.basename[1:]
1513+
1514+ # Traverse the entire directory for recursive escapes;
1515+ # sorted_subpaths_to_rename is already filtered so is not complete by
1516+ # itself
1517+ for entry in path.listdir():
1518+ if entry.recursive:
1519+ _escape_unescape_dot_git(entry, direction=direction)
1520+
1521+
1522+def escape_dot_git(path):
1523+ """Apply .git escaping to a filesystem path.
1524+
1525+ :param str path: path to filesystem to change
1526+ """
1527+ return _escape_unescape_dot_git(
1528+ path=RenameableDir(path),
1529+ direction=_EscapeDirection.ESCAPE,
1530+ )
1531+
1532+
1533+def unescape_dot_git(path):
1534+ """Unapply .git escaping to a filesystem path.
1535+
1536+ :param str path: path to filesystem to change
1537+
1538+ Any entry (including recursively) called '.git' in path is an error and
1539+ will raise a RuntimeError. If an exception is raised, path may be left in a
1540+ partially unescaped state.
1541+ """
1542+ return _escape_unescape_dot_git(
1543+ path=RenameableDir(path),
1544+ direction=_EscapeDirection.UNESCAPE,
1545+ )
1546+
1547+
1548+class ChangelogError(Exception):
1549+ pass
1550+
1551+class Changelog:
1552+ '''Representation of a debian/changelog file found inside a git tree-ish
1553+
1554+ Uses dpkg-parsechangelog for parsing, but when this fails we fall
1555+ back to grep/sed-based pattern matching automatically.
1556+ '''
1557+ def __init__(self, content_bytes):
1558+ '''
1559+ contents: bytes string of file contents
1560+ '''
1561+ self._contents = content_bytes
1562+ try:
1563+ self._changelog = debian.changelog.Changelog(
1564+ self._contents,
1565+ strict=True
1566+ )
1567+ if not len(self._changelog.versions):
1568+ # assume bad read, so fall back to shell later
1569+ self._changelog = None
1570+ except (
1571+ UnicodeDecodeError,
1572+ ValueError,
1573+ debian.changelog.ChangelogParseError
1574+ ):
1575+ self._changelog = None
1576+
1577+ @classmethod
1578+ def from_treeish(cls, repo, treeish_object):
1579+ '''
1580+ repo: pygit2.Repository instance
1581+ treeish_object: pygit2.Object subclass instance (must peel to pygit2.Tree)
1582+ '''
1583+ blob = follow_symlinks_to_blob(
1584+ repo=repo,
1585+ treeish_object=treeish_object,
1586+ path='debian/changelog'
1587+ )
1588+ return cls(blob.data)
1589+
1590+ @classmethod
1591+ def from_path(cls, path):
1592+ with open(path, 'rb') as f:
1593+ return cls(f.read())
1594+
1595+ @lru_cache()
1596+ def _dpkg_parsechangelog(self, parse_params):
1597+ stdout, _ = run(
1598+ 'dpkg-parsechangelog -l- %s' % parse_params,
1599+ input=self._contents,
1600+ shell=True,
1601+ verbose_on_failure=False,
1602+ )
1603+ return stdout.strip()
1604+
1605+ @lru_cache()
1606+ def _shell(self, cmd):
1607+ stdout, _ = run(
1608+ cmd,
1609+ input=self._contents,
1610+ shell=True,
1611+ verbose_on_failure=False,
1612+ )
1613+ return stdout.strip()
1614+
1615+ @property
1616+ def _shell_version(self):
1617+ parse_params = '-n1 -SVersion'
1618+ shell_cmd = "grep -m1 '^\\S' | sed 's/.*(\\(.*\\)).*/\\1/'"
1619+ try:
1620+ raw_out = self._dpkg_parsechangelog(parse_params)
1621+ except CalledProcessError:
1622+ raw_out = self._shell(shell_cmd)
1623+ return None if raw_out == '' else raw_out
1624+
1625+ @property
1626+ def upstream_version(self):
1627+ if self._changelog:
1628+ return self._changelog.upstream_version
1629+ version = self._shell_version
1630+ m = debian.debian_support.Version.re_valid_version.match(version)
1631+ if m is None:
1632+ raise ValueError("Invalid version string: %s", version)
1633+ return m.group('upstream_version')
1634+
1635+ @property
1636+ def version(self):
1637+ if self._changelog:
1638+ try:
1639+ ret = str(self._changelog.versions[0]).strip()
1640+ shell_version = self._shell_version
1641+ if shell_version != 'unknown' and ret != shell_version:
1642+ raise ChangelogError(
1643+ 'Old (%s) and new (%s) changelog values do not agree' %
1644+ (self._shell_version, ret)
1645+ )
1646+ return ret
1647+ except IndexError:
1648+ return None
1649+ return self._shell_version
1650+
1651+ @property
1652+ def _shell_previous_version(self):
1653+ parse_params = '-n1 -o1 -SVersion'
1654+ shell_cmd = "grep -m1 '^\\S' | tail -1 | sed 's/.*(\\(.*\\)).*/\\1/'"
1655+ try:
1656+ raw_out = self._dpkg_parsechangelog(parse_params)
1657+ except CalledProcessError:
1658+ raw_out = self._shell(shell_cmd)
1659+ return None if raw_out == '' else raw_out
1660+
1661+ @property
1662+ def previous_version(self):
1663+ if self._changelog:
1664+ try:
1665+ ret = str(self._changelog.versions[1]).strip()
1666+ if ret != self._shell_previous_version:
1667+ raise ChangelogError(
1668+ 'Old (%s) and new (%s) changelog values do not agree' %
1669+ (self._shell_previous_version, ret)
1670+ )
1671+ return ret
1672+ except IndexError:
1673+ return None
1674+ return self._shell_previous_version
1675+
1676+ @property
1677+ def _shell_maintainer(self):
1678+ parse_params = '-SMaintainer'
1679+ shell_cmd = "grep -m1 '^ --' | sed 's/ -- \\(.*\\) \\(.*\\)/\\1/'"
1680+ try:
1681+ return self._dpkg_parsechangelog(parse_params)
1682+ except CalledProcessError:
1683+ return self._shell(shell_cmd)
1684+
1685+ @property
1686+ def maintainer(self):
1687+ if self._changelog:
1688+ ret = self._changelog.author.strip()
1689+ if ret != self._shell_maintainer:
1690+ raise ChangelogError(
1691+ 'Old (%s) and new (%s) changelog values do not agree' %
1692+ (self._shell_maintainer, ret)
1693+ )
1694+ else:
1695+ ret = self._shell_maintainer
1696+ if not ret:
1697+ raise ValueError("Unable to parse maintainer from changelog")
1698+ return ret
1699+
1700+ @property
1701+ def _shell_date(self):
1702+ parse_params = '-SDate'
1703+ shell_cmd = "grep -m1 '^ --' | sed 's/ -- \\(.*\\) \\(.*\\)/\\2/'"
1704+ try:
1705+ return self._dpkg_parsechangelog(parse_params)
1706+ except CalledProcessError:
1707+ return self._shell(shell_cmd)
1708+
1709+ @property
1710+ def date(self):
1711+ if self._changelog:
1712+ ret = self._changelog.date.strip()
1713+ if ret != self._shell_date:
1714+ raise ChangelogError(
1715+ 'Old (%s) and new (%s) changelog values do not agree' %
1716+ (self._shell_date, ret)
1717+ )
1718+ return ret
1719+ return self._shell_date
1720+
1721+ @property
1722+ def _shell_all_versions(self):
1723+ parse_params = '--format rfc822 -SVersion --all'
1724+ shell_cmd = "grep '^\\S' | sed 's/.*(\\(.*\\)).*/\\1/'"
1725+ try:
1726+ version_lines = self._dpkg_parsechangelog(parse_params)
1727+ except CalledProcessError:
1728+ version_lines = self._shell(shell_cmd)
1729+ return [
1730+ v_stripped
1731+ for v_stripped in (
1732+ v.strip() for v in version_lines.splitlines()
1733+ )
1734+ if v_stripped
1735+ ]
1736+
1737+ @property
1738+ def all_versions(self):
1739+ if self._changelog:
1740+ ret = [str(v).strip() for v in self._changelog.versions]
1741+ shell_all_versions = self._shell_all_versions
1742+ is_equivalent = (
1743+ len(ret) == len(shell_all_versions) and
1744+ all(
1745+ shell_version == 'unknown' or shell_version == api_version
1746+ for shell_version, api_version
1747+ in zip(shell_all_versions, ret)
1748+ )
1749+ )
1750+ if not is_equivalent:
1751+ raise ChangelogError(
1752+ "Old and new changelog values do not agree"
1753+ )
1754+ return ret
1755+ else:
1756+ return self._shell_all_versions
1757+
1758+ @property
1759+ def _shell_distribution(self):
1760+ parse_params = '-SDistribution'
1761+ shell_cmd = "grep -m1 '^\\S' | sed 's/.*\\ .*\\ \\(.*\\);.*/\\1/'"
1762+ try:
1763+ return self._dpkg_parsechangelog(parse_params)
1764+ except CalledProcessError:
1765+ return self._shell(shell_cmd)
1766+
1767+ @property
1768+ def distribution(self):
1769+ if self._changelog:
1770+ ret = self._changelog.distributions
1771+ if ret != self._shell_distribution:
1772+ raise ChangelogError(
1773+ 'Old (%s) and new (%s) changelog values do not agree' %
1774+ (self._shell_distribution, ret)
1775+ )
1776+ return ret
1777+ return self._shell_distribution
1778+
1779+ @property
1780+ def _shell_srcpkg(self):
1781+ parse_params = '-SSource'
1782+ shell_cmd = "grep -m1 '^\\S' | sed 's/\\(.*\\)\\ .*\\ .*;.*/\\1/'"
1783+ try:
1784+ return self._dpkg_parsechangelog(parse_params)
1785+ except CalledProcessError:
1786+ return self._shell(shell_cmd)
1787+
1788+ @property
1789+ def srcpkg(self):
1790+ if self._changelog:
1791+ ret = self._changelog.package.strip()
1792+ if ret != self._shell_srcpkg:
1793+ raise ChangelogError(
1794+ 'Old (%s) and new (%s) changelog values do not agree' %
1795+ (self._shell_srcpkg, ret)
1796+ )
1797+ return ret
1798+ return self._shell_srcpkg
1799+
1800+ @staticmethod
1801+ def _parse_changelog_date(changelog_timestamp_string):
1802+ """Convert changelog timestamp into datetime object
1803+
1804+ This function currently requires the locale to have been set to C.UTF-8
1805+ by the caller. This would typically be done at the main entry point to
1806+ the importer.
1807+
1808+ :param str changelog_timestamp_string: the timestamp part of the the
1809+ signoff line from a changelog entry
1810+ :rtype: datetime.datetime
1811+ :returns: the timestamp as a datetime object
1812+ :raises ValueError: if the string could not be parsed
1813+ """
1814+ # We avoid using something like dateutil.parser here because the
1815+ # parsing behaviour of malformed or unusually formatted dates must be
1816+ # precisely as specified and not ever change behaviour. If it did, then
1817+ # imports would no longer be reproducible.
1818+ #
1819+ # However, adding new a form of parsing an unambigious date is
1820+ # acceptable if the spec is first updated accordingly since that would
1821+ # only introduce new imports that would have previously failed.
1822+ #
1823+ # time.strptime ignores time zones, so we must use datetime.strptime()
1824+
1825+ # strptime doesn't support anything other than standard locale names
1826+ # for days of the week, so handle the "Thur" abbreviation as a special
1827+ # case as defined in the spec as it is unambiguous.
1828+ adjusted_changelog_timestamp_string = re.sub(
1829+ r'^Thur,',
1830+ 'Thu,',
1831+ changelog_timestamp_string,
1832+ )
1833+
1834+ acceptable_date_formats = [
1835+ '%a, %d %b %Y %H:%M:%S %z', # standard
1836+ '%A, %d %b %Y %H:%M:%S %z', # full day of week
1837+ '%d %b %Y %H:%M:%S %z', # missing day of week
1838+ '%a, %d %B %Y %H:%M:%S %z', # full month name
1839+ '%A, %d %B %Y %H:%M:%S %z', # full day of week and month name
1840+ '%d %B %Y %H:%M:%S %z', # missing day of week with full month
1841+ # name
1842+ ]
1843+ for date_format in acceptable_date_formats:
1844+ try:
1845+ return datetime.datetime.strptime(
1846+ adjusted_changelog_timestamp_string,
1847+ date_format,
1848+ )
1849+ except ValueError:
1850+ pass
1851+ else:
1852+ raise ValueError(
1853+ "Could not parse date %r" % changelog_timestamp_string,
1854+ )
1855+
1856+ def git_authorship(self, author_date=None):
1857+ """Extract last changelog entry's maintainer and timestamp
1858+
1859+ Parse the first changelog entry's sign-off line into git's commit
1860+ authorship metadata model according to the import specification.
1861+
1862+ :param datetime.datetime author_date: overrides the author date
1863+ normally parsed from the changelog entry (i.e. for handling date
1864+ parsing edge cases). Any sub-second part of the timestamp is
1865+ truncated.
1866+ :rtype: tuple(str, str, int, int)
1867+ :returns: tuple of name, email, time (in seconds since epoch) and
1868+ offset from UTC (in minutes)
1869+ :raises ValueError: if the changelog sign-off line cannot be parsed
1870+ """
1871+ m = re.match(r'(?P<name>.*)<+(?P<email>.*?)>+', self.maintainer)
1872+ if m is None:
1873+ raise ValueError('Cannot get authorship')
1874+
1875+ author_epoch_seconds, author_tz_offset = datetime_to_signature_spec(
1876+ self._parse_changelog_date(self.date)
1877+ if author_date is None
1878+ else author_date
1879+ )
1880+
1881+ return (
1882+ # If the author name is empty, then it must be
1883+ # EMPTY_GIT_AUTHOR_NAME because git will not accept an empty author
1884+ # name. See the specification for details.
1885+ (
1886+ m.group('name').strip()
1887+ or gitubuntu.spec.EMPTY_GIT_AUTHOR_NAME
1888+ ),
1889+ m.group('email'),
1890+ author_epoch_seconds,
1891+ author_tz_offset,
1892+ )
1893+
1894+
1895+class GitUbuntuChangelogError(Exception):
1896+ pass
1897+
1898+class PristineTarError(Exception):
1899+ pass
1900+
1901+class PristineTarNotFoundError(PristineTarError):
1902+ pass
1903+
1904+class MultiplePristineTarFoundError(PristineTarError):
1905+ pass
1906+
1907+
1908+def git_dep14_tag(version):
1909+ """Munge a version string according to http://dep.debian.net/deps/dep14/"""
1910+ version = str(version)
1911+ version = version.replace('~', '_')
1912+ version = version.replace(':', '%')
1913+ version = version.replace('..', '.#.')
1914+ if version.endswith('.'):
1915+ version = version + '#'
1916+ if version.endswith('.lock'):
1917+ pre, _, _ = version.partition('.lock')
1918+ version = pre + '.#lock'
1919+ return version
1920+
1921+def import_tag(version, namespace, patch_state=PatchState.UNAPPLIED):
1922+ return '%s/%s/%s' % (
1923+ namespace,
1924+ {
1925+ PatchState.UNAPPLIED: 'import',
1926+ PatchState.APPLIED: 'applied',
1927+ }[patch_state],
1928+ git_dep14_tag(version),
1929+ )
1930+
1931+def reimport_tag_prefix(version, namespace, patch_state=PatchState.UNAPPLIED):
1932+ return '%s/reimport/%s/%s' % (
1933+ namespace,
1934+ {
1935+ PatchState.UNAPPLIED: 'import',
1936+ PatchState.APPLIED: 'applied',
1937+ }[patch_state],
1938+ git_dep14_tag(version),
1939+ )
1940+
1941+def reimport_tag(
1942+ version,
1943+ namespace,
1944+ reimport,
1945+ patch_state=PatchState.UNAPPLIED,
1946+):
1947+ return '%s/%s' % (
1948+ reimport_tag_prefix(version, namespace, patch_state=patch_state),
1949+ reimport,
1950+ )
1951+
1952+def upload_tag(version, namespace):
1953+ return '%s/upload/%s' % (namespace, git_dep14_tag(version))
1954+
1955+def upstream_tag(version, namespace):
1956+ return '%s/upstream/%s' % (namespace, git_dep14_tag(version))
1957+
1958+def orphan_tag(version, namespace):
1959+ return '%s/orphan/%s' % (namespace, git_dep14_tag(version))
1960+
1961+def is_dir_3_0_quilt(_dir=None):
1962+ _dir = _dir if _dir else '.'
1963+ try:
1964+ fmt, _ = run(['dpkg-source', '--print-format', _dir])
1965+ if '3.0 (quilt)' in fmt:
1966+ return True
1967+ except CalledProcessError as e:
1968+ try:
1969+ with open(os.path.join(_dir, 'debian/source/format'), 'r') as f:
1970+ for line in f:
1971+ if re.match(r'3.0 (.*)', line):
1972+ return True
1973+ # `man dpkg-source` indicates no d/s/format implies 1.0
1974+ except OSError:
1975+ pass
1976+
1977+ return False
1978+
1979+def is_3_0_quilt(repo, commitish='HEAD'):
1980+ with repo.temporary_worktree(commitish):
1981+ return is_dir_3_0_quilt()
1982+
1983+class GitUbuntuRepositoryFetchError(Exception):
1984+ pass
1985+
1986+
1987+def determine_quilt_series_path(pygit2_repo, treeish_obj):
1988+ """Find the active quilt series file path in use.
1989+
1990+ Look in the given tree for the Debian patch series file that is active
1991+ according to the search algorithm described in dpkg-source(1). If none are
1992+ found, return the default series path (again from dpkg-source(1)).
1993+
1994+ :param pygit2.Repo pygit2_repo: repository to look in.
1995+ :param pygit2.Object treeish_obj: object that peels to a pygit2.Tree.
1996+ :returns: relative path to series file.
1997+ :rtype: str
1998+ """
1999+ for series_name in ['debian.series', 'series']:
2000+ try:
2001+ series_path = posixpath.join('debian/patches', series_name)
2002+ blob = follow_symlinks_to_blob(
2003+ repo=pygit2_repo,
2004+ treeish_object=treeish_obj,
2005+ path=series_path,
2006+ )
2007+ except KeyError:
2008+ continue # try the next path using our search list
2009+ return series_path # series file blob found at this path
2010+
2011+ logging.debug("Unable to find a series file in %r", treeish_obj)
2012+ return 'debian/patches/series' # default when no series file found
2013+
2014+
2015+def quilt_env(pygit2_repo, treeish):
2016+ """Find the appropriate quilt environment to use.
2017+
2018+ Return the canonical environment that should be used when calling quilt.
2019+ Since the series file doesn't necessarily always have the same name, a
2020+ source tree is examined to determine the name and set QUILT_SERIES
2021+ appropriately.
2022+
2023+ This does not integrate any other environment variables. Only environment
2024+ variables that influence quilt are returned.
2025+
2026+ :param pygit2.Repo pygit2_repo: repository to look in.
2027+ :param pygit2.Object treeish: object that peels to a pygit2.Tree.
2028+ :returns: quilt-specific environment settings
2029+ :rtype: dict
2030+ """
2031+ return {
2032+ 'QUILT_PATCHES': 'debian/patches',
2033+ 'QUILT_SERIES': determine_quilt_series_path(pygit2_repo, treeish),
2034+ 'QUILT_NO_DIFF_INDEX': '1',
2035+ 'QUILT_NO_DIFF_TIMESTAMPS': '1',
2036+ 'EDITOR': 'true',
2037+ }
2038+
2039+
2040+def datetime_to_signature_spec(datetime):
2041+ """Convert a datetime to the time and offset required by a pygit2.Signature
2042+
2043+ :param datetime datetime: the timezone-aware datetime to convert
2044+ :rtype: tuple(int, int)
2045+ :returns: the time since epoch and timezone offset in minutes as suitable
2046+ for passing to the pygit2.Signature constructor parameters time and
2047+ offset.
2048+ """
2049+ # Divide by 60 for seconds -> minutes
2050+ offset_td = datetime.utcoffset()
2051+ offset_mins = (
2052+ int(offset_td.total_seconds()) // 60
2053+ if offset_td
2054+ else 0
2055+ )
2056+
2057+ return int(datetime.timestamp()), offset_mins
2058+
2059+
2060+class HeadInfoItem(collections.namedtuple(
2061+ 'HeadInfoItem',
2062+ [
2063+ 'version',
2064+ 'commit_time',
2065+ 'commit_id',
2066+ ],
2067+)):
2068+ """Information associated with a single branch head
2069+
2070+ :ivar str version: the package version found in debian/changelog at the
2071+ branch head.
2072+ :ivar int commit_time: the timestamp of the commit at the branch head,
2073+ expressed as seconds since the Unix epoch.
2074+ :ivar pygit2.Oid commit_id: the hash of the commit at the branch head.
2075+ """
2076+ pass
2077+
2078+
2079+class GitUbuntuRepository:
2080+ """A class for interacting with an importer git repository
2081+
2082+ This class attempts to put all objects it manipulates in an
2083+ 'importer/' namespace. It also uses tags in one of three namespaces:
2084+ 'import/' for successfully imported published versions (these are
2085+ created by the importer); 'upload/' for uploaded version by Ubuntu
2086+ developers (these are understood by the importer and are aliased by
2087+ import/ tags when succesfully imported); and 'orphan/' for published
2088+ versions for which no parents can be found (these are also created
2089+ by the importer).
2090+
2091+ To access the underlying pygit2.Repository object, use the raw_repo
2092+ property.
2093+ """
2094+
2095+ def __init__(
2096+ self,
2097+ local_dir,
2098+ lp_user=None,
2099+ fetch_proto=None,
2100+ delete_on_close=True,
2101+ ):
2102+ """
2103+ If fetch_proto is None, the default value from
2104+ gitubuntu.__main__ will be used (top_level_defaults.proto).
2105+ """
2106+ if local_dir is None:
2107+ self._local_dir = tempfile.mkdtemp()
2108+ else:
2109+ local_dir = os.path.abspath(local_dir)
2110+ try:
2111+ os.mkdir(local_dir)
2112+ except FileExistsError:
2113+ local_dir_list = os.listdir(local_dir)
2114+ if local_dir_list and os.getenv(
2115+ 'GIT_DIR',
2116+ '.git',
2117+ ) not in local_dir_list:
2118+ logging.error('Specified directory %s must either '
2119+ 'be empty or have been previously '
2120+ 'imported to.', local_dir)
2121+ sys.exit(1)
2122+ self._local_dir = local_dir
2123+
2124+ self.raw_repo = pygit2.init_repository(self._local_dir)
2125+ # We rely on raw_repo.workdir to be identical to self._local_dir to
2126+ # avoid changing previous behaviour in the setting of GIT_WORK_TREE, so
2127+ # assert that it is so. This may not be the case if the git repository
2128+ # has a different workdir stored in its configuration or if the git
2129+ # repository is a bare repository. We didn't handle these cases before
2130+ # anyway, so with this assertion we can fail noisily and early.
2131+ assert (
2132+ os.path.normpath(self.raw_repo.workdir) ==
2133+ os.path.normpath(self._local_dir)
2134+ )
2135+
2136+ # Since previous behaviour of this class depended on the state of the
2137+ # environment at the time it was constructed, save this for later use
2138+ # (for example in deriving the environment to use for calls to the git
2139+ # CLI). This permits the behaviour to remain identical for now.
2140+ # Eventually we can break previous behaviour and eliminate the need for
2141+ # this. See also: gitubuntu.test_fixtures.repo; the handling of EMAIL
2142+ # there could be made cleaner when this is cleaned up.
2143+ self._initial_env = os.environ.copy()
2144+
2145+ self.set_git_attributes()
2146+
2147+ if lp_user:
2148+ self._lp_user = lp_user
2149+ else:
2150+ try:
2151+ self._lp_user, _ = self.git_run(
2152+ ['config', 'gitubuntu.lpuser'],
2153+ verbose_on_failure=False,
2154+ )
2155+ self._lp_user = self._lp_user.strip()
2156+ except CalledProcessError:
2157+ self._lp_user = None
2158+
2159+ if fetch_proto is None:
2160+ fetch_proto = top_level_defaults.proto
2161+
2162+ self._fetch_proto = fetch_proto
2163+ self._delete_on_close = delete_on_close
2164+
2165+ def close(self):
2166+ """Free resources associated with this instance
2167+
2168+ If delete_on_close was True on instance construction, local_dir (as
2169+ specified on instance construction) will be deleted.
2170+
2171+ After this method is called, the instance is invalid and can no longer
2172+ be used.
2173+ """
2174+ if self.raw_repo and self._delete_on_close:
2175+ shutil.rmtree(self.local_dir)
2176+ self.raw_repo = None
2177+
2178+ def create_orphan_branch(self, branch_name, msg):
2179+ if self.get_head_by_name(branch_name) is None:
2180+ self.git_run(['checkout', '--orphan', branch_name])
2181+ self.git_run(['commit', '--allow-empty', '-m', msg])
2182+ self.git_run(['checkout', '--orphan', 'master'])
2183+
2184+ @contextmanager
2185+ def pristine_tar_branches(self, dist, namespace='pkg', create=True):
2186+ """Context manager wrapping pristine-tar branch manipulation
2187+
2188+ In this context, the repository pristine-tar branch will point to
2189+ the pristine-tar branch for @dist distribution in @namespace.
2190+
2191+ Because of our model, the distribution-pristine-tar branch may
2192+ be a local branch (import-time) or a remote-tracking branch
2193+ (build-time) and we need different behavior in both cases.
2194+ Specifically, we want to affect the local branch's contents, but
2195+ we cannot do that to a remote-tracking branch.
2196+
2197+ Upon entry to the context, detect the former case (by doing a
2198+ local only lookup first) and doing a branch rename there.
2199+ Otherwise, create a new local branch.
2200+
2201+ Upon exit, if a local branch had been found, rename pristine-tar
2202+ back to the original name. Otherwise, simply delete the created
2203+ pristine-tar branch.
2204+
2205+ If a local branch named pristine-tar existed outside this
2206+ context, it will be restored upon leaving the context.
2207+
2208+ :param dist str One of 'ubuntu' or 'debian'
2209+ :param namespace str Namespace under which Git refs are found
2210+ :param create bool If an appropriate local pristine-tar Git
2211+ branch does not exist, create one using the above algorithm.
2212+ """
2213+ pt_branch = '%s/importer/%s/pristine-tar' % (namespace, dist)
2214+ old_pt_branch = self.raw_repo.lookup_branch('pristine-tar')
2215+ old_pt_branch_commit = None
2216+ if old_pt_branch:
2217+ old_pt_branch_commit = old_pt_branch.peel(pygit2.Commit)
2218+ old_pt_branch.delete()
2219+ local_pt_branch = self.raw_repo.lookup_branch(pt_branch)
2220+ remote_pt_branch = self.raw_repo.lookup_branch(
2221+ pt_branch,
2222+ pygit2.GIT_BRANCH_REMOTE,
2223+ )
2224+ if local_pt_branch:
2225+ local_pt_branch.rename('pristine-tar')
2226+ elif remote_pt_branch:
2227+ self.raw_repo.create_branch(
2228+ 'pristine-tar',
2229+ remote_pt_branch.peel(pygit2.Commit),
2230+ )
2231+ elif create:
2232+ # This should only be possible when importing and the first
2233+ # pristine-tar usage, create an orphan branch at the local
2234+ # pt branch location and flag it for cleanup
2235+ local_pt_branch = True
2236+ self.create_orphan_branch(
2237+ 'pristine-tar',
2238+ 'Initial %s pristine-tar branch.' % dist,
2239+ )
2240+ if not self.raw_repo.lookup_branch('do-not-push'):
2241+ self.create_orphan_branch(
2242+ 'do-not-push',
2243+ 'Initial upstream branch.',
2244+ )
2245+ try:
2246+ yield
2247+ except:
2248+ raise
2249+ finally:
2250+ if local_pt_branch: # or create above
2251+ self.raw_repo.lookup_branch('pristine-tar').rename(pt_branch)
2252+ elif remote_pt_branch:
2253+ self.raw_repo.lookup_branch('pristine-tar').delete()
2254+ if old_pt_branch_commit:
2255+ self.raw_repo.create_branch(
2256+ 'pristine-tar',
2257+ old_pt_branch_commit,
2258+ )
2259+
2260+ def pristine_tar_list(self, dist, namespace='pkg'):
2261+ """List tarballs stored in pristine-tar branch for @dist distribution in @namespace.
2262+
2263+ If there is no pristine-tar branch, `pristine-tar list` returns
2264+ nothing.
2265+
2266+ :param dist str One of 'ubuntu' or 'debian'
2267+ :param namespace str Namespace under which Git refs are found
2268+ :rtype list(str)
2269+ :returns List of orig tarball names stored in the pristine-tar
2270+ branches
2271+ """
2272+ with self.pristine_tar_branches(dist, namespace, create=False):
2273+ stdout, _ = run(['pristine-tar', 'list'])
2274+ return stdout.strip().splitlines()
2275+
2276+ def pristine_tar_extract(self, pkgname, version, dist=None, namespace='pkg'):
2277+ '''Extract orig tarballs for a given package and upstream version
2278+
2279+ This function will fail if the expected tarballs are already
2280+ present by name in the parent directory. If, at some point, this
2281+ is not desired, we would need to pass --git-force-create to
2282+ gbp-buildpackage.
2283+
2284+ The files, once created, are the responsibility of the caller to
2285+ remove, if necessary.
2286+
2287+ raises:
2288+ - PristineTarNotFoundError if no suitable tarballs are found
2289+ - MultiplePristineTarFoundError if multiple distinct suitable tarballs
2290+ are found
2291+ - CalledProcessError if gbp-buildpackage fails
2292+
2293+ :param pkgname str Source package name
2294+ :param version str Source package upstream version
2295+ :param dist str One of 'ubuntu' or 'debian'
2296+ :param namespace str Namespace under which Git refs are found
2297+ :rtype list(str)
2298+ :returns List of tarball paths that are now present on the
2299+ filesystem. They will be in the parent directory.
2300+ '''
2301+ dists = [dist] if dist else ['debian', 'ubuntu']
2302+ for dist in dists:
2303+ main_tarball = '%s_%s.orig.tar' % (pkgname, version)
2304+
2305+ all_tarballs = self.pristine_tar_list(dist, namespace)
2306+
2307+ potential_main_tarballs = [tarball for tarball
2308+ in all_tarballs if tarball.startswith(main_tarball)]
2309+ if len(potential_main_tarballs) == 0:
2310+ continue
2311+ if len(potential_main_tarballs) > 1:
2312+ # This will need some extension/flag for the case of there
2313+ # being multiple imports with varying compression
2314+ raise MultiplePristineTarFoundError(
2315+ 'More than one pristine-tar tarball found for %s: %s' %
2316+ (version, potential_main_tarballs)
2317+ )
2318+ ext = os.path.splitext(potential_main_tarballs[0])[1]
2319+ tarballs = []
2320+ tarballs.append(
2321+ os.path.join(os.path.pardir, potential_main_tarballs[0])
2322+ )
2323+ args = ['buildpackage', '--git-builder=/bin/true',
2324+ '--git-pristine-tar', '--git-ignore-branch',
2325+ '--git-upstream-tag=%s/upstream/%s/%%(version)s%s' %
2326+ (namespace, dist, ext)]
2327+ # This will probably break if the component tarballs get
2328+ # compressed differently, as each component tarball will show up
2329+ # multiple times
2330+ # Breaks may be too strong -- we will 'over cache' tarballs, and
2331+ # then it's up to dpkg-buildpackage to use the 'correct' one
2332+ potential_component_tarballs = {
2333+ component_tarball_matches(tarball, pkgname, version).group('component') : tarball
2334+ for tarball in all_tarballs
2335+ if component_tarball_matches(tarball, pkgname, version)
2336+ }
2337+ tarballs.extend(map(lambda x : os.path.join(os.path.pardir, x),
2338+ list(potential_component_tarballs.values()))
2339+ )
2340+ args.extend(map(lambda x : '--git-component=%s' % x,
2341+ list(potential_component_tarballs.keys()))
2342+ )
2343+ with self.pristine_tar_branches(dist, namespace):
2344+ run_gbp(args, env=self.env)
2345+ return tarballs
2346+
2347+ raise PristineTarNotFoundError(
2348+ 'No pristine-tar tarball found for %s' % version
2349+ )
2350+
2351+ def pristine_tar_exists(self, pkgname, version, namespace='pkg'):
2352+ '''Report distributions that contain pristine-tar data for @version
2353+
2354+ raises:
2355+ - MultiplePristineTarFoundError if multiple distinct suitable tarballs
2356+ are found
2357+
2358+ :param pkgname str Source package name
2359+ :param version str Source package upstream version
2360+ :param namespace str Namespace under which Git refs are found
2361+ :rtype list(str)
2362+ :returns List of distribution names which contain a pristine-tar
2363+ import for @pkgname and @version
2364+ '''
2365+ results = []
2366+ for dist in ['debian', 'ubuntu']:
2367+ main_tarball = '%s_%s.orig.tar' % (pkgname, version)
2368+
2369+ all_tarballs = self.pristine_tar_list(dist, namespace)
2370+
2371+ potential_main_tarballs = [tarball for tarball
2372+ in all_tarballs if tarball.startswith(main_tarball)]
2373+ if len(potential_main_tarballs) == 0:
2374+ continue
2375+ if len(potential_main_tarballs) > 1:
2376+ # This will need some extension/flag for the case of there
2377+ # being multiple imports with varying compression
2378+ raise MultiplePristineTarFoundError(
2379+ 'More than one pristine-tar tarball found for %s: %s' %
2380+ (version, potential_main_tarballs)
2381+ )
2382+ results.append(dist)
2383+
2384+ return results
2385+
2386+ def verify_pristine_tar(self, tarball_paths, dist, namespace='pkg'):
2387+ '''Verify the pristine-tar data matches for a set of paths
2388+
2389+ raises:
2390+ PristineTarError - if a tarball has been imported before,
2391+ but the contents of the new tarball do not match
2392+
2393+ :param tarball_paths list(str) List of filesystem paths of orig
2394+ tarballs to verify
2395+ :param dist str One of 'ubuntu' or 'debian'
2396+ :param namespace str Namespace under which Git refs are found
2397+ :rtype bool
2398+ :returns True if all paths in @tarball_paths exist in @dist's
2399+ pristine-tar branch under @namespace and match the
2400+ corresponding pristine-tar contents exactly
2401+ '''
2402+ all_tarballs = self.pristine_tar_list(dist, namespace)
2403+ for path in tarball_paths:
2404+ if os.path.basename(path) not in all_tarballs:
2405+ break
2406+ try:
2407+ with self.pristine_tar_branches(dist, namespace):
2408+ # need to handle this not existing
2409+ run(['pristine-tar', 'verify', path])
2410+ except CalledProcessError as e:
2411+ raise PristineTarError(
2412+ 'Tarball has already been imported to %s with '
2413+ 'different contents' % dist
2414+ )
2415+ else:
2416+ return True
2417+
2418+ return False
2419+
2420+ def set_git_attributes(self):
2421+ git_attr_path = os.path.join(self.raw_repo.path,
2422+ 'info',
2423+ 'attributes'
2424+ )
2425+ try:
2426+ # common-case: create an attributes file
2427+ with open(git_attr_path, 'x') as f:
2428+ f.write('* -ident\n')
2429+ f.write('* -text\n')
2430+ f.write('* -eol\n')
2431+ except FileExistsError:
2432+ # next-most common-case: attributes file already exists and
2433+ # contains our desired value
2434+ try:
2435+ runq(['grep', '-q', '* -ident', git_attr_path])
2436+ except CalledProcessError:
2437+ # least-common case: attributes file exists, but does
2438+ # not contain our desired value
2439+ try:
2440+ with open(git_attr_path, 'a') as f:
2441+ f.write('* -ident\n')
2442+ except:
2443+ # failed all three cases to set our desired value in
2444+ # attributes file
2445+ logging.exception('Unable to set \'* -ident\' in %s' %
2446+ git_attr_path
2447+ )
2448+ sys.exit(1)
2449+ try:
2450+ runq(['grep', '-q', '* -text', git_attr_path])
2451+ except CalledProcessError:
2452+ # least-common case: attributes file exists, but does
2453+ # not contain our desired value
2454+ try:
2455+ with open(git_attr_path, 'a') as f:
2456+ f.write('* -text\n')
2457+ except:
2458+ # failed all three cases to set our desired value in
2459+ # attributes file
2460+ logging.exception('Unable to set \'* -text\' in %s' %
2461+ git_attr_path
2462+ )
2463+ sys.exit(1)
2464+ try:
2465+ runq(['grep', '-q', '* -eol', git_attr_path])
2466+ except CalledProcessError:
2467+ # least-common case: attributes file exists, but does
2468+ # not contain our desired value
2469+ try:
2470+ with open(git_attr_path, 'a') as f:
2471+ f.write('* -eol\n')
2472+ except:
2473+ # failed all three cases to set our desired value in
2474+ # attributes file
2475+ logging.exception('Unable to set \'* -eol\' in %s' %
2476+ git_attr_path
2477+ )
2478+ sys.exit(1)
2479+
2480+ def remote_exists(self, remote_name):
2481+ # https://github.com/libgit2/pygit2/issues/671
2482+ return any(remote.name == remote_name for remote in self.raw_repo.remotes)
2483+
2484+ def _add_remote_by_fetch_url(
2485+ self,
2486+ remote_name,
2487+ fetch_url,
2488+ push_url=None,
2489+ changelog_notes=False,
2490+ ):
2491+ """Add a remote by URL
2492+
2493+ If a remote with the given name doesn't exist, then create it.
2494+ Otherwise, do nothing.
2495+
2496+ :param str remote_name: the name of the remote to create
2497+ :param str fetch_url: the fetch URL for the remote
2498+ :param str push_url: the push URL for the remote. If None, then a
2499+ specific push URL will not be set.
2500+ :param bool changelog_notes: if True, then a fetch refspec will be
2501+ added to fetch changelog notes. This only makes sense for an
2502+ official importer remote such as 'pkg'.
2503+ :returns: None
2504+ """
2505+ if not self._fetch_proto:
2506+ raise Exception('Cannot fetch using an object without a protocol')
2507+
2508+ logging.debug('Adding %s as remote %s', fetch_url, remote_name)
2509+
2510+ if not self.remote_exists(remote_name):
2511+ self.raw_repo.remotes.create(
2512+ remote_name,
2513+ fetch_url,
2514+ '+refs/heads/*:refs/remotes/%s/*' % remote_name,
2515+ )
2516+ # grab unreachable tags (orphans)
2517+ self.raw_repo.remotes.add_fetch(
2518+ remote_name,
2519+ '+refs/tags/*:refs/tags/%s/*' % remote_name,
2520+ )
2521+ if changelog_notes:
2522+ # The changelog notes are kept at refs/notes/commits on
2523+ # Launchpad due to LP: #1871838 even though our standard place
2524+ # for them is refs/notes/changelog.
2525+ self.raw_repo.remotes.add_fetch(
2526+ remote_name,
2527+ '+refs/notes/commits:refs/notes/changelog',
2528+ )
2529+ if push_url:
2530+ self.raw_repo.remotes.set_push_url(
2531+ remote_name,
2532+ push_url,
2533+ )
2534+ self.git_run(
2535+ [
2536+ 'config',
2537+ 'remote.%s.tagOpt' % remote_name,
2538+ '--no-tags',
2539+ ]
2540+ )
2541+
2542+ def _add_remote(self, remote_name, remote_url, changelog_notes=False):
2543+ """Add a remote by URL location
2544+
2545+ URL location means the part of the URL after the proto:// prefix. The
2546+ protocol to be used will be determined by what was specified by the
2547+ fetch_proto at class instance construction time. Separate fetch and
2548+ push URL protocols will be automatically determined.
2549+
2550+ If a remote with the given name doesn't exist, then create it.
2551+ Otherwise, do nothing.
2552+
2553+ :param str remote_name: the name of the remote to create
2554+ :param str remote_url: the URL for the remote but with the proto://
2555+ prefix missing.
2556+ :param bool changelog_notes: if True, then a fetch refspec will be
2557+ added to fetch changelog notes. This only makes sense for an
2558+ official importer remote such as 'pkg'.
2559+ :returns: None
2560+ """
2561+ if not self._fetch_proto:
2562+ raise Exception('Cannot fetch using an object without a protocol')
2563+ if not self._lp_user:
2564+ raise RuntimeError("Cannot add remote without knowing lp_user")
2565+ fetch_url = '%s://%s' % (self._fetch_proto, remote_url)
2566+ push_url = 'ssh://%s@%s' % (self.lp_user, remote_url)
2567+
2568+ self._add_remote_by_fetch_url(
2569+ remote_name=remote_name,
2570+ fetch_url=fetch_url,
2571+ push_url=push_url,
2572+ changelog_notes=changelog_notes,
2573+ )
2574+
2575+ def add_remote(
2576+ self,
2577+ pkgname,
2578+ repo_owner,
2579+ remote_name,
2580+ changelog_notes=False,
2581+ ):
2582+ """Add a remote to the repository configuration
2583+ :param str pkgname: the name of the source package reflected by this
2584+ repository.
2585+ :param str repo_owner: the name of the Launchpad user or team whose
2586+ repository for the package will be pointed to by this new remote.
2587+ If None, the default repository for the source package will be
2588+ used.
2589+ :param str remote_name: the name of the remote to add.
2590+ :param bool changelog_notes: if True, then a fetch refspec will be
2591+ added to fetch changelog notes. This only makes sense for an
2592+ official importer remote such as 'pkg'.
2593+ :returns: None
2594+ """
2595+ if not self._fetch_proto:
2596+ raise Exception('Cannot fetch using an object without a protocol')
2597+ if repo_owner:
2598+ remote_url = ('git.launchpad.net/~%s/ubuntu/+source/%s' %
2599+ (repo_owner, pkgname))
2600+ else:
2601+ remote_url = ('git.launchpad.net/ubuntu/+source/%s' % pkgname)
2602+
2603+ self._add_remote(
2604+ remote_name=remote_name,
2605+ remote_url=remote_url,
2606+ changelog_notes=changelog_notes,
2607+ )
2608+
2609+ def add_remote_by_url(self, remote_name, fetch_url):
2610+ if not self._fetch_proto:
2611+ raise Exception('Cannot fetch using an object without a protocol')
2612+
2613+ self._add_remote_by_fetch_url(remote_name, fetch_url)
2614+
2615+ def add_base_remotes(self, pkgname, repo_owner=None):
2616+ """Add the 'pkg' base remote to the repository configuration
2617+
2618+ :param str pkgname: the name of the source package reflected by this
2619+ repository.
2620+ :param str repo_owner: the name of the Launchpad user or team whose
2621+ repository for the package will be pointed to by this new remote.
2622+ If None, the default repository for the source package will be
2623+ used.
2624+ :returns: None
2625+ """
2626+ self.add_remote(pkgname, repo_owner, 'pkg', changelog_notes=True)
2627+
2628+ def add_lpuser_remote(self, pkgname):
2629+ if not self._fetch_proto:
2630+ raise Exception('Cannot add a fetch using an object without a protocol')
2631+ if not self._lp_user:
2632+ raise RuntimeError("Cannot add remote without knowing lp_user")
2633+ remote_url = ('git.launchpad.net/~%s/ubuntu/+source/%s' %
2634+ (self.lp_user, pkgname))
2635+
2636+ self._add_remote(remote_name=self.lp_user, remote_url=remote_url)
2637+ # XXX: want a remote alias of 'lpme' -> self.lp_user
2638+ # self.git_run(['config', 'url.%s.insteadof' % self.lp_user, 'lpme'])
2639+
2640+ def fetch_remote(self, remote_name, verbose=False):
2641+ # Does not seem to be working with https
2642+ # https://github.com/libgit2/pygit2/issues/573
2643+ # https://github.com/libgit2/libgit2/issues/3786
2644+ # self.raw_repo.remotes[remote_name].fetch()
2645+ kwargs = {}
2646+ kwargs['verbose_on_failure'] = True
2647+ if verbose:
2648+ # If we are redirecting stdout/stderr to the console, we
2649+ # do not need to have run() also emit it
2650+ kwargs['verbose_on_failure'] = False
2651+ kwargs['stdout'] = None
2652+ kwargs['stderr'] = None
2653+ try:
2654+ logging.debug("Fetching remote %s", remote_name)
2655+ self.git_run(
2656+ args=['fetch', remote_name],
2657+ env={'GIT_TERMINAL_PROMPT': '0',},
2658+ **kwargs
2659+ )
2660+ except CalledProcessError:
2661+ raise GitUbuntuRepositoryFetchError(
2662+ "Unable to fetch remote %s" % remote_name
2663+ )
2664+
2665+ def fetch_base_remotes(self, verbose=False):
2666+ self.fetch_remote(remote_name='pkg', verbose=verbose)
2667+
2668+ def fetch_remote_refspecs(self, remote_name, refspecs, verbose=False):
2669+ # Does not seem to be working with https
2670+ # https://github.com/libgit2/pygit2/issues/573
2671+ # https://github.com/libgit2/libgit2/issues/3786
2672+ # self.raw_repo.remotes[remote_name].fetch()
2673+ for refspec in refspecs:
2674+ kwargs = {}
2675+ kwargs['verbose_on_failure'] = True
2676+ if verbose:
2677+ # If we are redirecting stdout/stderr to the console, we
2678+ # do not need to have run() also emit it
2679+ kwargs['verbose_on_failure'] = False
2680+ kwargs['stdout'] = None
2681+ kwargs['stderr'] = None
2682+ try:
2683+ logging.debug(
2684+ "Fetching refspec %s from remote %s",
2685+ refspec,
2686+ remote_name,
2687+ )
2688+ self.git_run(
2689+ args=['fetch', remote_name, refspec],
2690+ env={'GIT_TERMINAL_PROMPT': '0',},
2691+ **kwargs,
2692+ )
2693+ except CalledProcessError:
2694+ raise GitUbuntuRepositoryFetchError(
2695+ "Unable to fetch %s from remote %s" % (
2696+ refspecs,
2697+ remote_name,
2698+ )
2699+ )
2700+
2701+ def fetch_lpuser_remote(self, verbose=False):
2702+ if not self._fetch_proto:
2703+ raise Exception('Cannot fetch using an object without a protocol')
2704+ if not self._lp_user:
2705+ raise RuntimeError("Cannot fetch without knowing lp_user")
2706+ self.fetch_remote(remote_name=self.lp_user, verbose=verbose)
2707+
2708+ def copy_base_references(self, namespace):
2709+ for ref in self.references:
2710+ for (target_refs, source_refs) in [
2711+ ('refs/heads/%s/' % namespace, 'refs/remotes/pkg/'),]:
2712+ if ref.name.startswith(source_refs):
2713+ self.raw_repo.create_reference(
2714+ '%s/%s' % (target_refs, ref.name[len(source_refs):]),
2715+ ref.peel().id)
2716+
2717+ def delete_branches_in_namespace(self, namespace):
2718+ _local_branches = copy(self.local_branches)
2719+ for head in self.local_branches:
2720+ if head.branch_name.startswith(namespace):
2721+ head.delete()
2722+
2723+ def delete_tags_in_namespace(self, namespace):
2724+ _tags = copy(self.tags)
2725+ for ref in self.tags:
2726+ if ref.name.startswith('refs/tags/%s' % namespace):
2727+ ref.delete()
2728+
2729+ @property
2730+ def env(self):
2731+ # Return a copy of the cached _derive_env method result so that the
2732+ # caller cannot inadvertently modify our cached answer. Unfortunately
2733+ # this leaks the lru_cache-ness of the _derive_env method to this
2734+ # property getter, but this seems better than nothing.
2735+ return dict(self._derive_env())
2736+
2737+ @lru_cache()
2738+ def _derive_env(self):
2739+ """Determine what the git CLI environment should be
2740+
2741+ This depends on the initial environment saved from the constructor and
2742+ the paths associated with self.raw_repo, neither of which should change
2743+ in the lifetime of this class instance.
2744+ """
2745+ return _derive_git_cli_env(
2746+ self.raw_repo,
2747+ initial_env=self._initial_env
2748+ )
2749+
2750+ @property
2751+ def local_dir(self):
2752+ """Base directory of this git repository (contains .git/)"""
2753+ return self._local_dir
2754+
2755+ @property
2756+ def git_dir(self):
2757+ """Same as cached object in the environment"""
2758+ return self.raw_repo.path
2759+
2760+ def _references(self, prefix=''):
2761+ return [self.raw_repo.lookup_reference(r) for r in
2762+ self.raw_repo.listall_references() if
2763+ r.startswith(prefix)]
2764+
2765+ def references_with_prefix(self, prefix):
2766+ return self._references(prefix)
2767+
2768+ @property
2769+ def references(self):
2770+ return self._references()
2771+
2772+ @property
2773+ def tags(self):
2774+ return self._references('refs/tags')
2775+
2776+ def _branches(self,
2777+ branch_type=pygit2.GIT_BRANCH_LOCAL | pygit2.GIT_BRANCH_REMOTE):
2778+ branches = []
2779+ if branch_type & pygit2.GIT_BRANCH_LOCAL:
2780+ branches.extend([self.raw_repo.lookup_branch(b) for b in
2781+ self.raw_repo.listall_branches(pygit2.GIT_BRANCH_LOCAL)])
2782+ if branch_type & pygit2.GIT_BRANCH_REMOTE:
2783+ branches.extend([self.raw_repo.lookup_branch(b, pygit2.GIT_BRANCH_REMOTE) for b in
2784+ self.raw_repo.listall_branches(pygit2.GIT_BRANCH_REMOTE)])
2785+ return branches
2786+
2787+ @property
2788+ def branches(self):
2789+ return self._branches()
2790+
2791+ @property
2792+ def branch_names(self):
2793+ return [b.branch_name for b in self.branches]
2794+
2795+ @property
2796+ def local_branches(self):
2797+ return self._branches(pygit2.GIT_BRANCH_LOCAL)
2798+
2799+ @property
2800+ def local_branch_names(self):
2801+ return [b.branch_name for b in self.local_branches]
2802+
2803+ @property
2804+ def remote_branches(self):
2805+ return self._branches(pygit2.GIT_BRANCH_REMOTE)
2806+
2807+ @property
2808+ def remote_branch_names(self):
2809+ return [b.branch_name for b in self.remote_branches]
2810+
2811+ @property
2812+ def lp_user(self):
2813+ if not self._lp_user:
2814+ raise RuntimeError("lp_user is not set")
2815+ return self._lp_user
2816+
2817+ def get_commitish(self, commitish):
2818+ return self.raw_repo.revparse_single(commitish)
2819+
2820+ def head_to_commit(self, head_name):
2821+ return str(self.get_head_by_name(head_name).peel().id)
2822+
2823+ def get_short_hash(self, hash):
2824+ """Return an unambiguous but abbreviated form of a commit hash
2825+
2826+ Note that the hash may still become ambiguous in the future.
2827+ """
2828+ stdout, _ = self.git_run(['rev-parse', '--short', hash])
2829+ return stdout.strip()
2830+
2831+ def git_run(self, args, env=None, **kwargs):
2832+ """Run the git CLI with the provided arguments
2833+
2834+ :param list(str) args: arguments to the git CLI
2835+ :param dict env: additional environment variables to use
2836+ :param dict **kwargs: further arguments to pass through to
2837+ gitubuntu.run.run()
2838+ :raises subprocess.CalledProcessError: if git exits non-zero
2839+ :rtype: (str, str)
2840+ :returns: stdout and stderr strings containing the subprocess output
2841+
2842+ The environment used is based on the Python process' environment at the
2843+ time this class instance was constructed.
2844+
2845+ The GIT_DIR and GIT_WORK_TREE environment variables are set
2846+ automatically based on the repository's existing location and settings.
2847+
2848+ If env is set, then the environment to be used is updated with env
2849+ before the call to git is made. This can override GIT_DIR,
2850+ GIT_WORK_TREE, and anything else.
2851+ """
2852+ return git_run(
2853+ pygit2_repo=self.raw_repo,
2854+ args=args,
2855+ initial_env=self._initial_env,
2856+ update_env=env,
2857+ **kwargs,
2858+ )
2859+
2860+ def garbage_collect(self):
2861+ self.git_run(['gc'])
2862+
2863+ def extract_file_from_treeish(self, treeish_string, filename):
2864+ """extract a file from @treeish to a local file
2865+
2866+ Arguments:
2867+ treeish - SHA1 of treeish
2868+ filename - file to extract from @treeish
2869+
2870+ Returns a NamedTemporaryFile that is flushed but not rewound.
2871+ """
2872+ blob = follow_symlinks_to_blob(
2873+ self.raw_repo,
2874+ treeish_object=self.raw_repo.revparse_single(treeish_string),
2875+ path=filename,
2876+ )
2877+ outfile = tempfile.NamedTemporaryFile()
2878+ outfile.write(blob.data)
2879+ outfile.flush()
2880+ return outfile
2881+
2882+ @lru_cache()
2883+ def get_changelog_from_treeish(self, treeish_string):
2884+ return Changelog.from_treeish(
2885+ self.raw_repo,
2886+ self.raw_repo.revparse_single(treeish_string),
2887+ )
2888+
2889+ def get_changelog_versions_from_treeish(self, treeish_string):
2890+ """Extract current and prior versions from debian/changelog in a
2891+ given @treeish_string
2892+
2893+ Returns (None, None) if the treeish supplied is None or if
2894+ 'debian/changelog' does not exist in the treeish.
2895+
2896+ Returns (current, previous) on success.
2897+ """
2898+ try:
2899+ changelog = self.get_changelog_from_treeish(treeish_string)
2900+ except KeyError:
2901+ # If 'debian/changelog' does
2902+ # not exist, then (None, None) is returned. KeyError propagates up
2903+ # from Changelog's __init__.
2904+ return None, None
2905+ try:
2906+ return changelog.version, changelog.previous_version
2907+ except CalledProcessError:
2908+ raise GitUbuntuChangelogError(
2909+ 'Cannot get changelog versions'
2910+ )
2911+
2912+ def get_changelog_distribution_from_treeish(self, treeish_string):
2913+ """Extract targetted distribution from debian/changelog in a
2914+ given treeish
2915+ """
2916+
2917+ if treeish_string is None:
2918+ return None
2919+
2920+ try:
2921+ return self.get_changelog_from_treeish(treeish_string).distribution
2922+ except (KeyError, CalledProcessError):
2923+ raise GitUbuntuChangelogError(
2924+ 'Cannot get changelog distribution'
2925+ )
2926+
2927+ def get_changelog_srcpkg_from_treeish(self, treeish_string):
2928+ """Extract srcpkg from debian/changelog in a given treeish
2929+ """
2930+
2931+ if treeish_string is None:
2932+ return None
2933+
2934+ try:
2935+ return self.get_changelog_from_treeish(treeish_string).srcpkg
2936+ except (KeyError, CalledProcessError):
2937+ raise GitUbuntuChangelogError(
2938+ 'Cannot get changelog source package name'
2939+ )
2940+
2941+ def get_head_info(self, head_prefix, namespace):
2942+ """Extract package versions at branch heads
2943+
2944+ Extract the version from debian/changelog of all
2945+ f'{namespace}/{head_prefix>/*' branches, excluding any branch that
2946+ contains 'ubuntu/devel'.
2947+
2948+ :param str namespace: the namespace under which git refs are found
2949+ :param str head_prefix: the prefix to look for
2950+ :rtype: dict(str, HeadInfoItem)
2951+ :returns: a dictionary keyed by the namespaced branch name (ie. without
2952+ a 'refs/heads/' prefix but with the namespace prefix, eg.
2953+ 'importer/ubuntu/focal-devel').
2954+ """
2955+ head_info = dict()
2956+ for head in self.local_branches:
2957+ prefix = '%s/%s' % (namespace, head_prefix)
2958+ if not head.branch_name.startswith(prefix):
2959+ continue
2960+ if 'ubuntu/devel' in head.branch_name:
2961+ continue
2962+ version, _ = (
2963+ self.get_changelog_versions_from_treeish(str(head.peel().id))
2964+ )
2965+ head_info[head.branch_name] = HeadInfoItem(
2966+ version=version,
2967+ commit_time=head.peel().commit_time,
2968+ commit_id=head.peel().id,
2969+ )
2970+
2971+ return head_info
2972+
2973+ def treeishs_identical(self, treeish_string1, treeish_string2):
2974+ if treeish_string1 is None or treeish_string2 is None:
2975+ return False
2976+ _tree_obj1 = self.raw_repo.revparse_single(treeish_string1)
2977+ _tree_id1 = _tree_obj1.peel(pygit2.Tree).id
2978+ _tree_obj2 = self.raw_repo.revparse_single(treeish_string2)
2979+ _tree_id2 = _tree_obj2.peel(pygit2.Tree).id
2980+ return _tree_id1 == _tree_id2
2981+
2982+ def get_head_by_name(self, name):
2983+ try:
2984+ return self.raw_repo.lookup_branch(name)
2985+ except TypeError:
2986+ return None
2987+
2988+ def get_tag_reference(self, tag):
2989+ """Return the tag object if it exists in the repository"""
2990+ try:
2991+ return self.raw_repo.lookup_reference('refs/tags/%s' % tag)
2992+ except (KeyError, ValueError):
2993+ return None
2994+
2995+ def get_import_tag(
2996+ self,
2997+ version,
2998+ namespace,
2999+ patch_state=PatchState.UNAPPLIED,
3000+ ):
3001+ """
3002+ Return the import tag matching the given specification.
3003+
3004+ :param str version: the package version string to match
3005+ :param str namespace: the namespace under which git refs are found
3006+ :param PatchState patch_state: whether to look for unapplied or applied
3007+ tags
3008+ :returns: the matching import tag, or None if there is no match
3009+ :rtype: pygit2.Reference or None
3010+ """
3011+ return self.get_tag_reference(
3012+ import_tag(version, namespace, patch_state)
3013+ )
3014+
3015+ def get_reimport_tag(
3016+ self,
3017+ version,
3018+ namespace,
3019+ reimport,
3020+ patch_state=PatchState.UNAPPLIED,
3021+ ):
3022+ """
3023+ Return the reimport tag matching the given specification.
3024+
3025+ :param str version: the package version string to match
3026+ :param str namespace: the namespace under which git refs are found
3027+ :param int reimport: the sequence number of the reimport tag
3028+ :param PatchState patch_state: whether to look for unapplied or applied
3029+ tags
3030+ :returns: the matching reimport tag, or None if there is no match
3031+ :rtype: pygit2.Reference or None
3032+ """
3033+ return self.get_tag_reference(
3034+ reimport_tag(version, namespace, reimport, patch_state)
3035+ )
3036+
3037+ def get_all_reimport_tags(
3038+ self,
3039+ version,
3040+ namespace,
3041+ patch_state=PatchState.UNAPPLIED,
3042+ ):
3043+ """
3044+ Return all reimport tags matching the given specification.
3045+
3046+ :param str version: the package version string to match
3047+ :param str namespace: the namespace under which git refs are found
3048+ :param PatchState patch_state: whether to look for unapplied or applied
3049+ tags
3050+ :returns: matching reimport tags
3051+ :rtype: sequence(pygit2.Reference)
3052+ """
3053+ return self.references_with_prefix(
3054+ 'refs/tags/%s/' % reimport_tag_prefix(
3055+ version,
3056+ namespace,
3057+ patch_state,
3058+ )
3059+ )
3060+
3061+ def get_upload_tag(self, version, namespace):
3062+ """
3063+ Return the upload tag matching the given specification.
3064+
3065+ :param str version: the package version string to match
3066+ :param str namespace: the namespace under which git refs are found
3067+ :returns: the matching upload tag, or None if there is no match
3068+ :rtype: pygit2.Reference or None
3069+ """
3070+ return self.get_tag_reference(upload_tag(version, namespace))
3071+
3072+ def get_upstream_tag(self, version, namespace):
3073+ """
3074+ Return the upstream tag matching the given specification.
3075+
3076+ :param str version: the package version string to match
3077+ :param str namespace: the namespace under which git refs are found
3078+ :returns: the matching upstream tag, or None if there is no match
3079+ :rtype: pygit2.Reference or None
3080+ """
3081+ return self.get_tag_reference(upstream_tag(version, namespace))
3082+
3083+ def get_orphan_tag(self, version, namespace):
3084+ """
3085+ Return the orphan tag matching the given specification.
3086+
3087+ :param str version: the package version string to match
3088+ :param str namespace: the namespace under which git refs are found
3089+ :returns: the matching orphan tag, or None if there is no match
3090+ :rtype: pygit2.Reference or None
3091+ """
3092+ return self.get_tag_reference(orphan_tag(version, namespace))
3093+
3094+ def create_tag(self,
3095+ commit_hash,
3096+ tag_name,
3097+ tag_msg,
3098+ tagger=None,
3099+ ):
3100+ """Create a tag in the repository
3101+
3102+ :param str commit_hash: the commit hash the tag will point to.
3103+ :param str tag_name: the name of the tag to be created.
3104+ :param str tag_msg: the text of the tag annotation.
3105+ :param pygit2.Signature tagger: if supplied, use this signature in the
3106+ created tag's "tagger" metadata. If not supplied, an arbitrary name
3107+ and email address is used with the current time.
3108+ :returns: None
3109+ """
3110+ if not tagger:
3111+ tagger_time, tagger_offset = datetime_to_signature_spec(
3112+ datetime.datetime.now(),
3113+ )
3114+ tagger = pygit2.Signature(
3115+ gitubuntu.spec.SYNTHESIZED_COMMITTER_NAME,
3116+ gitubuntu.spec.SYNTHESIZED_COMMITTER_EMAIL,
3117+ tagger_time,
3118+ tagger_offset,
3119+ )
3120+
3121+ logging.debug("Creating tag %s pointing to %s", tag_name, commit_hash)
3122+ self.raw_repo.create_tag(
3123+ tag_name,
3124+ pygit2.Oid(hex=commit_hash),
3125+ pygit2.GIT_OBJ_COMMIT,
3126+ tagger,
3127+ tag_msg,
3128+ )
3129+
3130+ def nearest_remote_branches(self, commit_hash, prefix=None,
3131+ max_commits=100
3132+ ):
3133+ '''Return the set of remote branches nearest to @commit_hash
3134+
3135+ This is a set of remote branch objects that are currently
3136+ pointing at a commit, where that commit is the nearest ancestor
3137+ to @commit_hash among the possible commits.
3138+
3139+ If no such commit is found, an empty set is returned.
3140+
3141+ Only consider remote branches that start with @prefix.
3142+
3143+ Stop searching beyond the @max_commits'-th ancestor. Usually this method
3144+ is only used as a heuristic that generally will never need to go too far
3145+ back in history, and this avoids searching all the way back to the root
3146+ commit, which may be a long way.
3147+ '''
3148+
3149+ # 1) cache all prefixed branch names by commit
3150+ remote_heads_by_commit = collections.defaultdict(set)
3151+ for b in self.remote_branches:
3152+ if prefix is None or b.branch_name.startswith(prefix):
3153+ remote_heads_by_commit[b.peel().id].add(b)
3154+
3155+ # 2) walk from commit_hash backwards until a cached commit is found
3156+ commits = self.raw_repo.walk(
3157+ self.get_commitish(commit_hash).id,
3158+ pygit2.GIT_SORT_TOPOLOGICAL,
3159+ )
3160+ for commit in itertools.islice(commits, max_commits):
3161+ if commit.id not in remote_heads_by_commit:
3162+ continue # avoid creating a bunch of empty sets
3163+
3164+ if remote_heads_by_commit[commit.id]:
3165+ return remote_heads_by_commit[commit.id]
3166+
3167+ # in the currently impossible (but permitted in this state) case
3168+ # that the dictionary returned an empty set, we loop around again
3169+ # which is what we want.
3170+
3171+ return set()
3172+
3173+
3174+ def nearest_tag(
3175+ self,
3176+ commitish_string,
3177+ prefix,
3178+ max_commits=100,
3179+ ):
3180+ # 1) cache all patterned tag names by commit
3181+ pattern_tags_by_commit = collections.defaultdict(set)
3182+ for t in self.tags:
3183+ if t.name.startswith('refs/tags/' + prefix):
3184+ pattern_tags_by_commit[t.peel(pygit2.Commit).id].add(t)
3185+
3186+ commits = self.raw_repo.walk(
3187+ self.get_commitish(commitish_string).id,
3188+ pygit2.GIT_SORT_TOPOLOGICAL,
3189+ )
3190+ for commit in itertools.islice(commits, max_commits):
3191+ if commit.id not in pattern_tags_by_commit:
3192+ continue
3193+
3194+ return pattern_tags_by_commit[commit.id].pop()
3195+
3196+ return None
3197+
3198+ @staticmethod
3199+ def tag_to_pretty_name(tag):
3200+ _, _, pretty_name = tag.name.partition('refs/tags/')
3201+ return pretty_name
3202+
3203+ def create_tracking_branch(self, branch_name, upstream_name, force=False):
3204+ return self.raw_repo.create_branch(
3205+ branch_name,
3206+ self.raw_repo.lookup_branch(
3207+ upstream_name,
3208+ pygit2.GIT_BRANCH_REMOTE
3209+ ).peel(pygit2.Commit),
3210+ force
3211+ )
3212+
3213+ def checkout_commitish(self, commitish):
3214+ # pygit2 checkout does not accept hashes
3215+ # https://github.com/libgit2/pygit2/issues/412
3216+ # self.raw_repo.checkout_tree(self.get_commitish(commitish))
3217+ self.git_run(['checkout', commitish])
3218+
3219+ def reset_commitish(self, commitish):
3220+ # pygit2 checkout does not accept hashes
3221+ # https://github.com/libgit2/pygit2/issues/412
3222+ # self.checkout_tree(self.get_commitish(commitish))
3223+ self.git_run(['reset', '--hard', commitish])
3224+
3225+ def update_head_to_commit(self, head_name, commit_hash):
3226+ try:
3227+ self.raw_repo.lookup_branch(head_name).set_target(commit_hash)
3228+ except AttributeError:
3229+ self.raw_repo.create_branch(head_name,
3230+ self.raw_repo.get(commit_hash)
3231+ )
3232+
3233+ def clean_repository_state(self):
3234+ """Cleanup working tree"""
3235+ runq(['git', 'checkout', '--orphan', 'master'],
3236+ check=False, env=self.env)
3237+ runq(['git', 'reset', '--hard'], env=self.env)
3238+ runq(['git', 'clean', '-f', '-d'], env=self.env)
3239+
3240+ def get_all_changelog_versions_from_treeish(self, treeish):
3241+ changelog = self.get_changelog_from_treeish(treeish)
3242+ return changelog.all_versions
3243+
3244+ def annotated_tag(self, tag_name, commitish, force, msg=None):
3245+ try:
3246+ args = ['tag', '-a', tag_name, commitish]
3247+ if force:
3248+ args += ['-f']
3249+ if msg is not None:
3250+ args += ['-m', msg]
3251+ self.git_run(args, stdin=None, stdout=None, stderr=None)
3252+ version, _ = self.get_changelog_versions_from_treeish(commitish)
3253+ logging.info('Created annotated tag %s for version %s' % (tag_name, version))
3254+ except:
3255+ logging.error('Unable to tag %s. Does it already exist (pass -f)?' %
3256+ tag_name
3257+ )
3258+ raise
3259+
3260+ def tag(self, tag_name, commitish, force):
3261+ try:
3262+ args = ['tag', tag_name, commitish]
3263+ if force:
3264+ args += ['-f']
3265+ self.git_run(args)
3266+ version, _ = self.get_changelog_versions_from_treeish(commitish)
3267+ logging.info('Created tag %s for version %s' % (tag_name, version))
3268+ except:
3269+ logging.error('Unable to tag %s. Does it already exist (pass -f)?' %
3270+ tag_name
3271+ )
3272+ raise
3273+
3274+ def commit_source_tree(
3275+ self,
3276+ tree,
3277+ parents,
3278+ log_message,
3279+ commit_date=None,
3280+ author_date=None,
3281+ ):
3282+ """Commit a git tree with appropriate parents and message
3283+
3284+ Given a git tree that contains a source package, create a matching
3285+ commit using metadata derived from the tree as required according to
3286+ the import specification.
3287+
3288+ Commit metadata elements that are not specified as derived from the
3289+ tree itself are required as parameters.
3290+
3291+ :param pygit2.Oid tree: reference to the git tree in this repository
3292+ that contains a debian/changelog file
3293+ :param list(pygit2.Oid) parents: parent commits of the commit to be
3294+ created
3295+ :param bytes log_message: commit message
3296+ :param datetime.datetime commit_date: the commit date to use (any
3297+ sub-second part of the timestamp is truncated). If None, use the
3298+ current date.
3299+ :param datetime.datetime author_date: overrides the author date
3300+ normally parsed from the changelog entry (i.e. for handling date
3301+ parsing edge cases). Any sub-second part of the timestamp is
3302+ truncated.
3303+ :returns: reference to the created commit
3304+ :rtype: pygit2.Oid
3305+ """
3306+ if commit_date is None:
3307+ commit_date = datetime.datetime.now()
3308+
3309+ commit_time, commit_offset = datetime_to_signature_spec(commit_date)
3310+ changelog = self.get_changelog_from_treeish(str(tree))
3311+
3312+ return self.raw_repo.create_commit(
3313+ None, # ref: do not update any ref
3314+ pygit2.Signature(*changelog.git_authorship(author_date)), # author
3315+ pygit2.Signature( # committer
3316+ name=gitubuntu.spec.SYNTHESIZED_COMMITTER_NAME,
3317+ email=gitubuntu.spec.SYNTHESIZED_COMMITTER_EMAIL,
3318+ time=commit_time,
3319+ offset=commit_offset,
3320+ ),
3321+ log_message, # message
3322+ tree, # tree
3323+ parents, # parents
3324+ )
3325+
3326+
3327+ @classmethod
3328+ def _create_replacement_tree_builder(cls, repo, treeish, sub_path):
3329+ '''Create a replacement TreeBuilder
3330+
3331+ Create a TreeBuilder based on an existing repository, top-level
3332+ tree-ish and path inside that tree.
3333+
3334+ A sub_path of '' is taken to mean a request for a replacement
3335+ TreeBuilder for the top level tree.
3336+
3337+ Returns a TreeBuilder object pre-populated with the previous contents.
3338+ If the path did not previously exist in the tree-ish, then return an
3339+ empty TreeBuilder instead.
3340+ '''
3341+
3342+ tree = treeish.peel(pygit2.GIT_OBJ_TREE)
3343+
3344+ # Short path: sub_path == '' means want root
3345+ if not sub_path:
3346+ return repo.TreeBuilder(tree)
3347+
3348+ try:
3349+ tree_entry = tree[sub_path]
3350+ except KeyError:
3351+ # sub_path does not exist in tree, so return an empty TreeBuilder
3352+ tree_builder = repo.TreeBuilder()
3353+ else:
3354+ # The tree entry must itself be a tree
3355+ assert tree_entry.filemode == pygit2.GIT_FILEMODE_TREE
3356+ sub_tree = repo.get(tree_entry.id).peel(pygit2.GIT_OBJ_TREE)
3357+ tree_builder = repo.TreeBuilder(sub_tree)
3358+
3359+ return tree_builder
3360+
3361+ @classmethod
3362+ def _add_missing_tree_dirs(cls, repo, top_path, top_tree_object, _sub_path=''):
3363+ """
3364+ Recursively add empty directories to a tree object
3365+
3366+ Find empty directories under top_path and make sure that empty tree
3367+ objects exist for them. If this means that the tree object must change,
3368+ then a replacement tree object is created accordingly.
3369+
3370+ repo: pygit2.Repository object
3371+ top_path: path to the extracted contents of the tree
3372+ top_tree_object: tree object
3373+ _sub_path (internal): relative path for where we are for recursive call
3374+
3375+ Returns None if oid unchanged, or oid if it changed.
3376+ """
3377+
3378+ # full path to our _sub_path, including top_path
3379+ full_path = os.path.join(top_path, _sub_path)
3380+
3381+ dir_list = os.listdir(full_path)
3382+ if not dir_list:
3383+ # directory is empty, so this is always the empty tree object
3384+ return repo.TreeBuilder().write()
3385+
3386+ # tree_builder is None if we don't need one yet, or is the replacement
3387+ # for the tree object for this recursive call
3388+ tree_builder = None
3389+ for entry in dir_list:
3390+ entry_path = os.path.join(full_path, entry)
3391+ # We cannot use os.path.isdir() here as we don't want to recurse
3392+ # down symlinks to directories.
3393+ if stat.S_ISDIR(os.lstat(entry_path).st_mode):
3394+ # this is a directory, so recurse down
3395+ entry_oid = cls._add_missing_tree_dirs(
3396+ repo=repo,
3397+ top_path=top_path,
3398+ top_tree_object=top_tree_object,
3399+ _sub_path=os.path.join(_sub_path, entry),
3400+ )
3401+ if entry_oid:
3402+ # The recursive call reported a change to the tree, so we
3403+ # must adopt it in what we return to propogate the change
3404+ # upwards.
3405+ if tree_builder is None:
3406+ # There is no replacement in progress for this
3407+ # recursive call's tree object, so start one.
3408+ tree_builder = cls._create_replacement_tree_builder(
3409+ repo=repo,
3410+ treeish=top_tree_object,
3411+ sub_path=_sub_path,
3412+ )
3413+ # If the entry previous existed, remove it.
3414+ if tree_builder.get(entry):
3415+ tree_builder.remove(entry)
3416+ # Add the replacement tree entry
3417+ tree_builder.insert( # (takes no kwargs)
3418+ entry, # name
3419+ entry_oid, # oid
3420+ pygit2.GIT_FILEMODE_TREE, # attr
3421+ )
3422+
3423+ if tree_builder is None:
3424+ return None # no changes
3425+ else:
3426+ return tree_builder.write() # create replacement tree object
3427+
3428+ @classmethod
3429+ def dir_to_tree(cls, pygit2_repo, path, escape=False):
3430+ """Create a git tree object from the given filesystem path
3431+
3432+ :param pygit2.Repository pygit2_repo: the repository on which to
3433+ operate. If you have a GitUbuntuRepository instance, you can use
3434+ its raw_repo property.
3435+ :param path: path to filesystem directory to be the root of the tree
3436+ :param escape: if True, escape using escape_dot_git() first. This
3437+ mutates the provided filesystem tree.
3438+
3439+ escape should be used when the directory being moved into git is
3440+ directly from a source package, since the source package may contain
3441+ files or directories named '.git' and these cannot otherwise be
3442+ represented in a git tree object.
3443+
3444+ escape should not be used if the directory has already been escaped
3445+ previously. For example: if escape was previously used to move into a
3446+ git tree object, and that git tree object has been extracted to a
3447+ working directory for manipulation without unescaping, then escape
3448+ should not be used again to move that result back into a git tree
3449+ object.
3450+ """
3451+ if escape:
3452+ escape_dot_git(path)
3453+ # git expects the index file to not exist (in order to create a fresh
3454+ # one), so create a temporary directory to put it in so we have a name
3455+ # we can use safely.
3456+ with tempfile.TemporaryDirectory() as index_dir:
3457+ index_path = os.path.join(index_dir, 'index')
3458+ def indexed_git_run(*args):
3459+ return git_run(
3460+ pygit2_repo=pygit2_repo,
3461+ args=args,
3462+ work_tree_path=path,
3463+ index_path=index_path,
3464+ )
3465+ indexed_git_run('add', '-f', '-A')
3466+ indexed_git_run('reset', 'HEAD', '--', '.git')
3467+ indexed_git_run('reset', 'HEAD', '--', '.pc')
3468+ tree_hash_str, _ = indexed_git_run('write-tree')
3469+ tree_hash_str = tree_hash_str.strip()
3470+ tree = pygit2_repo.get(tree_hash_str)
3471+
3472+ # Add any empty directories that git did not import. Workaround for LP:
3473+ # #1687057.
3474+ replacement_oid = cls._add_missing_tree_dirs(
3475+ repo=pygit2_repo,
3476+ top_path=path,
3477+ top_tree_object=tree,
3478+ )
3479+ if replacement_oid:
3480+ # Empty directories had to be added
3481+ return str(replacement_oid) # return the replacement instead
3482+ else:
3483+ # No empty directories were added
3484+ return tree_hash_str # no replacement was needed
3485+
3486+ @contextmanager
3487+ def temporary_worktree(self, commitish, prefix=None):
3488+ with tempfile.TemporaryDirectory(prefix=prefix) as tempdir:
3489+ self.git_run(
3490+ [
3491+ 'worktree',
3492+ 'add',
3493+ '--detach',
3494+ '--force',
3495+ tempdir,
3496+ commitish,
3497+ ]
3498+ )
3499+
3500+ oldcwd = os.getcwd()
3501+ os.chdir(tempdir)
3502+
3503+ try:
3504+ yield
3505+ except:
3506+ raise
3507+ finally:
3508+ os.chdir(oldcwd)
3509+
3510+ self.git_run(['worktree', 'prune'])
3511+
3512+ def tree_hash_after_command(self, commitish, cmd):
3513+ with self.temporary_worktree(commitish):
3514+ try:
3515+ run(cmd)
3516+ except CalledProcessError as e:
3517+ logging.error("Unable to execute `%s`", ' '.join(cmd))
3518+ raise
3519+
3520+ run(["git", "add", "-f", ".",])
3521+ tree_hash, _ = run(["git", "write-tree"])
3522+ return tree_hash.strip()
3523+
3524+ def tree_hash_subpath(self, treeish_string, path):
3525+ """Get the tree hash for path at a given treeish
3526+
3527+ Arguments:
3528+ @treeish_string: a string Git treeish
3529+ @path: a string path present in @treeish_string
3530+
3531+ Returns:
3532+ String hash of Git tree corresponding to @path in @treeish_string
3533+ """
3534+ tree_obj = self.raw_repo.revparse_single(treeish_string).peel(
3535+ pygit2.Tree
3536+ )
3537+ return str(tree_obj[path].id)
3538+
3539+ def paths_are_identical(self, treeish1_string, treeish2_string, path):
3540+ """Determine if a given path is the same in two treeishs
3541+
3542+ Arguments:
3543+ @treeish1_string: a string Git treeish
3544+ @treeish2_string: a string Git treeish
3545+ @path: a string path present in @treeish1_string and @treeish2_string
3546+
3547+ Returns:
3548+ True, if @path is the same in @treeish1_string and @treeish2_string
3549+ False, otherwise
3550+ """
3551+ try:
3552+ subpath_tree_hash1 = self.tree_hash_subpath(
3553+ treeish1_string,
3554+ path,
3555+ )
3556+ except KeyError:
3557+ # if the path does not exist in treeish
3558+ subpath_tree_hash1 = None
3559+ try:
3560+ subpath_tree_hash2 = self.tree_hash_subpath(
3561+ treeish2_string,
3562+ path,
3563+ )
3564+ except KeyError:
3565+ subpath_tree_hash2 = None
3566+
3567+ return subpath_tree_hash1 == subpath_tree_hash2
3568+
3569+ @lru_cache()
3570+ def quilt_env(self, treeish):
3571+ """Return a suitable environment for running quilt.
3572+
3573+ This varies depending on the supplied commit since both
3574+ debian/patches/series and debian/patches/debian.series may be valid.
3575+ See dpkg-source(1) for details.
3576+
3577+ The returned environment includes all necessary variables by
3578+ combining self.env with the needed quilt-specific environment.
3579+
3580+ :param pygit.Object treeish: object that peels to the pygit2.Tree on
3581+ which quilt will operate.
3582+ :rtype: dict
3583+ :returns: an environment suitable for running quilt.
3584+ """
3585+ env = self.env.copy()
3586+ env.update(quilt_env(self.raw_repo, treeish))
3587+ return env
3588+
3589+ def quilt_env_from_treeish_str(self, treeish_str):
3590+ """Return a suitable environment for running quilt.
3591+
3592+ This is a thin wrapper around quilt_env() that works with a treeish hex
3593+ string instead of directly with a treeish object.
3594+
3595+ :param str treeish_str: the hash of the tree on which quilt will
3596+ operate, in hex.
3597+ :rtype: dict
3598+ :returns: an environment suitable for running quilt.
3599+ """
3600+ return self.quilt_env(self.raw_repo.get(treeish_str))
3601+
3602+ def is_patches_applied(self, commit_hash, regenerated_pc_path):
3603+ # first see if quilt push -a would do anything to
3604+ # differentiate between applied and unapplied
3605+ with self.temporary_worktree(commit_hash):
3606+ try:
3607+ run_quilt(
3608+ ['push', '-a'],
3609+ env=self.quilt_env_from_treeish_str(commit_hash),
3610+ )
3611+ # False if in an unapplied state, which is signified by
3612+ # successful push (rc=0)
3613+ return False
3614+ except CalledProcessError as e:
3615+ # non-zero return might be an error or it might mean no
3616+ # patches exist
3617+ if e.returncode == 1:
3618+ # an error may occur if we need to recreate the .pc
3619+ # first
3620+ try:
3621+ # the first quilt push may have created a .pc/
3622+ shutil.rmtree('.pc')
3623+ shutil.copytree(
3624+ regenerated_pc_path,
3625+ '.pc',
3626+ )
3627+ except FileNotFoundError:
3628+ # if there was no .pc directory, then the first
3629+ # quilt push failure was a real error
3630+ raise e
3631+
3632+ try:
3633+ run_quilt(
3634+ ['push', '-a'],
3635+ env=self.quilt_env_from_treeish_str(commit_hash),
3636+ )
3637+ # False if in an unapplied state
3638+ return False
3639+ except CalledProcessError as e:
3640+ # True if in a patches-applied state or
3641+ # there are no patches to apply
3642+ if e.returncode == 2:
3643+ return True
3644+ else:
3645+ raise
3646+ # True if in a patches-applied state or there are
3647+ # no patches to apply
3648+ elif e.returncode == 2:
3649+ return True
3650+ else:
3651+ raise
3652+
3653+ def _maybe_quiltify_tree_hash(self, commit_hash):
3654+ """Determine if quiltify is needed and yield the quiltify'd tree hash
3655+
3656+ The imported patches-applied trees do not contain .pc
3657+ directories. To determine if an additional quilt patch is
3658+ necessary, we have to first regenerate the .pc directory, then
3659+ see if dpkg-source --commit generates a new quilt patch.
3660+
3661+ In order for dpkg-source --commit to function, we need to know
3662+ if the commit we are building is patches-unapplied or
3663+ patches-applied. In the latter case, we can build the commit
3664+ directly after copying the regenerated .pc directory. In the
3665+ former case, we do not want to copy the regenerated .pc
3666+ directory, as dpkg-source will do this for us, as it applies the
3667+ current patches. We determine if patches are applied or
3668+ unapplied by relying on `quilt push -a`'s exit status at
3669+ @commit_hash.
3670+
3671+ This is a common method used by multiple callers.
3672+
3673+ Arguments:
3674+ @commit_hash: a string Git commit hash
3675+
3676+ Returns:
3677+ String tree hash of quiltify'ing @commit_hash.
3678+ If no quiltify is needed, the return value is @commit_hash's
3679+ tree hash
3680+ """
3681+ commit_tree_hash = str(
3682+ self.raw_repo.get(commit_hash).peel(pygit2.Tree).id
3683+ )
3684+ if not is_3_0_quilt(self, commit_hash):
3685+ return commit_tree_hash
3686+ # the tarballs need to be in the parent directory from where
3687+ # we need the orig tarballs for quilt and dpkg-source
3688+ # but suppress any logging
3689+ logger = logging.getLogger()
3690+ oldLevel = logger.getEffectiveLevel()
3691+ logger.setLevel(logging.WARNING)
3692+ tarballs = gitubuntu.build.fetch_orig(
3693+ orig_search_list=gitubuntu.build.derive_orig_search_list_from_args(
3694+ self,
3695+ commitish=commit_hash,
3696+ for_merge=False,
3697+ no_pristine_tar=False,
3698+ ),
3699+ changelog=Changelog.from_treeish(
3700+ self.raw_repo,
3701+ self.raw_repo.get(commit_hash)
3702+ ),
3703+ )
3704+ logger.setLevel(oldLevel)
3705+ # run dpkg-source
3706+ with tempfile.TemporaryDirectory() as tempdir:
3707+ # copy the generated tarballs
3708+ new_tarballs = []
3709+ for tarball in tarballs:
3710+ new_tarballs.append(shutil.copy(tarball, tempdir))
3711+ tarballs = new_tarballs
3712+
3713+ # create a nested temporary directory where we will recreate
3714+ # the .pc directory
3715+ with tempfile.TemporaryDirectory(prefix=tempdir+'/') as ttempdir:
3716+ oldcwd = os.getcwd()
3717+ os.chdir(ttempdir)
3718+
3719+ for tarball in tarballs:
3720+ run(['tar', '-x', '--strip-components=1', '-f', tarball,])
3721+
3722+ # need the debia/patches
3723+ shutil.copytree(
3724+ os.path.join(self.local_dir, 'debian',),
3725+ 'debian',
3726+ )
3727+
3728+ # generate the equivalent .pc directory
3729+ run_quilt(
3730+ ['push', '-a'],
3731+ env=self.quilt_env_from_treeish_str(commit_hash),
3732+ rcs=[2],
3733+ )
3734+
3735+ regenerated_pc_path = os.path.join(tempdir, '.pc')
3736+
3737+ if os.path.exists(".pc"):
3738+ shutil.copytree(
3739+ '.pc',
3740+ regenerated_pc_path,
3741+ )
3742+
3743+ os.chdir(oldcwd)
3744+
3745+ patches_applied = self.is_patches_applied(
3746+ commit_hash,
3747+ regenerated_pc_path,
3748+ )
3749+
3750+ with self.temporary_worktree(commit_hash, prefix=tempdir+'/'):
3751+ # we only need to copy the generated .pc directory
3752+ # if we are building a patches-applied tree, which
3753+ # we determine by comparing our current tree hash to
3754+ # the generated tree hash.
3755+ if patches_applied:
3756+ try:
3757+ shutil.copytree(
3758+ regenerated_pc_path,
3759+ '.pc',
3760+ )
3761+ except FileNotFoundError:
3762+ # it is possible no quilt patches exist yet
3763+ pass
3764+
3765+ fixup_patch_path = os.path.join(
3766+ 'debian',
3767+ 'patches',
3768+ 'git-ubuntu-fixup.patch'
3769+ )
3770+
3771+ if os.path.exists(fixup_patch_path):
3772+ raise ValueError(
3773+ "A quilt patch with the name git-ubuntu-fixup.patch "
3774+ "already exists in %s" % commit_hash
3775+ )
3776+
3777+ run(
3778+ [
3779+ 'dpkg-source',
3780+ '--commit',
3781+ '.',
3782+ 'git-ubuntu-fixup.patch',
3783+ ],
3784+ env=self.quilt_env_from_treeish_str(commit_hash),
3785+ )
3786+
3787+ # do not want the .pc directory in the resulting
3788+ # treeish
3789+ if os.path.exists('.pc'):
3790+ shutil.rmtree('.pc')
3791+
3792+ if os.path.exists(fixup_patch_path):
3793+ # dpkg-source uses debian/changelog to generate some
3794+ # fields. We do not know yet if the changelog has
3795+ # been updated, so elide that section of comments.
3796+ with open(fixup_patch_path, 'r+') as f:
3797+ for line in f:
3798+ if '---' in line:
3799+ break
3800+ text = """Description: git-ubuntu generated quilt fixup patch
3801+TODO: Put a short summary on the line above and replace this paragraph
3802+with a longer explanation of this change. Complete the meta-information
3803+with other relevant fields (see below for details).
3804+---\n"""
3805+ for line in f:
3806+ text += line
3807+ f.seek(0)
3808+ f.write(text)
3809+ f.truncate()
3810+
3811+ # If we are on a patches-unapplied tree, then we
3812+ # need to reset ourselves back to @commit_hash with
3813+ # our new patch.
3814+ # In order for this to be buildable, we have to
3815+ # reverse-apply our patch, to undo the git-commited
3816+ # upstream changes.
3817+ if not patches_applied:
3818+ run(['git', 'add', '-f', 'debian/patches',])
3819+ # if any patches add files that are untracked,
3820+ # remove them
3821+ run(['git', 'clean', '-f', '-d',])
3822+ # reset all the other files to their status in
3823+ # HEAD
3824+ run(['git', 'checkout', commit_hash, '--', '*',])
3825+ with open(fixup_patch_path, 'rb') as f:
3826+ run(['patch', '-Rp1',], input=f.read())
3827+
3828+ return self.dir_to_tree(self.raw_repo, '.')
3829+ else:
3830+ return commit_tree_hash
3831+
3832+ def maybe_quiltify_tree_hash(self, commitish_string):
3833+ """Determine if quiltify is needed and return the quiltify'd tree hash
3834+
3835+ See _maybe_quiltify_tree_hash for details.
3836+
3837+ Arguments:
3838+ @commitish_string: a string Git commitish
3839+
3840+ Returns:
3841+ String tree hash of quiltify'ing @commitish_string.
3842+ If no quiltify is needed, the return value is the tree hash of
3843+ @commitish_string.
3844+ """
3845+ commit_hash = str(
3846+ self.get_commitish(commitish_string).peel(pygit2.Commit).id
3847+ )
3848+ return self._maybe_quiltify_tree_hash(commit_hash)
3849+
3850+ def maybe_changelogify_tree_hash(self, commit_hash):
3851+ """Determine if changelogify is needed and yield the changelogify'd tree hash
3852+
3853+ Given a commit, we need to detect if the user has inserted a
3854+ changelog entry relative to a published version for the purpose
3855+ of test builds.
3856+
3857+ Arguments:
3858+ @commit_hash: a string Git commit hash
3859+
3860+ Returns:
3861+ String tree hash of changelogify'ing @commit_hash.
3862+ If no changelogify is needed, the return value is the tree hash of
3863+ @commit_hash.
3864+ """
3865+ commit_tree_hash = str(
3866+ self.raw_repo.get(commit_hash).peel(pygit2.Tree).id
3867+ )
3868+
3869+ # one of these are the "base" pkg that @commit_hash's changes
3870+ # are based on
3871+ remote_tag = self.nearest_tag(
3872+ commit_hash,
3873+ prefix='pkg/',
3874+ )
3875+ remote_branch = derive_target_branch(
3876+ self,
3877+ commit_hash,
3878+ )
3879+
3880+ assert remote_tag or remote_branch
3881+
3882+ if remote_tag:
3883+ if remote_branch:
3884+ try:
3885+ self.git_run(
3886+ [
3887+ 'merge-base',
3888+ '--is-ancestor',
3889+ remote_tag.name,
3890+ remote_branch,
3891+ ],
3892+ verbose_on_failure=False,
3893+ )
3894+ parent_ref = remote_branch
3895+ except CalledProcessError as e:
3896+ if e.returncode == 1:
3897+ parent_ref = remote_tag.name
3898+ else:
3899+ raise
3900+ else:
3901+ parent_ref = remote_tag.name
3902+ else:
3903+ parent_ref = remote_branch
3904+
3905+ # If there are any changes relative to parent_ref but there are
3906+ # not any changelog changes, insert a snapshot changelog entry,
3907+ # starting from parent_ref, and return the resulting tree hash.
3908+ if str(self.raw_repo.revparse_single(parent_ref).peel(
3909+ pygit2.Tree
3910+ ).id) != commit_tree_hash and self.paths_are_identical(
3911+ parent_ref,
3912+ commit_hash,
3913+ 'debian/changelog',
3914+ ):
3915+ with self.temporary_worktree(commit_hash):
3916+ run_gbp(
3917+ [
3918+ 'dch',
3919+ '--snapshot',
3920+ '--ignore-branch',
3921+ '--since=%s' % str(parent_ref),
3922+ ],
3923+ env=self.env,
3924+ )
3925+ return self.dir_to_tree(self.raw_repo, '.')
3926+
3927+ # otherwise, return @commit_hash's tree hash
3928+ return commit_tree_hash
3929+
3930+ def quiltify_and_changelogify_tree_hash(self, commitish_string):
3931+ """Given a commitish, possibly quiltify and changelogify its tree
3932+
3933+ Definitions:
3934+ quiltify: generate a quilt patch from untracked upstream
3935+ changes
3936+ changelogify: generate a snapshot changelog entry if any
3937+ changes exist, and no new changelog entry yet exists
3938+
3939+ Arguments:
3940+ @commitish_string: string Git commitish
3941+
3942+ Returns:
3943+ string Git tree hash of quiltify-ing and changelogify-ing
3944+ @commitish_string, if needed
3945+ if neither quiltify or changelogify are needed, return
3946+ @commitish_string's tree hash
3947+ """
3948+ commit_hash = str(
3949+ self.get_commitish(commitish_string).peel(pygit2.Commit).id
3950+ )
3951+ quiltify_tree_hash = self._maybe_quiltify_tree_hash(commit_hash)
3952+ changelogify_tree_hash = self.maybe_changelogify_tree_hash(commit_hash)
3953+
3954+ quiltify_tree_obj = self.raw_repo.get(quiltify_tree_hash)
3955+ changelogify_tree_obj = self.raw_repo.get(changelogify_tree_hash)
3956+
3957+ # There are multiple ways to solve this problem, but the
3958+ # simplest is to use a TreeBuilder to merge the quiltify tree
3959+ # with the changelog from the changelogify tree
3960+ # top-level TreeBuilder
3961+ tb = self.raw_repo.TreeBuilder(quiltify_tree_obj)
3962+ te = tb.get('debian')
3963+ # TreeBuilder for debian/
3964+ dtb = self.raw_repo.TreeBuilder(self.raw_repo.get(te.id))
3965+ dtb.insert( # does not take kwargs
3966+ 'changelog', # name
3967+ changelogify_tree_obj['debian/changelog'].oid, # oid
3968+ pygit2.GIT_FILEMODE_BLOB, # attr
3969+ )
3970+ # insert can replace
3971+ tb.insert( # does not take kwargs
3972+ 'debian', # name
3973+ dtb.write(), # oid
3974+ pygit2.GIT_FILEMODE_TREE, # attr
3975+ )
3976+ return str(tb.write())
3977+
3978+ def find_ubuntu_merge_base(
3979+ self,
3980+ ubuntu_commitish,
3981+ ):
3982+ """Find the Ubuntu merge point for a given Ubuntu version
3983+
3984+ :param ubuntu_commitish str A commitish describing the latest
3985+ Ubuntu commit
3986+
3987+ :rtype str
3988+ :returns Commit hash of import of Debian version
3989+ @ubuntu_commitish is based on. The imported Debian version
3990+ must be an ancestor of @ubuntu_commitish. If no suitable
3991+ commit is found, an empty string is returned.
3992+ """
3993+ merge_base_tag = None
3994+
3995+ # obtain the nearest imported Debian version per the changelog
3996+ for version in self.get_all_changelog_versions_from_treeish(
3997+ ubuntu_commitish,
3998+ ):
3999+ # extract corresponding Debian version
4000+ debian_parts, _ = gitubuntu.versioning.split_version_string(
4001+ version
4002+ )
4003+ expected_debian_version = "".join(debian_parts)
4004+
4005+ # We do not currently handle the case of a Debian version
4006+ # being reimported. I think the proper way to support that
4007+ # would be to add a parameter to `git ubuntu merge` for the
4008+ # user to tell us which reimport tag is the one the Ubuntu
4009+ # delta is based on.
4010+ merge_base_tag = self.get_import_tag(
4011+ expected_debian_version,
4012+ 'pkg',
4013+ )
4014+
4015+ if merge_base_tag:
4016+ assert not self.get_all_reimport_tags(
4017+ expected_debian_version,
4018+ 'pkg',
4019+ )
4020+ break
4021+
4022+ if not merge_base_tag:
4023+ logging.error(
4024+ "Unable to find an import tag for any Debian version "
4025+ "in %s:debian/changelog.",
4026+ ubuntu_commitish,
4027+ )
4028+ return ''
4029+
4030+ merge_base_commit_hash = str(merge_base_tag.peel(pygit2.Commit).id)
4031+
4032+ try:
4033+ self.git_run(
4034+ [
4035+ 'merge-base',
4036+ '--is-ancestor',
4037+ merge_base_commit_hash,
4038+ ubuntu_commitish,
4039+ ],
4040+ verbose_on_failure=False,
4041+ )
4042+ except CalledProcessError as e:
4043+ if e.returncode != 1:
4044+ raise
4045+ logging.error(
4046+ "Found an import tag for %s (commit: %s), but it is "
4047+ "not an ancestor of %s.",
4048+ expected_debian_version,
4049+ merge_base_commit_hash,
4050+ ubuntu_commitish,
4051+ )
4052+ return ''
4053+
4054+ return merge_base_commit_hash
4055+>>>>>>> gitubuntu/git_repository.py
4056diff --git a/gitubuntu/git_repository_test.py b/gitubuntu/git_repository_test.py
4057new file mode 100644
4058index 0000000..72055f9
4059--- /dev/null
4060+++ b/gitubuntu/git_repository_test.py
4061@@ -0,0 +1,1191 @@
4062+<<<<<<< gitubuntu/git_repository_test.py
4063+=======
4064+import copy
4065+import datetime
4066+import itertools
4067+import os
4068+import pkg_resources
4069+import shutil
4070+import tempfile
4071+import unittest
4072+import unittest.mock
4073+
4074+import pygit2
4075+import pytest
4076+
4077+import gitubuntu.git_repository as target
4078+from gitubuntu.git_repository import HeadInfoItem
4079+from gitubuntu.repo_builder import (
4080+ Blob,
4081+ Commit,
4082+ Placeholder,
4083+ Repo,
4084+ SourceTree,
4085+ Symlink,
4086+ Tree,
4087+)
4088+from gitubuntu.source_builder import Source, SourceSpec
4089+import gitubuntu.spec
4090+from gitubuntu.test_fixtures import (
4091+ repo,
4092+ pygit2_repo,
4093+)
4094+from gitubuntu.test_util import get_test_changelog
4095+
4096+
4097+@pytest.mark.parametrize('same_remote_branch_names, different_remote_branch_names, expected', [
4098+ ([], [], ''),
4099+ (['pkg/ubuntu/xenial-devel',], [], 'pkg/ubuntu/xenial-devel'),
4100+ (['pkg/ubuntu/xenial-security',], [], 'pkg/ubuntu/xenial-security'),
4101+ (['pkg/ubuntu/xenial-updates', 'pkg/ubuntu/xenial-devel'], [],
4102+ 'pkg/ubuntu/xenial-devel'
4103+ ),
4104+ ([], ['pkg/ubuntu/xenial-updates', 'pkg/ubuntu/xenial-devel'],
4105+ ''
4106+ ),
4107+ (['pkg/ubuntu/zesty-devel', 'pkg/ubuntu/zesty-proposed', 'pkg/ubuntu/devel'], [], 'pkg/ubuntu/devel'),
4108+])
4109+def test__derive_target_branch_string(same_remote_branch_names,
4110+ different_remote_branch_names, expected
4111+):
4112+ remote_branch_objects = []
4113+ for branch_name in same_remote_branch_names:
4114+ b = unittest.mock.Mock()
4115+ b.peel(pygit2.Tree).id = unittest.mock.sentinel.same_id
4116+ b.branch_name = branch_name
4117+ remote_branch_objects.append(b)
4118+ for branch_name in different_remote_branch_names:
4119+ b = unittest.mock.Mock()
4120+ b.peel(pygit2.Tree).id = object() # need a different sentinel for each
4121+ b.branch_name = branch_name
4122+ remote_branch_objects.append(b)
4123+ target_branch_string = target._derive_target_branch_string(
4124+ remote_branch_objects
4125+ )
4126+ assert target_branch_string == expected
4127+
4128+
4129+@pytest.mark.parametrize('changelog_name, expected', [
4130+ ('test_versions_1', ['1.0', None]),
4131+ ('test_versions_2', ['2.0', '1.0']),
4132+ ('test_versions_3', ['4.0', '3.0']),
4133+ ('test_versions_unknown', ['ss-970814-1', None]),
4134+])
4135+def test_changelog_versions(changelog_name, expected):
4136+ test_changelog = get_test_changelog(changelog_name)
4137+ assert [test_changelog.version, test_changelog.previous_version] == expected
4138+
4139+
4140+@pytest.mark.parametrize('changelog_name, expected', [
4141+ ('test_versions_unknown', ['ss-970814-1',]),
4142+])
4143+def test_changelog_all_versions(changelog_name, expected):
4144+ test_changelog = get_test_changelog(changelog_name)
4145+ assert test_changelog.all_versions == expected
4146+
4147+
4148+def test_changelog_distribution():
4149+ test_changelog = get_test_changelog('test_distribution')
4150+ assert test_changelog.distribution == 'xenial'
4151+
4152+
4153+def test_changelog_date():
4154+ test_changelog = get_test_changelog('test_date_1')
4155+ assert test_changelog.date == 'Mon, 12 May 2016 08:14:34 -0700'
4156+ test_changelog = get_test_changelog('test_date_2')
4157+ assert test_changelog.date == 'Mon, 12 May 2016 08:14:34 -0700'
4158+
4159+
4160+@pytest.mark.parametrize('changelog_name, expected', [
4161+ ('test_maintainer_1', 'Test Maintainer <test-maintainer@donotmail.com>'),
4162+ ('test_maintainer_2', '<test-maintainer@donotmail.com>'),
4163+])
4164+def test_changelog_maintainer(changelog_name, expected):
4165+ test_changelog = get_test_changelog(changelog_name)
4166+ assert test_changelog.maintainer == expected
4167+
4168+
4169+def test_changelog_maintainer_invalid():
4170+ with pytest.raises(ValueError):
4171+ test_changelog = get_test_changelog('test_maintainer_3')
4172+ test_changelog.maintainer
4173+
4174+
4175+def test_changelog_multiple_angle_brackets():
4176+ """An email address with extra angle brackets should still parse"""
4177+ test_changelog = get_test_changelog('test_multiple_angle_brackets')
4178+ assert test_changelog.git_authorship()[1] == 'micah@debian.org'
4179+
4180+
4181+@pytest.mark.parametrize(['input_date_string', 'expected_result'], [
4182+ # The normal complete form
4183+ ('Mon, 12 May 2016 08:14:34 -0700', (2016, 5, 12, 8, 14, 34, -7)),
4184+ # Day of week missing, such as in:
4185+ # datefudge 1.12
4186+ ('12 May 2016 08:14:34 -0700', (2016, 5, 12, 8, 14, 34, -7)),
4187+ # Full (not abbreviated) month name, such as in:
4188+ # dnsmasq 2.32-2
4189+ # dropbear 0.42-1
4190+ # e2fsprogs 1.42.11-1
4191+ # efibootmgr 0.5.4-7
4192+ # hunspell-br 0.11-1
4193+ # kubuntu-default-settings 1:6.06-22
4194+ # libvformat 1.13-4
4195+ ('12 June 2016 08:14:34 -0700', (2016, 6, 12, 8, 14, 34, -7)),
4196+ # Full (not abbreviated) day of week name, such as in:
4197+ # logcheck 1.2.22a
4198+ ('Thursday, 15 May 2016 08:14:34 -0700', (2016, 5, 15, 8, 14, 34, -7)),
4199+ # Part-abbreviated day of week name, such as in:
4200+ # kubuntu-meta 1.76
4201+ ('Thur, 15 May 2016 08:14:34 -0700', (2016, 5, 15, 8, 14, 34, -7)),
4202+])
4203+def test_parse_changelog_date(input_date_string, expected_result):
4204+ """_parse_changelog_date should parse a basic date string correctly
4205+
4206+ :param str input_date_string: the timestamp part of the changelog signoff
4207+ line
4208+ :param tuple(int, int, int, int, int, int, int) expected_result: the
4209+ expected parse result in (year, month, day, hour, minute, second,
4210+ timezone_offset_in_hours) form. The actual expected result needs to be
4211+ a datetime.datetime object; to avoid duplication in test parameters
4212+ this will be instantiated within the test.
4213+ """
4214+ actual_result = target.Changelog._parse_changelog_date(input_date_string)
4215+ expected_result_datetime = datetime.datetime(
4216+ *expected_result[:6],
4217+ tzinfo=datetime.timezone(datetime.timedelta(hours=expected_result[6])),
4218+ )
4219+ assert actual_result == expected_result_datetime
4220+
4221+
4222+@pytest.mark.parametrize(['input_date_string'], [
4223+ ('Mon, 30 Feb 2020 15:50:58 +0200',), # ghostscript 9.50~dfsg-5ubuntu4
4224+ ('Mon, 03 Sep 2018 00:43:25 -7000',), # lxqt-config 0.13.0-0ubuntu4
4225+ ('Tue, 17 May 2008 10:93:55 -0500',), # iscsitarget
4226+ # 0.4.15+svn148-2.1ubuntu1
4227+ ('Monu, 22 Jan 2007 22:10:50 -0500',), # mail-spf-perl 2.004-0ubuntu1
4228+ ('Wed, 29 Augl 2007 16:14:11 +0200',), # nut 2.2.0-2
4229+])
4230+def test_changelog_date_parse_errors(input_date_string):
4231+ """_parse_changelog_date should raise ValueError on illegal dates
4232+
4233+ :param str input_date_string: the timestamp part of the changelog signoff
4234+ line
4235+ """
4236+ with pytest.raises(ValueError):
4237+ target.Changelog._parse_changelog_date(input_date_string)
4238+
4239+
4240+@pytest.mark.parametrize(
4241+ 'changelog_name, name, email, epoch_seconds, offset', [
4242+ (
4243+ 'test_maintainer_1',
4244+ 'Test Maintainer',
4245+ 'test-maintainer@donotmail.com',
4246+ 0,
4247+ 0,
4248+ ),
4249+ (
4250+ 'test_maintainer_2',
4251+ 'Unnamed', # git won't handle empty names; see the spec
4252+ 'test-maintainer@donotmail.com',
4253+ 0,
4254+ 0,
4255+ ),
4256+ (
4257+ 'test_date_1',
4258+ 'Test Maintainer',
4259+ 'test-maintainer@donotmail.com',
4260+ 1463066074,
4261+ -420,
4262+ ),
4263+ (
4264+ 'test_date_2',
4265+ 'Test Maintainer',
4266+ 'test-maintainer@donotmail.com',
4267+ 1463066074,
4268+ -420,
4269+ ),
4270+ (
4271+ 'maintainer_name_leading_space',
4272+ 'Test Maintainer',
4273+ 'test-maintainer@example.com',
4274+ 0,
4275+ 0,
4276+ ),
4277+ (
4278+ 'maintainer_name_trailing_space',
4279+ 'Test Maintainer',
4280+ 'test-maintainer@example.com',
4281+ 0,
4282+ 0,
4283+ ),
4284+ (
4285+ 'maintainer_name_inner_space',
4286+ 'Test Maintainer',
4287+ 'test-maintainer@example.com',
4288+ 0,
4289+ 0,
4290+ ),
4291+])
4292+def test_changelog_authorship(
4293+ changelog_name,
4294+ name,
4295+ email,
4296+ epoch_seconds,
4297+ offset,
4298+):
4299+ result = get_test_changelog(changelog_name).git_authorship()
4300+ assert result == (name, email, epoch_seconds, offset)
4301+
4302+
4303+def test_changelog_utf8():
4304+ test_changelog = get_test_changelog('test_utf8_error')
4305+ assert test_changelog.version == '1.0.3-2'
4306+
4307+
4308+def test_changelog_duplicate():
4309+ # Changelog.all_versions should successfully return without an assertion
4310+
4311+ # Xenial's dpkg-parsechangelog eliminates duplicate versions. Bionic's
4312+ # dpkg-parsechangelog does not. We rely on the behaviour of
4313+ # dpkg-parsechangelog from Bionic, where this test passes. The test fails
4314+ # when using Xenial's dpkg-parsechangelog, where its behaviour doesn't
4315+ # match our assumptions elsewhere.
4316+
4317+ # -with-extra includes an extra changelog entry at the end. This is
4318+ # currently needed to trip the assertion because it truncates the longer
4319+ # list before its comparison. This will get fixed in a subsequent commit,
4320+ # but using it here ensures that this test will correctly trip regardless
4321+ # of the presence of that unrelated bug.
4322+ test_changelog = get_test_changelog('duplicate-version-with-extra')
4323+ test_changelog.all_versions
4324+
4325+
4326+def test_changelog_all_versions_assertion_mismatched_length():
4327+ # if Changelog.all_versions finds that self._changelog.versions mismatches
4328+ # self._shell_all_versions, it is supposed to raise an assertion. Here is
4329+ # an edge case where at one point in development it did not. We fake both
4330+ # _changelog.versions and _shell_all_versions to an edge case where they
4331+ # mismatch.
4332+ with unittest.mock.patch(
4333+ 'gitubuntu.git_repository.Changelog._shell_all_versions',
4334+ new_callable=unittest.mock.PropertyMock
4335+ ) as mock_shell_all_versions:
4336+ mock_shell_all_versions.return_value = ['a']
4337+ test_changelog = target.Changelog(b'')
4338+ test_changelog._changelog = unittest.mock.Mock()
4339+ test_changelog._changelog.versions = ['a', 'b']
4340+ with pytest.raises(target.ChangelogError):
4341+ test_changelog.all_versions
4342+
4343+
4344+@pytest.mark.parametrize('tree_func', [
4345+ # The tree_func parameter is a function that accepts a mock Blob that is to
4346+ # represent the changelog blob itself and returns a mock Tree with the mock
4347+ # Blob embedded somewhere within it. The test function can then ensure that
4348+ # follow_symlinks_to_blob can correctly find the changelog Blob given the
4349+ # Tree.
4350+
4351+ # Of course this is only expected to work if, after checking out the Tree,
4352+ # "cat debian/changelog" would work. But this allows us to test the various
4353+ # permutations of symlink following in Trees that _are_ valid.
4354+
4355+ # Simple case
4356+ lambda b: Tree({
4357+ 'debian': Tree({'changelog': b}),
4358+ }),
4359+
4360+ # Symlink in debian/
4361+ lambda b: Tree({
4362+ 'debian': Tree({
4363+ 'changelog.real': b,
4364+ 'changelog': Symlink('changelog.real'),
4365+ }),
4366+ }),
4367+
4368+ # Symlink to parent directory
4369+ lambda b: Tree({
4370+ 'changelog': b,
4371+ 'debian': Tree({
4372+ 'changelog': Symlink('../changelog'),
4373+ })
4374+ }),
4375+
4376+ # Symlink to subdirectory
4377+ lambda b: Tree({
4378+ 'debian': Tree({
4379+ 'changelog': Symlink('subdirectory/changelog'),
4380+ 'subdirectory': Tree({'changelog': b}),
4381+ })
4382+ }),
4383+
4384+ # debian/ itself is a symlink to a different directory
4385+ lambda b: Tree({
4386+ 'pkg': Tree({'changelog': b}),
4387+ 'debian': Symlink('pkg'),
4388+ })
4389+])
4390+def test_follow_symlinks_to_blob(pygit2_repo, tree_func):
4391+ blob = Blob(b'')
4392+ blob_id = blob.write(pygit2_repo)
4393+ tree = pygit2_repo.get(tree_func(blob).write(pygit2_repo))
4394+ result_blob = target.follow_symlinks_to_blob(
4395+ pygit2_repo,
4396+ tree,
4397+ 'debian/changelog',
4398+ )
4399+ assert result_blob.id == blob_id
4400+
4401+
4402+@pytest.mark.parametrize('tree', [
4403+ Tree({}),
4404+ Tree({'debian': Tree({})}),
4405+ Tree({'debian': Tree({'changelog': Symlink('other')})}),
4406+ Tree({'debian': Tree({'changelog': Symlink('../other')})}),
4407+])
4408+def test_follow_symlinks_to_blob_not_found(pygit2_repo, tree):
4409+ pygit2_tree = pygit2_repo.get(tree.write(pygit2_repo))
4410+ with pytest.raises(KeyError):
4411+ target.follow_symlinks_to_blob(
4412+ pygit2_repo,
4413+ pygit2_tree,
4414+ 'debian/changelog',
4415+ )
4416+
4417+
4418+def test_renameable_dir_basename(tmpdir):
4419+ p = tmpdir.join('foo')
4420+ p.ensure()
4421+ rd = target.RenameableDir(str(p))
4422+ assert rd.basename == 'foo'
4423+
4424+
4425+def test_renameable_dir_basename_setter(tmpdir):
4426+ p = tmpdir.join('foo')
4427+ p.ensure()
4428+ rd = target.RenameableDir(str(p))
4429+ rd.basename = 'bar'
4430+ assert rd.basename == 'bar'
4431+ assert tmpdir.join('bar').check()
4432+
4433+
4434+def test_dot_git_match(tmpdir):
4435+ for name in ['.git', 'git', '..git', 'other']:
4436+ tmpdir.join(name).ensure()
4437+
4438+ result = set(
4439+ x.basename
4440+ for x in tmpdir.listdir(
4441+ fil=lambda x: target._dot_git_match(str(x.basename))
4442+ )
4443+ )
4444+ assert result == set(['.git', '..git'])
4445+
4446+
4447+def test_renameable_dir_listdir(tmpdir):
4448+ for name in ['.git', 'git', '..git', 'other']:
4449+ tmpdir.join(name).ensure()
4450+ rd = target.RenameableDir(str(tmpdir))
4451+ result = set(rd.listdir(target._dot_git_match))
4452+ assert result == set([
4453+ target.RenameableDir(os.path.join(str(tmpdir), '.git')),
4454+ target.RenameableDir(os.path.join(str(tmpdir), '..git')),
4455+ ])
4456+
4457+
4458+def test_renamable_dir_recursive(tmpdir):
4459+ a = tmpdir.join('foo')
4460+ a.ensure_dir()
4461+ b = tmpdir.join('bar')
4462+ b.ensure()
4463+ assert target.RenameableDir(str(a)).recursive
4464+ assert not target.RenameableDir(str(b)).recursive
4465+
4466+
4467+def test_renameable_dir_recursive_symlink_directory(tmpdir):
4468+ """A RenameableDir should not treat a broken symlink as recursive"""
4469+ test_symlink = tmpdir.join('foo')
4470+ nonexistent_file = tmpdir.join('nonexistent_file')
4471+ test_symlink.mksymlinkto(nonexistent_file)
4472+ assert not target.RenameableDir(str(test_symlink)).recursive
4473+
4474+
4475+def test_renameable_dir_str(tmpdir):
4476+ p = tmpdir.join('foo')
4477+ p.ensure()
4478+ rd = target.RenameableDir(str(p))
4479+ assert str(rd) == os.path.join(str(tmpdir), 'foo')
4480+
4481+
4482+def test_renameable_dir_repr(tmpdir):
4483+ p = tmpdir.join('foo')
4484+ p.ensure()
4485+ rd = target.RenameableDir(str(p))
4486+ assert repr(rd) == ("RenameableDir('%s/foo')" % str(tmpdir))
4487+
4488+
4489+def test_renameable_dir_hash_eq(tmpdir):
4490+ p1a = tmpdir.join('foo')
4491+ p1b = tmpdir.join('foo')
4492+ p2 = tmpdir.join('bar')
4493+
4494+ p1a.ensure()
4495+ p2.ensure()
4496+
4497+ rd1a = target.RenameableDir(str(p1a))
4498+ rd1b = target.RenameableDir(str(p1b))
4499+ rd2 = target.RenameableDir(str(p2))
4500+
4501+ assert rd1a == rd1b
4502+ assert rd1a != rd2
4503+
4504+
4505+def test_renameable_dir_must_exist(tmpdir):
4506+ """A RenameableDir should reject a path that doesn't exist"""
4507+ with pytest.raises(FileNotFoundError):
4508+ target.RenameableDir(tmpdir.join('a'))
4509+
4510+
4511+def test_fake_renameable_dir_basename():
4512+ path = target.FakeRenameableDir('foo', None)
4513+ assert path.basename == 'foo'
4514+
4515+
4516+def test_fake_renameable_dir_basename_setter():
4517+ path = target.FakeRenameableDir('foo', None)
4518+ path.basename = 'bar'
4519+ assert path.basename == 'bar'
4520+
4521+
4522+def test_fake_renameable_dir_listdir():
4523+ path = target.FakeRenameableDir(None, [
4524+ target.FakeRenameableDir('.git', None),
4525+ target.FakeRenameableDir('git', None),
4526+ target.FakeRenameableDir('..git', None),
4527+ target.FakeRenameableDir('other', None),
4528+ ])
4529+ result = set(x.basename for x in path.listdir(fil=target._dot_git_match))
4530+ assert result == set(['.git', '..git'])
4531+
4532+
4533+def test_fake_renameable_dir_recursive():
4534+ assert target.FakeRenameableDir(['foo'], []).recursive
4535+ assert not target.FakeRenameableDir(['foo'], None).recursive
4536+
4537+
4538+def test_fake_renameable_dir_hash_eq():
4539+ variations = [
4540+ target.FakeRenameableDir(None, None),
4541+ target.FakeRenameableDir(None, []),
4542+ target.FakeRenameableDir('foo', []),
4543+ target.FakeRenameableDir(None, [
4544+ target.FakeRenameableDir('foo', None)]
4545+ ),
4546+ target.FakeRenameableDir(None, [
4547+ target.FakeRenameableDir('foo', [
4548+ target.FakeRenameableDir('bar', None)
4549+ ]),
4550+ ]),
4551+ ]
4552+ for a, b in itertools.product(variations, variations):
4553+ if a is b:
4554+ assert a == b
4555+ else:
4556+ assert a != b
4557+
4558+
4559+def test_fake_renameable_dir_repr():
4560+ rd = target.FakeRenameableDir('foo', [target.FakeRenameableDir('bar', [])])
4561+ assert (
4562+ repr(rd) == "FakeRenameableDir('foo', [FakeRenameableDir('bar', [])])"
4563+ )
4564+
4565+
4566+@pytest.mark.parametrize('initial,expected', [
4567+ # Empty directory remains unchanged
4568+ (
4569+ target.FakeRenameableDir(None, []),
4570+ target.FakeRenameableDir(None, []),
4571+ ),
4572+ # Basic .git -> ..git escape
4573+ (
4574+ target.FakeRenameableDir(
4575+ None,
4576+ [target.FakeRenameableDir('.git', None)],
4577+ ),
4578+ target.FakeRenameableDir(
4579+ None,
4580+ [target.FakeRenameableDir('..git', None)],
4581+ ),
4582+ ),
4583+ # .git contains a .git
4584+ (
4585+ target.FakeRenameableDir(
4586+ None,
4587+ [
4588+ target.FakeRenameableDir(
4589+ '.git',
4590+ [target.FakeRenameableDir('.git', None)],
4591+ )
4592+ ],
4593+ ),
4594+ target.FakeRenameableDir(
4595+ None,
4596+ [
4597+ target.FakeRenameableDir(
4598+ '..git',
4599+ [target.FakeRenameableDir('..git', None)],
4600+ )
4601+ ],
4602+ ),
4603+ ),
4604+ # git remains unchanged
4605+ (
4606+ target.FakeRenameableDir(
4607+ None,
4608+ [target.FakeRenameableDir('git', None)],
4609+ ),
4610+ target.FakeRenameableDir(
4611+ None,
4612+ [target.FakeRenameableDir('git', None)],
4613+ ),
4614+ ),
4615+ # .git and ..git both exist
4616+ (
4617+ target.FakeRenameableDir(
4618+ None,
4619+ [
4620+ target.FakeRenameableDir('.git', None),
4621+ target.FakeRenameableDir('..git', None),
4622+ ],
4623+ ),
4624+ target.FakeRenameableDir(
4625+ None,
4626+ [
4627+ target.FakeRenameableDir('..git', None),
4628+ target.FakeRenameableDir('...git', None),
4629+ ],
4630+ ),
4631+ ),
4632+ # Ordinary directory contains a .git
4633+ (
4634+ target.FakeRenameableDir(
4635+ None,
4636+ [
4637+ target.FakeRenameableDir(
4638+ 'foo',
4639+ [target.FakeRenameableDir('.git', None)],
4640+ )
4641+ ]
4642+ ),
4643+ target.FakeRenameableDir(
4644+ None,
4645+ [
4646+ target.FakeRenameableDir(
4647+ 'foo',
4648+ [target.FakeRenameableDir('..git', None)],
4649+ )
4650+ ]
4651+ ),
4652+ ),
4653+])
4654+def test_escape_dot_git(initial, expected):
4655+ state = copy.deepcopy(initial)
4656+ # Once escaped, we should get to what was expected
4657+ target._escape_unescape_dot_git(state, target._EscapeDirection.ESCAPE)
4658+ assert state == expected
4659+ # Once unescaped, we should get back to where we started since the escaping
4660+ # mechanism is lossless.
4661+ target._escape_unescape_dot_git(state, target._EscapeDirection.UNESCAPE)
4662+ assert state == initial
4663+
4664+
4665+def test_unescape_dot_git_raises():
4666+ """Test that unescaping something with '.git' raises an exception."""
4667+ with pytest.raises(RuntimeError):
4668+ target._escape_unescape_dot_git(
4669+ target.FakeRenameableDir(
4670+ None,
4671+ [target.FakeRenameableDir('.git', None)],
4672+ ),
4673+ direction=target._EscapeDirection.UNESCAPE,
4674+ )
4675+
4676+
4677+@pytest.mark.parametrize('direction', [
4678+ target._EscapeDirection.ESCAPE,
4679+ target._EscapeDirection.UNESCAPE,
4680+])
4681+def test_escape_dot_git_ordering(direction):
4682+ """Test that renames happen in the correct order.
4683+
4684+ ...git -> ....git must happen before ..git -> ...git to avoid a collision,
4685+ and vice versa in the unescape case.
4686+ """
4687+ # Avoid '.git' as it isn't valid in the reverse direction
4688+ inner2 = target.FakeRenameableDir('..git', None)
4689+ inner3 = target.FakeRenameableDir('...git', None)
4690+ inputs = [inner2, inner3]
4691+ if direction is target._EscapeDirection.ESCAPE:
4692+ expected_order = [inner3, inner2]
4693+ else:
4694+ expected_order = [inner2, inner3]
4695+ for given_order in [inputs, reversed(inputs)]:
4696+ top = target.FakeRenameableDir(None, given_order)
4697+ target._escape_unescape_dot_git(top, direction)
4698+ assert all(x is y for x, y in zip(top._rename_record, expected_order))
4699+
4700+
4701+def test_empty_dir_to_tree(pygit2_repo, tmpdir):
4702+ tree_hash = target.GitUbuntuRepository.dir_to_tree(
4703+ pygit2_repo,
4704+ str(tmpdir),
4705+ )
4706+ assert tree_hash == str(Tree({}).write(pygit2_repo))
4707+
4708+
4709+def test_onefile_dir_to_tree(pygit2_repo, tmpdir):
4710+ tmpdir.join('foo').write('bar')
4711+ tree_hash = target.GitUbuntuRepository.dir_to_tree(
4712+ pygit2_repo,
4713+ str(tmpdir),
4714+ )
4715+ assert tree_hash == str(Tree({'foo': Blob(b'bar')}).write(pygit2_repo))
4716+
4717+
4718+def test_git_escape_dir_to_tree(pygit2_repo, tmpdir):
4719+ tmpdir.mkdir('.git')
4720+ tree_hash = target.GitUbuntuRepository.dir_to_tree(
4721+ pygit2_repo,
4722+ str(tmpdir),
4723+ escape=True,
4724+ )
4725+ assert tree_hash == str(Tree({'..git': Tree({})}).write(pygit2_repo))
4726+
4727+
4728+@pytest.mark.parametrize('tree_data,expected_path', [
4729+ # Empty tree -> default
4730+ (Tree({}), 'debian/patches/series'),
4731+
4732+ # Empty debian/patches directory -> default
4733+ (Tree({'debian': Tree({'patches': Tree({})})}), 'debian/patches/series'),
4734+
4735+ # Only debian/patches/series -> that one
4736+ (
4737+ Tree({'debian': Tree({'patches': Tree({'series': Blob(b'')})})}),
4738+ 'debian/patches/series',
4739+ ),
4740+
4741+ # Only debian/patches/debian.series -> that one
4742+ (
4743+ Tree({'debian': Tree({'patches': Tree({
4744+ 'debian.series': Blob(b'')
4745+ })})}),
4746+ 'debian/patches/debian.series',
4747+ ),
4748+
4749+ # Both -> debian.series
4750+ (
4751+ Tree({'debian': Tree({'patches': Tree({
4752+ 'debian.series': Blob(b''),
4753+ 'series': Blob(b''),
4754+ })})}),
4755+ 'debian/patches/debian.series',
4756+ ),
4757+])
4758+def test_determine_quilt_series_path(pygit2_repo, tree_data, expected_path):
4759+ tree_obj = pygit2_repo.get(tree_data.write(pygit2_repo))
4760+ path = target.determine_quilt_series_path(pygit2_repo, tree_obj)
4761+ assert path == expected_path
4762+
4763+
4764+def test_quilt_env(pygit2_repo):
4765+ tree_builder = Tree({'debian':
4766+ Tree({'patches': Tree({'debian.series': Blob(b'')})})
4767+ })
4768+ tree_obj = pygit2_repo.get(tree_builder.write(pygit2_repo))
4769+ env = target.quilt_env(pygit2_repo, tree_obj)
4770+ assert env == {
4771+ 'EDITOR': 'true',
4772+ 'QUILT_NO_DIFF_INDEX': '1',
4773+ 'QUILT_NO_DIFF_TIMESTAMPS': '1',
4774+ 'QUILT_PATCHES': 'debian/patches',
4775+ 'QUILT_SERIES': 'debian/patches/debian.series',
4776+ }
4777+
4778+
4779+def test_repo_quilt_env(repo):
4780+ tree_builder = Tree({'debian':
4781+ Tree({'patches': Tree({'debian.series': Blob(b'')})})
4782+ })
4783+ tree_obj = repo.raw_repo.get(tree_builder.write(repo.raw_repo))
4784+ env = repo.quilt_env(tree_obj)
4785+ expected_inside = {
4786+ 'EDITOR': 'true',
4787+ 'QUILT_NO_DIFF_INDEX': '1',
4788+ 'QUILT_NO_DIFF_TIMESTAMPS': '1',
4789+ 'QUILT_PATCHES': 'debian/patches',
4790+ 'QUILT_SERIES': 'debian/patches/debian.series',
4791+ }
4792+ for k, v in expected_inside.items():
4793+ assert env[k] == v
4794+
4795+ # In addition to the settings above, check that
4796+ # GitUbuntuRepository.quilt_env has correctly merged in the usual
4797+ # environment. Testing that a few keys that we expect to be set are set
4798+ # should suffice.
4799+ expected_other_keys = ['HOME', 'GIT_DIR', 'GIT_WORK_TREE']
4800+ for k in expected_other_keys:
4801+ assert env[k]
4802+
4803+
4804+def test_repo_quilt_env_from_treeish_str(repo):
4805+ tree_builder = Tree({'debian':
4806+ Tree({'patches': Tree({'debian.series': Blob(b'')})})
4807+ })
4808+ tree_obj = repo.raw_repo.get(tree_builder.write(repo.raw_repo))
4809+ env = repo.quilt_env_from_treeish_str(str(tree_obj.id))
4810+ expected_inside = {
4811+ 'EDITOR': 'true',
4812+ 'QUILT_NO_DIFF_INDEX': '1',
4813+ 'QUILT_NO_DIFF_TIMESTAMPS': '1',
4814+ 'QUILT_PATCHES': 'debian/patches',
4815+ 'QUILT_SERIES': 'debian/patches/debian.series',
4816+ }
4817+ for k, v in expected_inside.items():
4818+ assert env[k] == v
4819+
4820+
4821+def test_repo_derive_env_change(repo):
4822+ # Changing the dictionary of a GitUbuntuRepository instance env attribute
4823+ # must not have any effect on the env itself. While this may stretch a
4824+ # little further than a normal instance property, it's worth enforcing this
4825+ # as this particular attribute is at particular risk due to how it tends to
4826+ # be used.
4827+ e1 = repo.env
4828+ e1[unittest.mock.sentinel.k] = unittest.mock.sentinel.v
4829+ assert unittest.mock.sentinel.k not in repo.env
4830+
4831+
4832+@pytest.mark.parametrize(
4833+ 'description, input_data, old_ubuntu, new_debian, expected',
4834+ [
4835+ (
4836+ 'Common case',
4837+ Repo(
4838+ commits=[
4839+ Commit.from_spec(
4840+ name='old/debian'
4841+ ),
4842+ Commit.from_spec(
4843+ parents=[Placeholder('old/debian')],
4844+ name='old/ubuntu',
4845+ changelog_versions=['1-1ubuntu1', '1-1'],
4846+ ),
4847+ Commit.from_spec(
4848+ parents=[Placeholder('old/debian')],
4849+ name='new/debian',
4850+ changelog_versions=['2-1', '1-1'],
4851+ ),
4852+ ],
4853+ tags={
4854+ 'pkg/import/1-1': Placeholder('old/debian'),
4855+ 'pkg/import/1-1ubuntu1': Placeholder('old/ubuntu'),
4856+ 'pkg/import/2-1': Placeholder('new/debian'),
4857+ },
4858+ ),
4859+ 'pkg/import/1-1ubuntu1',
4860+ 'pkg/import/2-1',
4861+ 'pkg/import/1-1',
4862+ ),
4863+ (
4864+ 'Ubuntu delta based on a NMU',
4865+ Repo(
4866+ commits=[
4867+ Commit.from_spec(
4868+ name='fork_point'
4869+ ),
4870+ Commit.from_spec(
4871+ parents=[Placeholder('fork_point')],
4872+ name='old/debian',
4873+ changelog_versions=['1-1.1', '1-1'],
4874+ ),
4875+ Commit.from_spec(
4876+ parents=[Placeholder('old/debian')],
4877+ name='old/ubuntu',
4878+ changelog_versions=['1-1.1ubuntu1', '1-1.1', '1-1'],
4879+ ),
4880+ Commit.from_spec(
4881+ parents=[Placeholder('fork_point')],
4882+ name='new/debian',
4883+ changelog_versions=['2-1', '1-1'],
4884+ ),
4885+ ],
4886+ tags={
4887+ 'pkg/import/1-1': Placeholder('fork_point'),
4888+ 'pkg/import/1-1.1': Placeholder('old/debian'),
4889+ 'pkg/import/1-1.1ubuntu1': Placeholder('old/ubuntu'),
4890+ 'pkg/import/2-1': Placeholder('new/debian'),
4891+ },
4892+ ),
4893+ 'pkg/import/1-1.1ubuntu1',
4894+ 'pkg/import/2-1',
4895+ 'pkg/import/1-1.1',
4896+ ),
4897+ (
4898+ 'Ubuntu upstream version head of Debian',
4899+ Repo(
4900+ commits=[
4901+ Commit.from_spec(
4902+ name='old/debian'
4903+ ),
4904+ Commit.from_spec(
4905+ parents=[Placeholder('old/debian')],
4906+ name='mid_ubuntu',
4907+ changelog_versions=['1-1ubuntu1', '1-1'],
4908+ ),
4909+ Commit.from_spec(
4910+ parents=[Placeholder('mid_ubuntu')],
4911+ name='old/ubuntu',
4912+ changelog_versions=['2-0ubuntu1', '1-1ubuntu1', '1-1'],
4913+ ),
4914+ Commit.from_spec(
4915+ parents=[Placeholder('old/debian')],
4916+ name='new/debian',
4917+ changelog_versions=['3-1', '1-1'],
4918+ ),
4919+ ],
4920+ tags={
4921+ 'pkg/import/1-1': Placeholder('old/debian'),
4922+ 'pkg/import/1-1ubuntu1': Placeholder('mid_ubuntu'),
4923+ 'pkg/import/2-0ubuntu1': Placeholder('old/ubuntu'),
4924+ 'pkg/import/3-1': Placeholder('new/debian'),
4925+ },
4926+ ),
4927+ 'pkg/import/2-0ubuntu1',
4928+ 'pkg/import/3-1',
4929+ 'pkg/import/1-1',
4930+ ),
4931+ ],
4932+)
4933+def test_repo_find_ubuntu_merge(
4934+ description,
4935+ repo,
4936+ input_data,
4937+ old_ubuntu,
4938+ new_debian,
4939+ expected,
4940+):
4941+ input_data.write(repo.raw_repo)
4942+ merge_base = repo.find_ubuntu_merge_base(old_ubuntu)
4943+
4944+ assert merge_base
4945+
4946+ assert str(
4947+ repo.get_commitish(merge_base).peel(pygit2.Commit).id
4948+ ) == str(
4949+ repo.get_commitish(expected).peel(pygit2.Commit).id
4950+ )
4951+
4952+
4953+def test_repo_does_cleanup():
4954+ path = tempfile.mkdtemp()
4955+ try:
4956+ repo = target.GitUbuntuRepository(
4957+ path,
4958+ delete_on_close=True,
4959+ )
4960+ repo.close()
4961+ assert not os.path.exists(path)
4962+ finally:
4963+ shutil.rmtree(path, ignore_errors=True)
4964+
4965+
4966+def test_repo_does_not_cleanup():
4967+ path = tempfile.mkdtemp()
4968+ try:
4969+ repo = target.GitUbuntuRepository(
4970+ path,
4971+ delete_on_close=False,
4972+ )
4973+ repo.close()
4974+ assert os.path.exists(path)
4975+ finally:
4976+ shutil.rmtree(path, ignore_errors=True)
4977+
4978+
4979+@pytest.mark.parametrize(
4980+ [
4981+ 'year',
4982+ 'month',
4983+ 'day',
4984+ 'hours',
4985+ 'minutes',
4986+ 'seconds',
4987+ 'milliseconds',
4988+ 'hour_delta',
4989+ 'expected',
4990+ ], [
4991+ (1970, 1, 1, 0, 0, 0, 0, 0, (0, 0)),
4992+ (1970, 1, 1, 0, 0, 0, 600, 0, (0, 0)),
4993+ (1970, 1, 1, 1, 0, 0, 0, 1, (0, 60)),
4994+ (1970, 1, 1, 0, 0, 0, 0, -1, (3600, -60)),
4995+ (1971, 2, 3, 4, 5, 6, 7, -8, (34430706, -480)),
4996+ ]
4997+)
4998+def test_datetime_to_signature_spec(
4999+ year,
5000+ month,
The diff has been truncated for viewing.

Subscribers

People subscribed via source and target branches