Merge ~pappacena/turnip:enable-http-v2 into turnip:master
- Git
- lp:~pappacena/turnip
- enable-http-v2
- Merge into master
Proposed by
Thiago F. Pappacena
Status: | Work in progress |
---|---|
Proposed branch: | ~pappacena/turnip:enable-http-v2 |
Merge into: | turnip:master |
Prerequisite: | ~pappacena/turnip:run-v2-commands |
Diff against target: |
684 lines (+166/-217) 7 files modified
turnip/pack/git.py (+20/-58) turnip/pack/helpers.py (+30/-63) turnip/pack/http.py (+27/-7) turnip/pack/tests/test_functional.py (+8/-5) turnip/pack/tests/test_git.py (+4/-56) turnip/pack/tests/test_helpers.py (+41/-28) turnip/pack/tests/test_http.py (+36/-0) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Launchpad code reviewers | Pending | ||
Review via email: mp+389131@code.launchpad.net |
Commit message
Enabling protocol v2 compatibility on HTTP frontend
Description of the change
To post a comment you must log in.
~pappacena/turnip:enable-http-v2
updated
- 6bd0254... by Thiago F. Pappacena
-
Cleaning up tests
Unmerged commits
- 6bd0254... by Thiago F. Pappacena
-
Cleaning up tests
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/turnip/pack/git.py b/turnip/pack/git.py | |||
2 | index 1b34ac0..85fa57e 100644 | |||
3 | --- a/turnip/pack/git.py | |||
4 | +++ b/turnip/pack/git.py | |||
5 | @@ -31,14 +31,11 @@ from turnip.config import config | |||
6 | 31 | from turnip.helpers import compose_path | 31 | from turnip.helpers import compose_path |
7 | 32 | from turnip.pack.helpers import ( | 32 | from turnip.pack.helpers import ( |
8 | 33 | decode_packet, | 33 | decode_packet, |
9 | 34 | DELIM_PKT, | ||
10 | 35 | decode_request, | 34 | decode_request, |
11 | 36 | encode_packet, | 35 | encode_packet, |
12 | 37 | encode_request, | 36 | encode_request, |
13 | 38 | ensure_config, | 37 | ensure_config, |
14 | 39 | ensure_hooks, | 38 | ensure_hooks, |
15 | 40 | FLUSH_PKT, | ||
16 | 41 | get_capabilities_advertisement, | ||
17 | 42 | INCOMPLETE_PKT, | 39 | INCOMPLETE_PKT, |
18 | 43 | translate_xmlrpc_fault, | 40 | translate_xmlrpc_fault, |
19 | 44 | ) | 41 | ) |
20 | @@ -242,20 +239,15 @@ class GitProcessProtocol(protocol.ProcessProtocol): | |||
21 | 242 | 239 | ||
22 | 243 | _err_buffer = b'' | 240 | _err_buffer = b'' |
23 | 244 | 241 | ||
25 | 245 | def __init__(self, peer, cmd_input=None): | 242 | def __init__(self, peer): |
26 | 246 | self.peer = peer | 243 | self.peer = peer |
27 | 247 | self.cmd_input = cmd_input | ||
28 | 248 | self.out_started = False | 244 | self.out_started = False |
29 | 249 | 245 | ||
30 | 250 | def connectionMade(self): | 246 | def connectionMade(self): |
31 | 251 | self.peer.setPeer(self) | 247 | self.peer.setPeer(self) |
32 | 252 | self.peer.transport.registerProducer(self, True) | 248 | self.peer.transport.registerProducer(self, True) |
39 | 253 | if not self.cmd_input: | 249 | self.transport.registerProducer( |
40 | 254 | self.transport.registerProducer( | 250 | UnstoppableProducerWrapper(self.peer.transport), True) |
35 | 255 | UnstoppableProducerWrapper(self.peer.transport), True) | ||
36 | 256 | else: | ||
37 | 257 | self.transport.write(self.cmd_input) | ||
38 | 258 | self.loseWriteConnection() | ||
41 | 259 | self.peer.resumeProducing() | 251 | self.peer.resumeProducing() |
42 | 260 | 252 | ||
43 | 261 | def outReceived(self, data): | 253 | def outReceived(self, data): |
44 | @@ -440,29 +432,12 @@ class PackBackendProtocol(PackServerProtocol): | |||
45 | 440 | hookrpc_key = None | 432 | hookrpc_key = None |
46 | 441 | expect_set_symbolic_ref = False | 433 | expect_set_symbolic_ref = False |
47 | 442 | 434 | ||
48 | 443 | def getV2CommandInput(self, params): | ||
49 | 444 | """Reconstruct what should be sent to git's stdin from the | ||
50 | 445 | parameters received.""" | ||
51 | 446 | cmd_input = encode_packet(b"command=%s\n" % params.get(b'command')) | ||
52 | 447 | for capability in params["capabilities"].split(b"\n"): | ||
53 | 448 | cmd_input += encode_packet(b"%s\n" % capability) | ||
54 | 449 | cmd_input += DELIM_PKT | ||
55 | 450 | ignore_keys = (b'capabilities', b'version') | ||
56 | 451 | for k, v in params.items(): | ||
57 | 452 | k = six.ensure_binary(k) | ||
58 | 453 | if k.startswith(b"turnip-") or k in ignore_keys: | ||
59 | 454 | continue | ||
60 | 455 | for param_value in v.split(b'\n'): | ||
61 | 456 | value = (b"" if not param_value else b" %s" % param_value) | ||
62 | 457 | cmd_input += encode_packet(b"%s%s\n" % (k, value)) | ||
63 | 458 | cmd_input += FLUSH_PKT | ||
64 | 459 | return cmd_input | ||
65 | 460 | |||
66 | 461 | @defer.inlineCallbacks | 435 | @defer.inlineCallbacks |
67 | 462 | def requestReceived(self, command, raw_pathname, params): | 436 | def requestReceived(self, command, raw_pathname, params): |
68 | 463 | self.extractRequestMeta(command, raw_pathname, params) | 437 | self.extractRequestMeta(command, raw_pathname, params) |
69 | 464 | self.command = command | 438 | self.command = command |
70 | 465 | self.raw_pathname = raw_pathname | 439 | self.raw_pathname = raw_pathname |
71 | 440 | self.params = params | ||
72 | 466 | self.path = compose_path(self.factory.root, self.raw_pathname) | 441 | self.path = compose_path(self.factory.root, self.raw_pathname) |
73 | 467 | auth_params = self.createAuthParams(params) | 442 | auth_params = self.createAuthParams(params) |
74 | 468 | 443 | ||
75 | @@ -482,31 +457,20 @@ class PackBackendProtocol(PackServerProtocol): | |||
76 | 482 | self.resumeProducing() | 457 | self.resumeProducing() |
77 | 483 | return | 458 | return |
78 | 484 | 459 | ||
79 | 485 | send_path_as_option = False | ||
80 | 486 | cmd_input = None | ||
81 | 487 | cmd_env = {} | 460 | cmd_env = {} |
82 | 488 | write_operation = False | 461 | write_operation = False |
102 | 489 | if not get_capabilities_advertisement(params.get(b'version', 1)): | 462 | version = self.params.get(b'version', 0) |
103 | 490 | if command == b'git-upload-pack': | 463 | cmd_env["GIT_PROTOCOL"] = 'version=%s' % version |
104 | 491 | subcmd = b'upload-pack' | 464 | if version == b'2': |
86 | 492 | elif command == b'git-receive-pack': | ||
87 | 493 | subcmd = b'receive-pack' | ||
88 | 494 | write_operation = True | ||
89 | 495 | else: | ||
90 | 496 | self.die(b'Unsupported command in request') | ||
91 | 497 | return | ||
92 | 498 | else: | ||
93 | 499 | v2_command = params.get(b'command') | ||
94 | 500 | if command == b'git-upload-pack' and not v2_command: | ||
95 | 501 | self.expectNextCommand() | ||
96 | 502 | self.transport.loseConnection() | ||
97 | 503 | return | ||
98 | 504 | subcmd = b'upload-pack' | ||
99 | 505 | cmd_env["GIT_PROTOCOL"] = 'version=2' | ||
100 | 506 | send_path_as_option = True | ||
101 | 507 | # Do not include "advertise-refs" parameter. | ||
105 | 508 | params.pop(b'turnip-advertise-refs', None) | 465 | params.pop(b'turnip-advertise-refs', None) |
107 | 509 | cmd_input = self.getV2CommandInput(params) | 466 | if command == b'git-upload-pack': |
108 | 467 | subcmd = b'upload-pack' | ||
109 | 468 | elif command == b'git-receive-pack': | ||
110 | 469 | subcmd = b'receive-pack' | ||
111 | 470 | write_operation = True | ||
112 | 471 | else: | ||
113 | 472 | self.die(b'Unsupported command in request') | ||
114 | 473 | return | ||
115 | 510 | 474 | ||
116 | 511 | args = [] | 475 | args = [] |
117 | 512 | if params.pop(b'turnip-stateless-rpc', None): | 476 | if params.pop(b'turnip-stateless-rpc', None): |
118 | @@ -516,14 +480,12 @@ class PackBackendProtocol(PackServerProtocol): | |||
119 | 516 | args.append(self.path) | 480 | args.append(self.path) |
120 | 517 | self.spawnGit( | 481 | self.spawnGit( |
121 | 518 | subcmd, args, | 482 | subcmd, args, |
126 | 519 | write_operation=write_operation, | 483 | write_operation=write_operation, auth_params=auth_params, |
127 | 520 | auth_params=auth_params, | 484 | cmd_env=cmd_env) |
124 | 521 | send_path_as_option=send_path_as_option, | ||
125 | 522 | cmd_env=cmd_env, cmd_input=cmd_input) | ||
128 | 523 | 485 | ||
129 | 524 | def spawnGit(self, subcmd, extra_args, write_operation=False, | 486 | def spawnGit(self, subcmd, extra_args, write_operation=False, |
130 | 525 | send_path_as_option=False, auth_params=None, | 487 | send_path_as_option=False, auth_params=None, |
132 | 526 | cmd_env=None, cmd_input=None): | 488 | cmd_env=None): |
133 | 527 | cmd = b'git' | 489 | cmd = b'git' |
134 | 528 | args = [b'git'] | 490 | args = [b'git'] |
135 | 529 | if send_path_as_option: | 491 | if send_path_as_option: |
136 | @@ -545,7 +507,7 @@ class PackBackendProtocol(PackServerProtocol): | |||
137 | 545 | env[b'TURNIP_HOOK_RPC_KEY'] = self.hookrpc_key | 507 | env[b'TURNIP_HOOK_RPC_KEY'] = self.hookrpc_key |
138 | 546 | 508 | ||
139 | 547 | self.log.info('Spawning {args}', args=args) | 509 | self.log.info('Spawning {args}', args=args) |
141 | 548 | self.peer = GitProcessProtocol(self, cmd_input) | 510 | self.peer = GitProcessProtocol(self) |
142 | 549 | self.spawnProcess(cmd, args, env=env) | 511 | self.spawnProcess(cmd, args, env=env) |
143 | 550 | 512 | ||
144 | 551 | def spawnProcess(self, cmd, args, env=None): | 513 | def spawnProcess(self, cmd, args, env=None): |
145 | @@ -674,7 +636,7 @@ class PackVirtServerProtocol(PackProxyServerProtocol): | |||
146 | 674 | @defer.inlineCallbacks | 636 | @defer.inlineCallbacks |
147 | 675 | def requestReceived(self, command, pathname, params): | 637 | def requestReceived(self, command, pathname, params): |
148 | 676 | self.extractRequestMeta(command, pathname, params) | 638 | self.extractRequestMeta(command, pathname, params) |
150 | 677 | permission = 'read' if command == b'git-upload-pack' else 'write' | 639 | permission = 'read' if command != b'git-receive-pack' else 'write' |
151 | 678 | proxy = xmlrpc.Proxy(self.factory.virtinfo_endpoint, allowNone=True) | 640 | proxy = xmlrpc.Proxy(self.factory.virtinfo_endpoint, allowNone=True) |
152 | 679 | try: | 641 | try: |
153 | 680 | auth_params = self.createAuthParams(params) | 642 | auth_params = self.createAuthParams(params) |
154 | diff --git a/turnip/pack/helpers.py b/turnip/pack/helpers.py | |||
155 | index 93aba33..5d78d68 100644 | |||
156 | --- a/turnip/pack/helpers.py | |||
157 | +++ b/turnip/pack/helpers.py | |||
158 | @@ -24,10 +24,10 @@ import six | |||
159 | 24 | import yaml | 24 | import yaml |
160 | 25 | 25 | ||
161 | 26 | import turnip.pack.hooks | 26 | import turnip.pack.hooks |
163 | 27 | 27 | from turnip.version_info import version_info | |
164 | 28 | 28 | ||
165 | 29 | FLUSH_PKT = b'0000' | 29 | FLUSH_PKT = b'0000' |
167 | 30 | DELIM_PKT = b'0001' | 30 | DELIM_PKT = object() |
168 | 31 | PKT_LEN_SIZE = 4 | 31 | PKT_LEN_SIZE = 4 |
169 | 32 | PKT_PAYLOAD_MAX = 65520 | 32 | PKT_PAYLOAD_MAX = 65520 |
170 | 33 | INCOMPLETE_PKT = object() | 33 | INCOMPLETE_PKT = object() |
171 | @@ -37,6 +37,8 @@ def encode_packet(payload): | |||
172 | 37 | if payload is None: | 37 | if payload is None: |
173 | 38 | # flush-pkt. | 38 | # flush-pkt. |
174 | 39 | return FLUSH_PKT | 39 | return FLUSH_PKT |
175 | 40 | if payload is DELIM_PKT: | ||
176 | 41 | return b'0001' | ||
177 | 40 | else: | 42 | else: |
178 | 41 | # data-pkt | 43 | # data-pkt |
179 | 42 | if len(payload) > PKT_PAYLOAD_MAX: | 44 | if len(payload) > PKT_PAYLOAD_MAX: |
180 | @@ -48,63 +50,25 @@ def encode_packet(payload): | |||
181 | 48 | 50 | ||
182 | 49 | def decode_packet(input): | 51 | def decode_packet(input): |
183 | 50 | """Consume a packet, returning the payload and any unconsumed tail.""" | 52 | """Consume a packet, returning the payload and any unconsumed tail.""" |
184 | 53 | if input.startswith(b'0001'): | ||
185 | 54 | return (DELIM_PKT, input[PKT_LEN_SIZE:]) | ||
186 | 51 | if len(input) < PKT_LEN_SIZE: | 55 | if len(input) < PKT_LEN_SIZE: |
187 | 52 | return (INCOMPLETE_PKT, input) | 56 | return (INCOMPLETE_PKT, input) |
188 | 53 | if input.startswith(FLUSH_PKT): | 57 | if input.startswith(FLUSH_PKT): |
189 | 54 | # flush-pkt | 58 | # flush-pkt |
190 | 55 | return (None, input[PKT_LEN_SIZE:]) | 59 | return (None, input[PKT_LEN_SIZE:]) |
243 | 56 | # data-pkt | 60 | else: |
244 | 57 | try: | 61 | # data-pkt |
245 | 58 | pkt_len = int(input[:PKT_LEN_SIZE], 16) | 62 | try: |
246 | 59 | except ValueError: | 63 | pkt_len = int(input[:PKT_LEN_SIZE], 16) |
247 | 60 | pkt_len = 0 | 64 | except ValueError: |
248 | 61 | if not (PKT_LEN_SIZE <= pkt_len <= (PKT_LEN_SIZE + PKT_PAYLOAD_MAX)): | 65 | pkt_len = 0 |
249 | 62 | raise ValueError("Invalid pkt-len") | 66 | if not (PKT_LEN_SIZE <= pkt_len <= (PKT_LEN_SIZE + PKT_PAYLOAD_MAX)): |
250 | 63 | if len(input) < pkt_len: | 67 | raise ValueError("Invalid pkt-len") |
251 | 64 | # Some of the packet is yet to be received. | 68 | if len(input) < pkt_len: |
252 | 65 | return (INCOMPLETE_PKT, input) | 69 | # Some of the packet is yet to be received. |
253 | 66 | # v2 protocol "hides" extra parameters after the end of the packet. | 70 | return (INCOMPLETE_PKT, input) |
254 | 67 | if len(input) > pkt_len and b'version=2\x00' in input: | 71 | return (input[PKT_LEN_SIZE:pkt_len], input[pkt_len:]) |
203 | 68 | if FLUSH_PKT not in input: | ||
204 | 69 | return INCOMPLETE_PKT, input | ||
205 | 70 | end = input.index(FLUSH_PKT) | ||
206 | 71 | return input[PKT_LEN_SIZE:end], input[end + len(FLUSH_PKT):] | ||
207 | 72 | return (input[PKT_LEN_SIZE:pkt_len], input[pkt_len:]) | ||
208 | 73 | |||
209 | 74 | |||
210 | 75 | def decode_packet_list(data): | ||
211 | 76 | remaining = data | ||
212 | 77 | retval = [] | ||
213 | 78 | while remaining: | ||
214 | 79 | pkt, remaining = decode_packet(remaining) | ||
215 | 80 | retval.append(pkt) | ||
216 | 81 | return retval | ||
217 | 82 | |||
218 | 83 | |||
219 | 84 | def decode_protocol_v2_params(data): | ||
220 | 85 | """Parse the protocol v2 extra parameters hidden behind the end of v1 | ||
221 | 86 | protocol. | ||
222 | 87 | |||
223 | 88 | :return: An ordered dict with parsed v2 parameters. | ||
224 | 89 | """ | ||
225 | 90 | params = OrderedDict() | ||
226 | 91 | cmd, remaining = decode_packet(data) | ||
227 | 92 | cmd = cmd.split(b'=', 1)[-1].strip() | ||
228 | 93 | capabilities, args = remaining.split(DELIM_PKT) | ||
229 | 94 | params[b"command"] = cmd | ||
230 | 95 | params[b"capabilities"] = decode_packet_list(capabilities) | ||
231 | 96 | for arg in decode_packet_list(args): | ||
232 | 97 | if arg is None: | ||
233 | 98 | continue | ||
234 | 99 | arg = arg.strip('\n') | ||
235 | 100 | if b' ' in arg: | ||
236 | 101 | k, v = arg.split(b' ', 1) | ||
237 | 102 | if k not in params: | ||
238 | 103 | params[k] = [] | ||
239 | 104 | params[k].append(v) | ||
240 | 105 | else: | ||
241 | 106 | params[arg] = b"" | ||
242 | 107 | return params | ||
255 | 108 | 72 | ||
256 | 109 | 73 | ||
257 | 110 | def decode_request(data): | 74 | def decode_request(data): |
258 | @@ -121,7 +85,7 @@ def decode_request(data): | |||
259 | 121 | # Following the command is a pathname, then any number of named | 85 | # Following the command is a pathname, then any number of named |
260 | 122 | # parameters. Each of these is NUL-terminated. | 86 | # parameters. Each of these is NUL-terminated. |
261 | 123 | # After that, v1 should end (v2 might have extra commands). | 87 | # After that, v1 should end (v2 might have extra commands). |
263 | 124 | if len(bits) < 2 or (b'version=2' not in bits and bits[-1] != b''): | 88 | if len(bits) < 2 or bits[-1] != b'': |
264 | 125 | raise ValueError('Invalid git-proto-request') | 89 | raise ValueError('Invalid git-proto-request') |
265 | 126 | pathname = bits[0] | 90 | pathname = bits[0] |
266 | 127 | params = OrderedDict() | 91 | params = OrderedDict() |
267 | @@ -139,12 +103,6 @@ def decode_request(data): | |||
268 | 139 | raise ValueError('Parameters must not be repeated') | 103 | raise ValueError('Parameters must not be repeated') |
269 | 140 | params[name] = value | 104 | params[name] = value |
270 | 141 | 105 | ||
271 | 142 | # If there are remaining bits at the end, we must be dealing with v2 | ||
272 | 143 | # protocol. So, we append v2 parameters at the end of original parameters. | ||
273 | 144 | if bits[-1]: | ||
274 | 145 | for k, v in decode_protocol_v2_params(bits[-1]).items(): | ||
275 | 146 | params[k] = v | ||
276 | 147 | |||
277 | 148 | return command, pathname, params | 106 | return command, pathname, params |
278 | 149 | 107 | ||
279 | 150 | 108 | ||
280 | @@ -297,5 +255,14 @@ def get_capabilities_advertisement(version='1'): | |||
281 | 297 | 255 | ||
282 | 298 | If no binary data is sent, no advertisement is done and we declare to | 256 | If no binary data is sent, no advertisement is done and we declare to |
283 | 299 | not be compatible with that specific version.""" | 257 | not be compatible with that specific version.""" |
286 | 300 | # XXX pappacena 2020-08-11: Return the correct data for protocol v2. | 258 | if version != '2': |
287 | 301 | return b"" | 259 | return b"" |
288 | 260 | turnip_version = six.ensure_binary(version_info.get("revision_id", '-1')) | ||
289 | 261 | return ( | ||
290 | 262 | encode_packet(b"version 2\n") + | ||
291 | 263 | encode_packet(b"agent=turnip/%s\n" % turnip_version) + | ||
292 | 264 | encode_packet(b"ls-refs\n") + | ||
293 | 265 | encode_packet(b"fetch=shallow\n") + | ||
294 | 266 | encode_packet(b"server-option\n") + | ||
295 | 267 | FLUSH_PKT | ||
296 | 268 | ) | ||
297 | diff --git a/turnip/pack/http.py b/turnip/pack/http.py | |||
298 | index c0becf0..9e0661c 100644 | |||
299 | --- a/turnip/pack/http.py | |||
300 | +++ b/turnip/pack/http.py | |||
301 | @@ -29,6 +29,7 @@ from paste.auth.cookie import ( | |||
302 | 29 | decode as decode_cookie, | 29 | decode as decode_cookie, |
303 | 30 | encode as encode_cookie, | 30 | encode as encode_cookie, |
304 | 31 | ) | 31 | ) |
305 | 32 | import six | ||
306 | 32 | from twisted.internet import ( | 33 | from twisted.internet import ( |
307 | 33 | defer, | 34 | defer, |
308 | 34 | error, | 35 | error, |
309 | @@ -58,8 +59,8 @@ from turnip.pack.helpers import ( | |||
310 | 58 | encode_packet, | 59 | encode_packet, |
311 | 59 | encode_request, | 60 | encode_request, |
312 | 60 | translate_xmlrpc_fault, | 61 | translate_xmlrpc_fault, |
315 | 61 | TurnipFaultCode, | 62 | TurnipFaultCode, get_capabilities_advertisement, |
316 | 62 | ) | 63 | ) |
317 | 63 | try: | 64 | try: |
318 | 64 | from turnip.version_info import version_info | 65 | from turnip.version_info import version_info |
319 | 65 | except ImportError: | 66 | except ImportError: |
320 | @@ -80,6 +81,16 @@ def fail_request(request, message, code=http.INTERNAL_SERVER_ERROR): | |||
321 | 80 | return b'' | 81 | return b'' |
322 | 81 | 82 | ||
323 | 82 | 83 | ||
324 | 84 | def get_protocol_version_from_request(request): | ||
325 | 85 | version_header = request.requestHeaders.getRawHeaders( | ||
326 | 86 | 'git-protocol', ['version=1'])[0] | ||
327 | 87 | try: | ||
328 | 88 | return version_header.split('version=', 1)[1] | ||
329 | 89 | except IndexError: | ||
330 | 90 | pass | ||
331 | 91 | return 1 | ||
332 | 92 | |||
333 | 93 | |||
334 | 83 | class HTTPPackClientProtocol(PackProtocol): | 94 | class HTTPPackClientProtocol(PackProtocol): |
335 | 84 | """Abstract bridge between a Git pack connection and a smart HTTP request. | 95 | """Abstract bridge between a Git pack connection and a smart HTTP request. |
336 | 85 | 96 | ||
337 | @@ -99,7 +110,11 @@ class HTTPPackClientProtocol(PackProtocol): | |||
338 | 99 | 110 | ||
339 | 100 | def startGoodResponse(self): | 111 | def startGoodResponse(self): |
340 | 101 | """Prepare the HTTP response for forwarding from the backend.""" | 112 | """Prepare the HTTP response for forwarding from the backend.""" |
342 | 102 | raise NotImplementedError() | 113 | self.factory.http_request.write( |
343 | 114 | get_capabilities_advertisement(self.getProtocolVersion())) | ||
344 | 115 | |||
345 | 116 | def getProtocolVersion(self): | ||
346 | 117 | return get_protocol_version_from_request(self.factory.http_request) | ||
347 | 103 | 118 | ||
348 | 104 | def backendConnected(self): | 119 | def backendConnected(self): |
349 | 105 | """Called when the backend is connected and has sent a good packet.""" | 120 | """Called when the backend is connected and has sent a good packet.""" |
350 | @@ -209,6 +224,7 @@ class HTTPPackClientRefsProtocol(HTTPPackClientProtocol): | |||
351 | 209 | self.factory.http_request.setHeader( | 224 | self.factory.http_request.setHeader( |
352 | 210 | b'Content-Type', | 225 | b'Content-Type', |
353 | 211 | b'application/x-%s-advertisement' % self.factory.command) | 226 | b'application/x-%s-advertisement' % self.factory.command) |
354 | 227 | super(HTTPPackClientRefsProtocol, self).startGoodResponse() | ||
355 | 212 | 228 | ||
356 | 213 | def backendConnected(self): | 229 | def backendConnected(self): |
357 | 214 | HTTPPackClientProtocol.backendConnected(self) | 230 | HTTPPackClientProtocol.backendConnected(self) |
358 | @@ -221,10 +237,11 @@ class HTTPPackClientCommandProtocol(HTTPPackClientProtocol): | |||
359 | 221 | 237 | ||
360 | 222 | def startGoodResponse(self): | 238 | def startGoodResponse(self): |
361 | 223 | """Prepare the HTTP response for forwarding from the backend.""" | 239 | """Prepare the HTTP response for forwarding from the backend.""" |
366 | 224 | self.factory.http_request.setResponseCode(http.OK) | 240 | if self.getProtocolVersion() != b'2': |
367 | 225 | self.factory.http_request.setHeader( | 241 | self.factory.http_request.setResponseCode(http.OK) |
368 | 226 | b'Content-Type', | 242 | self.factory.http_request.setHeader( |
369 | 227 | b'application/x-%s-result' % self.factory.command) | 243 | b'Content-Type', |
370 | 244 | b'application/x-%s-result' % self.factory.command) | ||
371 | 228 | 245 | ||
372 | 229 | 246 | ||
373 | 230 | class HTTPPackClientFactory(protocol.ClientFactory): | 247 | class HTTPPackClientFactory(protocol.ClientFactory): |
374 | @@ -280,8 +297,11 @@ class BaseSmartHTTPResource(resource.Resource): | |||
375 | 280 | by the virt service, if any. | 297 | by the virt service, if any. |
376 | 281 | """ | 298 | """ |
377 | 282 | params = { | 299 | params = { |
378 | 300 | b'turnip-frontend': b'http', | ||
379 | 283 | b'turnip-can-authenticate': b'yes', | 301 | b'turnip-can-authenticate': b'yes', |
380 | 284 | b'turnip-request-id': str(uuid.uuid4()), | 302 | b'turnip-request-id': str(uuid.uuid4()), |
381 | 303 | b'version': six.ensure_binary( | ||
382 | 304 | get_protocol_version_from_request(request)) | ||
383 | 285 | } | 305 | } |
384 | 286 | authenticated_params = yield self.authenticateUser(request) | 306 | authenticated_params = yield self.authenticateUser(request) |
385 | 287 | for key, value in authenticated_params.items(): | 307 | for key, value in authenticated_params.items(): |
386 | diff --git a/turnip/pack/tests/test_functional.py b/turnip/pack/tests/test_functional.py | |||
387 | index 27bc26e..c811b2a 100644 | |||
388 | --- a/turnip/pack/tests/test_functional.py | |||
389 | +++ b/turnip/pack/tests/test_functional.py | |||
390 | @@ -815,16 +815,19 @@ class TestSmartHTTPFrontendWithAuthFunctional(TestSmartHTTPFrontendFunctional): | |||
391 | 815 | test_root = self.useFixture(TempDir()).path | 815 | test_root = self.useFixture(TempDir()).path |
392 | 816 | clone = os.path.join(test_root, 'clone') | 816 | clone = os.path.join(test_root, 'clone') |
393 | 817 | yield self.assertCommandSuccess((b'git', b'clone', self.ro_url, clone)) | 817 | yield self.assertCommandSuccess((b'git', b'clone', self.ro_url, clone)) |
394 | 818 | expected_requests = 1 if self.protocol_version == '1' else 2 | ||
395 | 818 | self.assertEqual( | 819 | self.assertEqual( |
399 | 819 | [(b'test-user', b'test-password')], self.virtinfo.authentications) | 820 | [(b'test-user', b'test-password')] * expected_requests, |
400 | 820 | self.assertThat(self.virtinfo.translations, MatchesListwise([ | 821 | self.virtinfo.authentications) |
401 | 821 | MatchesListwise([ | 822 | self.assertEqual(expected_requests, len(self.virtinfo.translations)) |
402 | 823 | for translation in self.virtinfo.translations: | ||
403 | 824 | self.assertThat(translation, MatchesListwise([ | ||
404 | 822 | Equals(b'/test'), Equals(b'read'), | 825 | Equals(b'/test'), Equals(b'read'), |
405 | 823 | MatchesDict({ | 826 | MatchesDict({ |
406 | 824 | b'can-authenticate': Is(True), | 827 | b'can-authenticate': Is(True), |
407 | 825 | b'request-id': Not(Is(None)), | 828 | b'request-id': Not(Is(None)), |
410 | 826 | b'user': Equals(b'test-user'), | 829 | b'user': Equals(b'test-user')}) |
411 | 827 | })])])) | 830 | ])) |
412 | 828 | 831 | ||
413 | 829 | @defer.inlineCallbacks | 832 | @defer.inlineCallbacks |
414 | 830 | def test_authenticated_push(self): | 833 | def test_authenticated_push(self): |
415 | diff --git a/turnip/pack/tests/test_git.py b/turnip/pack/tests/test_git.py | |||
416 | index b69049c..d1a8162 100644 | |||
417 | --- a/turnip/pack/tests/test_git.py | |||
418 | +++ b/turnip/pack/tests/test_git.py | |||
419 | @@ -9,7 +9,6 @@ from __future__ import ( | |||
420 | 9 | 9 | ||
421 | 10 | import hashlib | 10 | import hashlib |
422 | 11 | import os.path | 11 | import os.path |
423 | 12 | from collections import OrderedDict | ||
424 | 13 | 12 | ||
425 | 14 | from fixtures import TempDir, MonkeyPatch | 13 | from fixtures import TempDir, MonkeyPatch |
426 | 15 | from pygit2 import init_repository | 14 | from pygit2 import init_repository |
427 | @@ -35,7 +34,6 @@ from turnip.pack import ( | |||
428 | 35 | git, | 34 | git, |
429 | 36 | helpers, | 35 | helpers, |
430 | 37 | ) | 36 | ) |
431 | 38 | from turnip.pack.git import GitProcessProtocol | ||
432 | 39 | from turnip.pack.tests.fake_servers import FakeVirtInfoService | 37 | from turnip.pack.tests.fake_servers import FakeVirtInfoService |
433 | 40 | from turnip.pack.tests.test_hooks import MockHookRPCHandler | 38 | from turnip.pack.tests.test_hooks import MockHookRPCHandler |
434 | 41 | from turnip.tests.compat import mock | 39 | from turnip.tests.compat import mock |
435 | @@ -51,18 +49,6 @@ class DummyPackServerProtocol(git.PackServerProtocol): | |||
436 | 51 | self.test_request = (command, pathname, host) | 49 | self.test_request = (command, pathname, host) |
437 | 52 | 50 | ||
438 | 53 | 51 | ||
439 | 54 | class TestGitProcessProtocol(TestCase): | ||
440 | 55 | def test_can_write_to_stdin_directly(self): | ||
441 | 56 | peer = mock.Mock() | ||
442 | 57 | transport = mock.Mock() | ||
443 | 58 | protocol = GitProcessProtocol(peer, b"this is the stdin") | ||
444 | 59 | protocol.transport = transport | ||
445 | 60 | protocol.connectionMade() | ||
446 | 61 | self.assertEqual( | ||
447 | 62 | [mock.call(b'this is the stdin', )], | ||
448 | 63 | transport.write.call_args_list) | ||
449 | 64 | |||
450 | 65 | |||
451 | 66 | class TestPackServerProtocol(TestCase): | 52 | class TestPackServerProtocol(TestCase): |
452 | 67 | """Test the base implementation of the git pack network protocol.""" | 53 | """Test the base implementation of the git pack network protocol.""" |
453 | 68 | 54 | ||
454 | @@ -243,8 +229,9 @@ class TestPackBackendProtocol(TestCase): | |||
455 | 243 | [('foo.git', )], self.virtinfo.confirm_repo_creation_call_args) | 229 | [('foo.git', )], self.virtinfo.confirm_repo_creation_call_args) |
456 | 244 | 230 | ||
457 | 245 | self.assertEqual( | 231 | self.assertEqual( |
460 | 246 | (b'git', [b'git', b'upload-pack', full_path], {}), | 232 | (b'git', [b'git', b'upload-pack', full_path], { |
461 | 247 | self.proto.test_process) | 233 | 'GIT_PROTOCOL': 'version=0' |
462 | 234 | }), self.proto.test_process) | ||
463 | 248 | 235 | ||
464 | 249 | @defer.inlineCallbacks | 236 | @defer.inlineCallbacks |
465 | 250 | def test_create_repo_fails_to_confirm(self): | 237 | def test_create_repo_fails_to_confirm(self): |
466 | @@ -281,47 +268,8 @@ class TestPackBackendProtocol(TestCase): | |||
467 | 281 | self.assertEqual( | 268 | self.assertEqual( |
468 | 282 | (b'git', | 269 | (b'git', |
469 | 283 | [b'git', b'upload-pack', full_path], | 270 | [b'git', b'upload-pack', full_path], |
498 | 284 | {}), | 271 | {'GIT_PROTOCOL': 'version=0'}), |
471 | 285 | self.proto.test_process) | ||
472 | 286 | |||
473 | 287 | def test_git_upload_pack_v2_calls_spawnProcess(self): | ||
474 | 288 | # If the command is git-upload-pack using v2 protocol, requestReceived | ||
475 | 289 | # calls spawnProcess with appropriate arguments. | ||
476 | 290 | advertise_capabilities = mock.Mock() | ||
477 | 291 | self.useFixture( | ||
478 | 292 | MonkeyPatch("turnip.pack.git.get_capabilities_advertisement", | ||
479 | 293 | advertise_capabilities)) | ||
480 | 294 | advertise_capabilities.return_value = b'fake capability' | ||
481 | 295 | |||
482 | 296 | self.proto.requestReceived( | ||
483 | 297 | b'git-upload-pack', b'/foo.git', OrderedDict([ | ||
484 | 298 | (b'turnip-x', b'yes'), | ||
485 | 299 | (b'turnip-request-id', b'123'), | ||
486 | 300 | (b'version', b'2'), | ||
487 | 301 | (b'command', b'ls-refs'), | ||
488 | 302 | (b'capabilities', b'agent=git/2.25.1'), | ||
489 | 303 | (b'peel', b''), | ||
490 | 304 | (b'symrefs', b''), | ||
491 | 305 | (b'ref-prefix', b'HEAD\nrefs/heads/\nrefs/tags/') | ||
492 | 306 | ])) | ||
493 | 307 | full_path = os.path.join(six.ensure_binary(self.root), b'foo.git') | ||
494 | 308 | self.assertEqual( | ||
495 | 309 | (b'git', | ||
496 | 310 | [b'git', b'-C', full_path, b'upload-pack', full_path], | ||
497 | 311 | {'GIT_PROTOCOL': 'version=2'}), | ||
499 | 312 | self.proto.test_process) | 272 | self.proto.test_process) |
500 | 313 | stdin_content = ( | ||
501 | 314 | b'0014command=ls-refs\n' | ||
502 | 315 | b'0015agent=git/2.25.1\n' | ||
503 | 316 | b'00010014command ls-refs\n' | ||
504 | 317 | b'0009peel\n' | ||
505 | 318 | b'000csymrefs\n' | ||
506 | 319 | b'0014ref-prefix HEAD\n' | ||
507 | 320 | b'001bref-prefix refs/heads/\n' | ||
508 | 321 | b'001aref-prefix refs/tags/\n' | ||
509 | 322 | b'0000' | ||
510 | 323 | ) | ||
511 | 324 | self.assertEqual(stdin_content, self.proto.peer.cmd_input) | ||
512 | 325 | 273 | ||
513 | 326 | def test_git_receive_pack_calls_spawnProcess(self): | 274 | def test_git_receive_pack_calls_spawnProcess(self): |
514 | 327 | # If the command is git-receive-pack, requestReceived calls | 275 | # If the command is git-receive-pack, requestReceived calls |
515 | diff --git a/turnip/pack/tests/test_helpers.py b/turnip/pack/tests/test_helpers.py | |||
516 | index bad7117..a5ba913 100644 | |||
517 | --- a/turnip/pack/tests/test_helpers.py | |||
518 | +++ b/turnip/pack/tests/test_helpers.py | |||
519 | @@ -7,25 +7,32 @@ from __future__ import ( | |||
520 | 7 | unicode_literals, | 7 | unicode_literals, |
521 | 8 | ) | 8 | ) |
522 | 9 | 9 | ||
523 | 10 | from collections import OrderedDict | ||
524 | 10 | import os.path | 11 | import os.path |
525 | 11 | import re | 12 | import re |
532 | 12 | from collections import OrderedDict | 13 | import shutil |
533 | 13 | 14 | import subprocess | |
534 | 14 | import stat | 15 | import tempfile |
529 | 15 | import sys | ||
530 | 16 | from textwrap import dedent | ||
531 | 17 | import time | ||
535 | 18 | 16 | ||
536 | 19 | from fixtures import TempDir | 17 | from fixtures import TempDir |
537 | 20 | from pygit2 import ( | 18 | from pygit2 import ( |
538 | 21 | Config, | 19 | Config, |
539 | 22 | init_repository, | 20 | init_repository, |
540 | 23 | ) | 21 | ) |
541 | 22 | import six | ||
542 | 23 | import stat | ||
543 | 24 | import sys | ||
544 | 24 | from testtools import TestCase | 25 | from testtools import TestCase |
545 | 26 | from textwrap import dedent | ||
546 | 27 | import time | ||
547 | 25 | 28 | ||
548 | 26 | from turnip.pack import helpers | 29 | from turnip.pack import helpers |
549 | 27 | import turnip.pack.hooks | 30 | import turnip.pack.hooks |
551 | 28 | from turnip.pack.helpers import FLUSH_PKT | 31 | from turnip.pack.helpers import ( |
552 | 32 | get_capabilities_advertisement, | ||
553 | 33 | encode_packet, | ||
554 | 34 | ) | ||
555 | 35 | from turnip.version_info import version_info | ||
556 | 29 | 36 | ||
557 | 30 | TEST_DATA = b'0123456789abcdef' | 37 | TEST_DATA = b'0123456789abcdef' |
558 | 31 | TEST_PKT = b'00140123456789abcdef' | 38 | TEST_PKT = b'00140123456789abcdef' |
559 | @@ -175,27 +182,6 @@ class TestDecodeRequest(TestCase): | |||
560 | 175 | b'git-do-stuff /foo\0host=foo\0host=bar\0', | 182 | b'git-do-stuff /foo\0host=foo\0host=bar\0', |
561 | 176 | b'Parameters must not be repeated') | 183 | b'Parameters must not be repeated') |
562 | 177 | 184 | ||
563 | 178 | def test_v2_extra_commands(self): | ||
564 | 179 | data = ( | ||
565 | 180 | b"git-upload-pack b306d" + | ||
566 | 181 | b"\x00turnip-x=yes\x00turnip-request-id=123\x00" + | ||
567 | 182 | b"version=2\x000014command=ls-refs\n0014agent=git/2.25.1" + | ||
568 | 183 | b'00010009peel\n000csymrefs\n0014ref-prefix HEAD\n' + | ||
569 | 184 | b'001bref-prefix refs/heads/\n' | ||
570 | 185 | b'001aref-prefix refs/tags/\n0000' + | ||
571 | 186 | FLUSH_PKT) | ||
572 | 187 | decoded = helpers.decode_request(data) | ||
573 | 188 | self.assertEqual((b'git-upload-pack', b'b306d', OrderedDict([ | ||
574 | 189 | (b'turnip-x', b'yes'), | ||
575 | 190 | (b'turnip-request-id', b'123'), | ||
576 | 191 | (b'version', b'2'), | ||
577 | 192 | (b'command', b'ls-refs'), | ||
578 | 193 | (b'capabilities', [b'agent=git/2.25.1']), | ||
579 | 194 | (b'peel', b''), | ||
580 | 195 | (b'symrefs', b''), | ||
581 | 196 | (b'ref-prefix', [b'HEAD', b'refs/heads/', b'refs/tags/']) | ||
582 | 197 | ])), decoded) | ||
583 | 198 | |||
584 | 199 | 185 | ||
585 | 200 | class TestEncodeRequest(TestCase): | 186 | class TestEncodeRequest(TestCase): |
586 | 201 | """Test git-proto-request encoding.""" | 187 | """Test git-proto-request encoding.""" |
587 | @@ -344,3 +330,30 @@ class TestEnsureHooks(TestCase): | |||
588 | 344 | self.assertEqual(expected_bytes, actual.read()) | 330 | self.assertEqual(expected_bytes, actual.read()) |
589 | 345 | # The hook is executable. | 331 | # The hook is executable. |
590 | 346 | self.assertTrue(os.stat(self.hook('hook.py')).st_mode & stat.S_IXUSR) | 332 | self.assertTrue(os.stat(self.hook('hook.py')).st_mode & stat.S_IXUSR) |
591 | 333 | |||
592 | 334 | |||
593 | 335 | class TestCapabilityAdvertisement(TestCase): | ||
594 | 336 | def test_returning_same_output_as_git_command(self): | ||
595 | 337 | """Make sure that our hard-coded feature advertisement matches what | ||
596 | 338 | our git command advertises.""" | ||
597 | 339 | root = tempfile.mkdtemp(prefix=b'turnip-test-root-') | ||
598 | 340 | self.addCleanup(shutil.rmtree, root, ignore_errors=True) | ||
599 | 341 | # Create a dummy repository | ||
600 | 342 | subprocess.call(['git', 'init', root]) | ||
601 | 343 | |||
602 | 344 | git_version = subprocess.check_output(['git', '--version']) | ||
603 | 345 | git_version_num = six.ensure_binary(git_version.split(' ')[-1].strip()) | ||
604 | 346 | git_agent = encode_packet(b"agent=git/%s\n" % git_version_num) | ||
605 | 347 | |||
606 | 348 | proc = subprocess.Popen( | ||
607 | 349 | ['git', 'upload-pack', root], env={"GIT_PROTOCOL": "version=2"}, | ||
608 | 350 | stdout=subprocess.PIPE, stdin=subprocess.PIPE) | ||
609 | 351 | git_advertised_capabilities, _ = proc.communicate() | ||
610 | 352 | |||
611 | 353 | turnip_capabilities = get_capabilities_advertisement(version=b'2') | ||
612 | 354 | turnip_agent = encode_packet( | ||
613 | 355 | b"agent=turnip/%s\n" % version_info["revision_id"]) | ||
614 | 356 | |||
615 | 357 | self.assertEqual( | ||
616 | 358 | turnip_capabilities, | ||
617 | 359 | git_advertised_capabilities.replace(git_agent, turnip_agent)) | ||
618 | diff --git a/turnip/pack/tests/test_http.py b/turnip/pack/tests/test_http.py | |||
619 | index 26e2754..9c547c1 100644 | |||
620 | --- a/turnip/pack/tests/test_http.py | |||
621 | +++ b/turnip/pack/tests/test_http.py | |||
622 | @@ -24,7 +24,10 @@ from turnip.pack import ( | |||
623 | 24 | helpers, | 24 | helpers, |
624 | 25 | http, | 25 | http, |
625 | 26 | ) | 26 | ) |
626 | 27 | from turnip.pack.helpers import encode_packet | ||
627 | 27 | from turnip.pack.tests.fake_servers import FakeVirtInfoService | 28 | from turnip.pack.tests.fake_servers import FakeVirtInfoService |
628 | 29 | from turnip.tests.compat import mock | ||
629 | 30 | from turnip.version_info import version_info | ||
630 | 28 | 31 | ||
631 | 29 | 32 | ||
632 | 30 | class LessDummyRequest(requesthelper.DummyRequest): | 33 | class LessDummyRequest(requesthelper.DummyRequest): |
633 | @@ -74,12 +77,14 @@ class FakeRoot(object): | |||
634 | 74 | 77 | ||
635 | 75 | def __init__(self): | 78 | def __init__(self): |
636 | 76 | self.backend_transport = None | 79 | self.backend_transport = None |
637 | 80 | self.client_factory = None | ||
638 | 77 | self.backend_connected = defer.Deferred() | 81 | self.backend_connected = defer.Deferred() |
639 | 78 | 82 | ||
640 | 79 | def authenticateWithPassword(self, user, password): | 83 | def authenticateWithPassword(self, user, password): |
641 | 80 | return {} | 84 | return {} |
642 | 81 | 85 | ||
643 | 82 | def connectToBackend(self, client_factory): | 86 | def connectToBackend(self, client_factory): |
644 | 87 | self.client_factory = client_factory | ||
645 | 83 | self.backend_transport = testing.StringTransportWithDisconnection() | 88 | self.backend_transport = testing.StringTransportWithDisconnection() |
646 | 84 | p = client_factory.buildProtocol(None) | 89 | p = client_factory.buildProtocol(None) |
647 | 85 | self.backend_transport.protocol = p | 90 | self.backend_transport.protocol = p |
648 | @@ -186,6 +191,37 @@ class TestSmartHTTPRefsResource(ErrorTestMixin, TestCase): | |||
649 | 186 | 'And I am raw, since we got a good packet to start with.', | 191 | 'And I am raw, since we got a good packet to start with.', |
650 | 187 | self.request.value) | 192 | self.request.value) |
651 | 188 | 193 | ||
652 | 194 | @defer.inlineCallbacks | ||
653 | 195 | def test_good_v2_included_version_and_capabilities(self): | ||
654 | 196 | self.request.requestHeaders.addRawHeader("Git-Protocol", "version=2") | ||
655 | 197 | yield self.performRequest( | ||
656 | 198 | helpers.encode_packet(b'I am git protocol data.') + | ||
657 | 199 | b'And I am raw, since we got a good packet to start with.') | ||
658 | 200 | self.assertEqual(200, self.request.responseCode) | ||
659 | 201 | self.assertEqual(self.root.client_factory.params, { | ||
660 | 202 | 'version': '2', | ||
661 | 203 | 'turnip-frontend': 'http', | ||
662 | 204 | 'turnip-advertise-refs': 'yes', | ||
663 | 205 | 'turnip-can-authenticate': 'yes', | ||
664 | 206 | 'turnip-request-id': mock.ANY, | ||
665 | 207 | 'turnip-stateless-rpc': 'yes'}) | ||
666 | 208 | |||
667 | 209 | ver = version_info["revision_id"] | ||
668 | 210 | capabilities = ( | ||
669 | 211 | encode_packet(b'version 2\n') + | ||
670 | 212 | encode_packet(b'agent=turnip/%s\n' % ver) + | ||
671 | 213 | encode_packet(b'ls-refs\n') + | ||
672 | 214 | encode_packet(b'fetch=shallow\n') + | ||
673 | 215 | encode_packet(b'server-option\n') + | ||
674 | 216 | b'0000' | ||
675 | 217 | ) | ||
676 | 218 | self.assertEqual( | ||
677 | 219 | capabilities + | ||
678 | 220 | '001e# service=git-upload-pack\n' | ||
679 | 221 | '0000001bI am git protocol data.' | ||
680 | 222 | 'And I am raw, since we got a good packet to start with.', | ||
681 | 223 | self.request.value) | ||
682 | 224 | |||
683 | 189 | 225 | ||
684 | 190 | class TestSmartHTTPCommandResource(ErrorTestMixin, TestCase): | 226 | class TestSmartHTTPCommandResource(ErrorTestMixin, TestCase): |
685 | 191 | 227 |