Merge "Report to gerrit over HTTP"
This commit is contained in:
Zuul
Gerrit Code Review
commit
2e35440b43
|
|
@ -36,6 +36,7 @@ The supported options in ``zuul.conf`` connections are:
|
|||
The connection must set ``driver=gerrit`` for Gerrit connections.
|
||||
|
||||
.. attr:: server
|
||||
:required:
|
||||
|
||||
Fully qualified domain name of Gerrit server.
|
||||
|
||||
|
|
@ -58,8 +59,9 @@ The supported options in ``zuul.conf`` connections are:
|
|||
Gerrit server port.
|
||||
|
||||
.. attr:: baseurl
|
||||
:default: https://{server}
|
||||
|
||||
Path to Gerrit web interface.
|
||||
Path to Gerrit web interface. Omit the trailing ``/``.
|
||||
|
||||
.. attr:: gitweb_url_template
|
||||
:default: {baseurl}/gitweb?p={project.name}.git;a=commitdiff;h={sha}
|
||||
|
|
@ -87,6 +89,40 @@ The supported options in ``zuul.conf`` connections are:
|
|||
|
||||
SSH connection keepalive timeout; ``0`` disables.
|
||||
|
||||
.. attr:: password
|
||||
|
||||
The HTTP authentication password for the user. This is
|
||||
optional, but if it is provided, Zuul will report to Gerrit via
|
||||
HTTP rather than SSH. It is required in order for file and line
|
||||
comments to reported (the Gerrit SSH API only supports review
|
||||
messages). Retrieve this password from the ``HTTP Password``
|
||||
section of the ``Settings`` page in Gerrit.
|
||||
|
||||
.. attr:: auth_type
|
||||
:default: digest
|
||||
|
||||
The HTTP authentication mechanism.
|
||||
|
||||
.. value:: digest
|
||||
|
||||
HTTP Digest authentication; the default for most Gerrit
|
||||
installations.
|
||||
|
||||
.. value:: basic
|
||||
|
||||
HTTP Basic authentication.
|
||||
|
||||
.. value:: form
|
||||
|
||||
Zuul will submit a username and password to a form in order
|
||||
to authenticate.
|
||||
|
||||
.. attr:: verify_ssl
|
||||
:default: true
|
||||
|
||||
When using a self-signed certificate, this may be set to
|
||||
``false`` to disable SSL certificate verification.
|
||||
|
||||
Trigger Configuration
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
features:
|
||||
- The Gerrit driver can now (optionally) report via HTTP instead of
|
||||
SSH. In the future, this will be used to report file and line
|
||||
comments (the SSH API only supports review messages).
|
||||
104
tests/base.py
104
tests/base.py
|
|
@ -478,6 +478,82 @@ class FakeGerritChange(object):
|
|||
self.reported += 1
|
||||
|
||||
|
||||
class GerritWebServer(object):
|
||||
|
||||
def __init__(self, fake_gerrit):
|
||||
super(GerritWebServer, self).__init__()
|
||||
self.fake_gerrit = fake_gerrit
|
||||
|
||||
def start(self):
|
||||
fake_gerrit = self.fake_gerrit
|
||||
|
||||
class Server(http.server.SimpleHTTPRequestHandler):
|
||||
log = logging.getLogger("zuul.test.FakeGerritConnection")
|
||||
review_re = re.compile('/a/changes/(.*?)/revisions/(.*?)/review')
|
||||
submit_re = re.compile('/a/changes/(.*?)/submit')
|
||||
|
||||
def do_POST(self):
|
||||
path = self.path
|
||||
self.log.debug("Got POST %s", path)
|
||||
|
||||
data = self.rfile.read(int(self.headers['Content-Length']))
|
||||
data = json.loads(data.decode('utf-8'))
|
||||
self.log.debug("Got data %s", data)
|
||||
|
||||
m = self.review_re.match(path)
|
||||
if m:
|
||||
return self.review(m.group(1), m.group(2), data)
|
||||
m = self.submit_re.match(path)
|
||||
if m:
|
||||
return self.submit(m.group(1), data)
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
|
||||
def _404(self):
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def _get_change(self, change_id):
|
||||
for c in fake_gerrit.changes.values():
|
||||
if c.data['id'] == change_id:
|
||||
return c
|
||||
|
||||
def review(self, change_id, revision, data):
|
||||
change = self._get_change(change_id)
|
||||
if not change:
|
||||
return self._404()
|
||||
|
||||
message = data['message']
|
||||
action = data['labels']
|
||||
fake_gerrit._test_handle_review(
|
||||
int(change.data['number']), message, action)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
|
||||
def submit(self, change_id, data):
|
||||
change = self._get_change(change_id)
|
||||
if not change:
|
||||
return self._404()
|
||||
|
||||
message = None
|
||||
action = {'submit': True}
|
||||
fake_gerrit._test_handle_review(
|
||||
int(change.data['number']), message, action)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
|
||||
self.httpd = socketserver.ThreadingTCPServer(('', 0), Server)
|
||||
self.port = self.httpd.socket.getsockname()[1]
|
||||
self.thread = threading.Thread(name='GerritWebServer',
|
||||
target=self.httpd.serve_forever)
|
||||
self.thread.daemon = True
|
||||
self.thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.httpd.shutdown()
|
||||
self.thread.join()
|
||||
|
||||
|
||||
class FakeGerritConnection(gerritconnection.GerritConnection):
|
||||
"""A Fake Gerrit connection for use in tests.
|
||||
|
||||
|
|
@ -490,6 +566,15 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
|||
|
||||
def __init__(self, driver, connection_name, connection_config,
|
||||
changes_db=None, upstream_root=None):
|
||||
|
||||
if connection_config.get('password'):
|
||||
self.web_server = GerritWebServer(self)
|
||||
self.web_server.start()
|
||||
url = 'http://localhost:%s' % self.web_server.port
|
||||
connection_config['baseurl'] = url
|
||||
else:
|
||||
self.web_server = None
|
||||
|
||||
super(FakeGerritConnection, self).__init__(driver, connection_name,
|
||||
connection_config)
|
||||
|
||||
|
|
@ -570,9 +655,15 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
|||
}
|
||||
return event
|
||||
|
||||
def review(self, project, changeid, message, action):
|
||||
number, ps = changeid.split(',')
|
||||
change = self.changes[int(number)]
|
||||
def review(self, change, message, action):
|
||||
if self.web_server:
|
||||
return super(FakeGerritConnection, self).review(
|
||||
change, message, action)
|
||||
self._test_handle_review(int(change.number), message, action)
|
||||
|
||||
def _test_handle_review(self, change_number, message, action):
|
||||
# Handle a review action from a test
|
||||
change = self.changes[change_number]
|
||||
|
||||
# Add the approval back onto the change (ie simulate what gerrit would
|
||||
# do).
|
||||
|
|
@ -588,7 +679,8 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
|||
if cat != 'submit':
|
||||
change.addApproval(cat, action[cat], username=self.user)
|
||||
|
||||
change.messages.append(message)
|
||||
if message:
|
||||
change.messages.append(message)
|
||||
|
||||
if 'submit' in action:
|
||||
change.setMerged()
|
||||
|
|
@ -2340,6 +2432,9 @@ class ZuulTestCase(BaseTestCase):
|
|||
con = FakeGerritConnection(driver, name, config,
|
||||
changes_db=db,
|
||||
upstream_root=self.upstream_root)
|
||||
if con.web_server:
|
||||
self.addCleanup(con.web_server.stop)
|
||||
|
||||
self.event_queues.append(con.event_queue)
|
||||
setattr(self, 'fake_' + name, con)
|
||||
return con
|
||||
|
|
@ -2651,6 +2746,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
'pydevd.Reader',
|
||||
'pydevd.Writer',
|
||||
'socketserver_Thread',
|
||||
'GerritWebServer',
|
||||
]
|
||||
threads = [t for t in threading.enumerate()
|
||||
if t.name not in whitelist]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[statsd]
|
||||
# note, use 127.0.0.1 rather than localhost to avoid getting ipv6
|
||||
# see: https://github.com/jsocol/pystatsd/issues/61
|
||||
server=127.0.0.1
|
||||
|
||||
[scheduler]
|
||||
tenant_config=main.yaml
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/merger-git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
|
||||
[executor]
|
||||
git_dir=/tmp/zuul-test/executor-git
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=fake_id_rsa_path
|
||||
password=badpassword
|
||||
|
||||
[connection smtp]
|
||||
driver=smtp
|
||||
server=localhost
|
||||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
|
@ -16,7 +16,7 @@ import os
|
|||
from unittest import mock
|
||||
|
||||
import tests.base
|
||||
from tests.base import BaseTestCase
|
||||
from tests.base import BaseTestCase, ZuulTestCase
|
||||
from zuul.driver.gerrit import GerritDriver
|
||||
from zuul.driver.gerrit.gerritconnection import GerritConnection
|
||||
|
||||
|
|
@ -76,3 +76,27 @@ class TestGerrit(BaseTestCase):
|
|||
'simple_query_pagination_old_3']
|
||||
expected_patches = 5
|
||||
self.run_query(files, expected_patches)
|
||||
|
||||
|
||||
class TestGerritWeb(ZuulTestCase):
|
||||
config_file = 'zuul-gerrit-web.conf'
|
||||
tenant_config_file = 'config/single-tenant/main.yaml'
|
||||
|
||||
def test_jobs_executed(self):
|
||||
"Test that jobs are executed and a change is merged"
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
A.addApproval('Code-Review', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(self.getJobFromHistory('project-merge').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(self.getJobFromHistory('project-test1').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(self.getJobFromHistory('project-test2').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(A.reported, 2)
|
||||
self.assertEqual(self.getJobFromHistory('project-test1').node,
|
||||
'label1')
|
||||
self.assertEqual(self.getJobFromHistory('project-test2').node,
|
||||
'label1')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
# Copyright 2015 Christoph Gysin <christoph.gysin@gmail.com>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import requests
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class FormAuth(requests.auth.AuthBase):
|
||||
log = logging.getLogger('zuul.GerritConnection')
|
||||
|
||||
def __init__(self, username, password):
|
||||
self.username = username
|
||||
self.password = password
|
||||
|
||||
def _retry_using_form_auth(self, response, args):
|
||||
adapter = requests.adapters.HTTPAdapter()
|
||||
request = _copy_request(response.request)
|
||||
|
||||
u = urlparse.urlparse(response.url)
|
||||
url = urlparse.urlunparse([u.scheme, u.netloc, '/login',
|
||||
None, None, None])
|
||||
auth = {'username': self.username,
|
||||
'password': self.password}
|
||||
request2 = requests.Request('POST', url, data=auth).prepare()
|
||||
response2 = adapter.send(request2, **args)
|
||||
|
||||
if response2.status_code == 401:
|
||||
self.log.error('Login failed: Invalid username or password?')
|
||||
return response
|
||||
|
||||
cookie = response2.headers.get('set-cookie')
|
||||
if cookie is not None:
|
||||
request.headers['Cookie'] = cookie
|
||||
|
||||
response3 = adapter.send(request, **args)
|
||||
return response3
|
||||
|
||||
def _response_hook(self, response, **kwargs):
|
||||
if response.status_code == 401:
|
||||
return self._retry_using_form_auth(response, kwargs)
|
||||
return response
|
||||
|
||||
def __call__(self, request):
|
||||
request.headers["Connection"] = "Keep-Alive"
|
||||
request.register_hook('response', self._response_hook)
|
||||
return request
|
||||
|
||||
|
||||
def _copy_request(request):
|
||||
new_request = requests.PreparedRequest()
|
||||
new_request.method = request.method
|
||||
new_request.url = request.url
|
||||
new_request.body = request.body
|
||||
new_request.hooks = request.hooks
|
||||
new_request.headers = request.headers.copy()
|
||||
return new_request
|
||||
|
|
@ -25,6 +25,7 @@ import pprint
|
|||
import shlex
|
||||
import queue
|
||||
import voluptuous as v
|
||||
import requests
|
||||
|
||||
from typing import Dict, List
|
||||
|
||||
|
|
@ -32,6 +33,11 @@ from zuul.connection import BaseConnection
|
|||
from zuul.model import Ref, Tag, Branch, Project
|
||||
from zuul import exceptions
|
||||
from zuul.driver.gerrit.gerritmodel import GerritChange, GerritTriggerEvent
|
||||
from zuul.driver.gerrit.auth import FormAuth
|
||||
from zuul import version as zuul_version
|
||||
|
||||
# HTTP timeout in seconds
|
||||
TIMEOUT = 30
|
||||
|
||||
|
||||
class GerritEventConnector(threading.Thread):
|
||||
|
|
@ -317,6 +323,53 @@ class GerritConnection(BaseConnection):
|
|||
self.gerrit_event_connector = None
|
||||
self.source = driver.getSource(self)
|
||||
|
||||
self.session = None
|
||||
self.password = self.connection_config.get('password', None)
|
||||
if self.password:
|
||||
self.auth_type = self.connection_config.get('auth_type', None)
|
||||
self.verify_ssl = self.connection_config.get('verify_ssl', True)
|
||||
if self.verify_ssl not in ['true', 'True', '1', 1, 'TRUE']:
|
||||
self.verify_ssl = False
|
||||
self.user_agent = 'Zuul/%s %s' % (
|
||||
zuul_version.release_string,
|
||||
requests.utils.default_user_agent())
|
||||
self.session = requests.Session()
|
||||
if self.auth_type == 'basic':
|
||||
authclass = requests.auth.HTTPBasicAuth
|
||||
elif self.auth_type == 'form':
|
||||
authclass = FormAuth
|
||||
else:
|
||||
authclass = requests.auth.HTTPDigestAuth
|
||||
self.auth = authclass(
|
||||
self.user, self.password)
|
||||
|
||||
def url(self, path):
|
||||
return self.baseurl + '/a/' + path
|
||||
|
||||
def post(self, path, data):
|
||||
url = self.url(path)
|
||||
self.log.debug('POST: %s' % (url,))
|
||||
self.log.debug('data: %s' % (data,))
|
||||
r = self.session.post(
|
||||
url, data=json.dumps(data).encode('utf8'),
|
||||
verify=self.verify_ssl,
|
||||
auth=self.auth, timeout=TIMEOUT,
|
||||
headers={'Content-Type': 'application/json;charset=UTF-8',
|
||||
'User-Agent': self.user_agent})
|
||||
self.log.debug('Received: %s %s' % (r.status_code, r.text,))
|
||||
if r.status_code != 200:
|
||||
raise Exception("Received response %s" % (r.status_code,))
|
||||
ret = None
|
||||
if r.text and len(r.text) > 4:
|
||||
try:
|
||||
ret = json.loads(r.text[4:])
|
||||
except Exception:
|
||||
self.log.exception(
|
||||
"Unable to parse result %s from post to %s" %
|
||||
(r.text, url))
|
||||
raise
|
||||
return ret
|
||||
|
||||
def getProject(self, name: str) -> Project:
|
||||
return self.projects.get(name)
|
||||
|
||||
|
|
@ -477,6 +530,7 @@ class GerritConnection(BaseConnection):
|
|||
if 'project' not in data:
|
||||
raise exceptions.ChangeNotFound(change.number, change.patchset)
|
||||
change.project = self.source.getProject(data['project'])
|
||||
change.id = data['id']
|
||||
change.branch = data['branch']
|
||||
change.url = data['url']
|
||||
urlparse = urllib.parse.urlparse(self.baseurl)
|
||||
|
|
@ -492,6 +546,7 @@ class GerritConnection(BaseConnection):
|
|||
for ps in data['patchSets']:
|
||||
if str(ps['number']) == change.patchset:
|
||||
change.ref = ps['ref']
|
||||
change.commit = ps['revision']
|
||||
for f in ps.get('files', []):
|
||||
files.append(f['file'])
|
||||
if int(ps['number']) > int(max_ps):
|
||||
|
|
@ -721,7 +776,15 @@ class GerritConnection(BaseConnection):
|
|||
def eventDone(self):
|
||||
self.event_queue.task_done()
|
||||
|
||||
def review(self, project, change, message, action={}):
|
||||
def review(self, change, message, action={}):
|
||||
if self.session:
|
||||
meth = self.review_http
|
||||
else:
|
||||
meth = self.review_ssh
|
||||
return meth(change, message, action)
|
||||
|
||||
def review_ssh(self, change, message, action={}):
|
||||
project = change.project.name
|
||||
cmd = 'gerrit review --project %s' % project
|
||||
if message:
|
||||
cmd += ' --message %s' % shlex.quote(message)
|
||||
|
|
@ -730,10 +793,51 @@ class GerritConnection(BaseConnection):
|
|||
cmd += ' --%s' % key
|
||||
else:
|
||||
cmd += ' --label %s=%s' % (key, val)
|
||||
cmd += ' %s' % change
|
||||
changeid = '%s,%s' % (change.number, change.patchset)
|
||||
cmd += ' %s' % changeid
|
||||
out, err = self._ssh(cmd)
|
||||
return err
|
||||
|
||||
def review_http(self, change, message, action={},
|
||||
file_comments={}):
|
||||
data = dict(message=message,
|
||||
strict_labels=False)
|
||||
submit = False
|
||||
labels = {}
|
||||
for key, val in action.items():
|
||||
if val is True:
|
||||
if key == 'submit':
|
||||
submit = True
|
||||
else:
|
||||
labels[key] = val
|
||||
if change.is_current_patchset:
|
||||
if labels:
|
||||
data['labels'] = labels
|
||||
if file_comments:
|
||||
data['comments'] = file_comments
|
||||
# { path: [
|
||||
# {line=42, message='foobar'},
|
||||
# {line=40, message='baz'},
|
||||
# ]
|
||||
# }
|
||||
for x in range(1, 4):
|
||||
try:
|
||||
self.post('changes/%s/revisions/%s/review' %
|
||||
(change.id, change.commit),
|
||||
data)
|
||||
break
|
||||
except Exception:
|
||||
self.log.exception(
|
||||
"Error submitting data to gerrit, attempt %s", x)
|
||||
time.sleep(x * 10)
|
||||
if change.is_current_patchset and submit:
|
||||
try:
|
||||
self.post('changes/%s/submit' % (change.id,), {})
|
||||
except Exception:
|
||||
self.log.exception(
|
||||
"Error submitting data to gerrit, attempt %s", x)
|
||||
time.sleep(x * 10)
|
||||
|
||||
def query(self, query):
|
||||
args = '--all-approvals --comments --commit-message'
|
||||
args += ' --current-patch-set --dependencies --files'
|
||||
|
|
|
|||
|
|
@ -42,12 +42,10 @@ class GerritReporter(BaseReporter):
|
|||
|
||||
self.log.debug("Report change %s, params %s, message: %s" %
|
||||
(item.change, self.config, message))
|
||||
changeid = '%s,%s' % (item.change.number, item.change.patchset)
|
||||
item.change._ref_sha = item.change.project.source.getRefSha(
|
||||
item.change.project, 'refs/heads/' + item.change.branch)
|
||||
|
||||
return self.connection.review(item.change.project.name, changeid,
|
||||
message, self.config)
|
||||
return self.connection.review(item.change, message, self.config)
|
||||
|
||||
def getSubmitAllowNeeds(self):
|
||||
"""Get a list of code review labels that are allowed to be
|
||||
|
|
|
|||
Loading…
Reference in New Issue