Report to gerrit over HTTP
Gerrit can only accept file comments over HTTP, not SSH. So add the ability to report over HTTP. This is optional; if no HTTP credentials are provided, the SSH mechanism will be used for now and no file comments will be left. Note, this change does not include the driver API change needed to sumpport file comments. This change includes a significant amount of ASL2 licensed code from Gertty. Change-Id: I851c62088f228de21441dbd86bf60722d82f94f8
This commit is contained in:
parent
801ed767ff
commit
77288488c3
|
@ -36,6 +36,7 @@ The supported options in ``zuul.conf`` connections are:
|
|||
The connection must set ``driver=gerrit`` for Gerrit connections.
|
||||
|
||||
.. attr:: server
|
||||
:required:
|
||||
|
||||
Fully qualified domain name of Gerrit server.
|
||||
|
||||
|
@ -58,8 +59,9 @@ The supported options in ``zuul.conf`` connections are:
|
|||
Gerrit server port.
|
||||
|
||||
.. attr:: baseurl
|
||||
:default: https://{server}
|
||||
|
||||
Path to Gerrit web interface.
|
||||
Path to Gerrit web interface. Omit the trailing ``/``.
|
||||
|
||||
.. attr:: gitweb_url_template
|
||||
:default: {baseurl}/gitweb?p={project.name}.git;a=commitdiff;h={sha}
|
||||
|
@ -87,6 +89,40 @@ The supported options in ``zuul.conf`` connections are:
|
|||
|
||||
SSH connection keepalive timeout; ``0`` disables.
|
||||
|
||||
.. attr:: password
|
||||
|
||||
The HTTP authentication password for the user. This is
|
||||
optional, but if it is provided, Zuul will report to Gerrit via
|
||||
HTTP rather than SSH. It is required in order for file and line
|
||||
comments to reported (the Gerrit SSH API only supports review
|
||||
messages). Retrieve this password from the ``HTTP Password``
|
||||
section of the ``Settings`` page in Gerrit.
|
||||
|
||||
.. attr:: auth_type
|
||||
:default: digest
|
||||
|
||||
The HTTP authentication mechanism.
|
||||
|
||||
.. value:: digest
|
||||
|
||||
HTTP Digest authentication; the default for most Gerrit
|
||||
installations.
|
||||
|
||||
.. value:: basic
|
||||
|
||||
HTTP Basic authentication.
|
||||
|
||||
.. value:: form
|
||||
|
||||
Zuul will submit a username and password to a form in order
|
||||
to authenticate.
|
||||
|
||||
.. attr:: verify_ssl
|
||||
:default: true
|
||||
|
||||
When using a self-signed certificate, this may be set to
|
||||
``false`` to disable SSL certificate verification.
|
||||
|
||||
Trigger Configuration
|
||||
---------------------
|
||||
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
features:
|
||||
- The Gerrit driver can now (optionally) report via HTTP instead of
|
||||
SSH. In the future, this will be used to report file and line
|
||||
comments (the SSH API only supports review messages).
|
104
tests/base.py
104
tests/base.py
|
@ -478,6 +478,82 @@ class FakeGerritChange(object):
|
|||
self.reported += 1
|
||||
|
||||
|
||||
class GerritWebServer(object):
|
||||
|
||||
def __init__(self, fake_gerrit):
|
||||
super(GerritWebServer, self).__init__()
|
||||
self.fake_gerrit = fake_gerrit
|
||||
|
||||
def start(self):
|
||||
fake_gerrit = self.fake_gerrit
|
||||
|
||||
class Server(http.server.SimpleHTTPRequestHandler):
|
||||
log = logging.getLogger("zuul.test.FakeGerritConnection")
|
||||
review_re = re.compile('/a/changes/(.*?)/revisions/(.*?)/review')
|
||||
submit_re = re.compile('/a/changes/(.*?)/submit')
|
||||
|
||||
def do_POST(self):
|
||||
path = self.path
|
||||
self.log.debug("Got POST %s", path)
|
||||
|
||||
data = self.rfile.read(int(self.headers['Content-Length']))
|
||||
data = json.loads(data.decode('utf-8'))
|
||||
self.log.debug("Got data %s", data)
|
||||
|
||||
m = self.review_re.match(path)
|
||||
if m:
|
||||
return self.review(m.group(1), m.group(2), data)
|
||||
m = self.submit_re.match(path)
|
||||
if m:
|
||||
return self.submit(m.group(1), data)
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
|
||||
def _404(self):
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def _get_change(self, change_id):
|
||||
for c in fake_gerrit.changes.values():
|
||||
if c.data['id'] == change_id:
|
||||
return c
|
||||
|
||||
def review(self, change_id, revision, data):
|
||||
change = self._get_change(change_id)
|
||||
if not change:
|
||||
return self._404()
|
||||
|
||||
message = data['message']
|
||||
action = data['labels']
|
||||
fake_gerrit._test_handle_review(
|
||||
int(change.data['number']), message, action)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
|
||||
def submit(self, change_id, data):
|
||||
change = self._get_change(change_id)
|
||||
if not change:
|
||||
return self._404()
|
||||
|
||||
message = None
|
||||
action = {'submit': True}
|
||||
fake_gerrit._test_handle_review(
|
||||
int(change.data['number']), message, action)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
|
||||
self.httpd = socketserver.ThreadingTCPServer(('', 0), Server)
|
||||
self.port = self.httpd.socket.getsockname()[1]
|
||||
self.thread = threading.Thread(name='GerritWebServer',
|
||||
target=self.httpd.serve_forever)
|
||||
self.thread.daemon = True
|
||||
self.thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.httpd.shutdown()
|
||||
self.thread.join()
|
||||
|
||||
|
||||
class FakeGerritConnection(gerritconnection.GerritConnection):
|
||||
"""A Fake Gerrit connection for use in tests.
|
||||
|
||||
|
@ -490,6 +566,15 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
|||
|
||||
def __init__(self, driver, connection_name, connection_config,
|
||||
changes_db=None, upstream_root=None):
|
||||
|
||||
if connection_config.get('password'):
|
||||
self.web_server = GerritWebServer(self)
|
||||
self.web_server.start()
|
||||
url = 'http://localhost:%s' % self.web_server.port
|
||||
connection_config['baseurl'] = url
|
||||
else:
|
||||
self.web_server = None
|
||||
|
||||
super(FakeGerritConnection, self).__init__(driver, connection_name,
|
||||
connection_config)
|
||||
|
||||
|
@ -570,9 +655,15 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
|||
}
|
||||
return event
|
||||
|
||||
def review(self, project, changeid, message, action):
|
||||
number, ps = changeid.split(',')
|
||||
change = self.changes[int(number)]
|
||||
def review(self, change, message, action):
|
||||
if self.web_server:
|
||||
return super(FakeGerritConnection, self).review(
|
||||
change, message, action)
|
||||
self._test_handle_review(int(change.number), message, action)
|
||||
|
||||
def _test_handle_review(self, change_number, message, action):
|
||||
# Handle a review action from a test
|
||||
change = self.changes[change_number]
|
||||
|
||||
# Add the approval back onto the change (ie simulate what gerrit would
|
||||
# do).
|
||||
|
@ -588,7 +679,8 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
|||
if cat != 'submit':
|
||||
change.addApproval(cat, action[cat], username=self.user)
|
||||
|
||||
change.messages.append(message)
|
||||
if message:
|
||||
change.messages.append(message)
|
||||
|
||||
if 'submit' in action:
|
||||
change.setMerged()
|
||||
|
@ -2340,6 +2432,9 @@ class ZuulTestCase(BaseTestCase):
|
|||
con = FakeGerritConnection(driver, name, config,
|
||||
changes_db=db,
|
||||
upstream_root=self.upstream_root)
|
||||
if con.web_server:
|
||||
self.addCleanup(con.web_server.stop)
|
||||
|
||||
self.event_queues.append(con.event_queue)
|
||||
setattr(self, 'fake_' + name, con)
|
||||
return con
|
||||
|
@ -2651,6 +2746,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
'pydevd.Reader',
|
||||
'pydevd.Writer',
|
||||
'socketserver_Thread',
|
||||
'GerritWebServer',
|
||||
]
|
||||
threads = [t for t in threading.enumerate()
|
||||
if t.name not in whitelist]
|
||||
|
|
|
@ -0,0 +1,32 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[statsd]
|
||||
# note, use 127.0.0.1 rather than localhost to avoid getting ipv6
|
||||
# see: https://github.com/jsocol/pystatsd/issues/61
|
||||
server=127.0.0.1
|
||||
|
||||
[scheduler]
|
||||
tenant_config=main.yaml
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/merger-git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
|
||||
[executor]
|
||||
git_dir=/tmp/zuul-test/executor-git
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=fake_id_rsa_path
|
||||
password=badpassword
|
||||
|
||||
[connection smtp]
|
||||
driver=smtp
|
||||
server=localhost
|
||||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
|
@ -16,7 +16,7 @@ import os
|
|||
from unittest import mock
|
||||
|
||||
import tests.base
|
||||
from tests.base import BaseTestCase
|
||||
from tests.base import BaseTestCase, ZuulTestCase
|
||||
from zuul.driver.gerrit import GerritDriver
|
||||
from zuul.driver.gerrit.gerritconnection import GerritConnection
|
||||
|
||||
|
@ -76,3 +76,27 @@ class TestGerrit(BaseTestCase):
|
|||
'simple_query_pagination_old_3']
|
||||
expected_patches = 5
|
||||
self.run_query(files, expected_patches)
|
||||
|
||||
|
||||
class TestGerritWeb(ZuulTestCase):
|
||||
config_file = 'zuul-gerrit-web.conf'
|
||||
tenant_config_file = 'config/single-tenant/main.yaml'
|
||||
|
||||
def test_jobs_executed(self):
|
||||
"Test that jobs are executed and a change is merged"
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
A.addApproval('Code-Review', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(self.getJobFromHistory('project-merge').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(self.getJobFromHistory('project-test1').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(self.getJobFromHistory('project-test2').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(A.reported, 2)
|
||||
self.assertEqual(self.getJobFromHistory('project-test1').node,
|
||||
'label1')
|
||||
self.assertEqual(self.getJobFromHistory('project-test2').node,
|
||||
'label1')
|
||||
|
|
|
@ -0,0 +1,69 @@
|
|||
# Copyright 2015 Christoph Gysin <christoph.gysin@gmail.com>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import requests
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class FormAuth(requests.auth.AuthBase):
|
||||
log = logging.getLogger('zuul.GerritConnection')
|
||||
|
||||
def __init__(self, username, password):
|
||||
self.username = username
|
||||
self.password = password
|
||||
|
||||
def _retry_using_form_auth(self, response, args):
|
||||
adapter = requests.adapters.HTTPAdapter()
|
||||
request = _copy_request(response.request)
|
||||
|
||||
u = urlparse.urlparse(response.url)
|
||||
url = urlparse.urlunparse([u.scheme, u.netloc, '/login',
|
||||
None, None, None])
|
||||
auth = {'username': self.username,
|
||||
'password': self.password}
|
||||
request2 = requests.Request('POST', url, data=auth).prepare()
|
||||
response2 = adapter.send(request2, **args)
|
||||
|
||||
if response2.status_code == 401:
|
||||
self.log.error('Login failed: Invalid username or password?')
|
||||
return response
|
||||
|
||||
cookie = response2.headers.get('set-cookie')
|
||||
if cookie is not None:
|
||||
request.headers['Cookie'] = cookie
|
||||
|
||||
response3 = adapter.send(request, **args)
|
||||
return response3
|
||||
|
||||
def _response_hook(self, response, **kwargs):
|
||||
if response.status_code == 401:
|
||||
return self._retry_using_form_auth(response, kwargs)
|
||||
return response
|
||||
|
||||
def __call__(self, request):
|
||||
request.headers["Connection"] = "Keep-Alive"
|
||||
request.register_hook('response', self._response_hook)
|
||||
return request
|
||||
|
||||
|
||||
def _copy_request(request):
|
||||
new_request = requests.PreparedRequest()
|
||||
new_request.method = request.method
|
||||
new_request.url = request.url
|
||||
new_request.body = request.body
|
||||
new_request.hooks = request.hooks
|
||||
new_request.headers = request.headers.copy()
|
||||
return new_request
|
|
@ -25,6 +25,7 @@ import pprint
|
|||
import shlex
|
||||
import queue
|
||||
import voluptuous as v
|
||||
import requests
|
||||
|
||||
from typing import Dict, List
|
||||
|
||||
|
@ -32,6 +33,11 @@ from zuul.connection import BaseConnection
|
|||
from zuul.model import Ref, Tag, Branch, Project
|
||||
from zuul import exceptions
|
||||
from zuul.driver.gerrit.gerritmodel import GerritChange, GerritTriggerEvent
|
||||
from zuul.driver.gerrit.auth import FormAuth
|
||||
from zuul import version as zuul_version
|
||||
|
||||
# HTTP timeout in seconds
|
||||
TIMEOUT = 30
|
||||
|
||||
|
||||
class GerritEventConnector(threading.Thread):
|
||||
|
@ -317,6 +323,53 @@ class GerritConnection(BaseConnection):
|
|||
self.gerrit_event_connector = None
|
||||
self.source = driver.getSource(self)
|
||||
|
||||
self.session = None
|
||||
self.password = self.connection_config.get('password', None)
|
||||
if self.password:
|
||||
self.auth_type = self.connection_config.get('auth_type', None)
|
||||
self.verify_ssl = self.connection_config.get('verify_ssl', True)
|
||||
if self.verify_ssl not in ['true', 'True', '1', 1, 'TRUE']:
|
||||
self.verify_ssl = False
|
||||
self.user_agent = 'Zuul/%s %s' % (
|
||||
zuul_version.release_string,
|
||||
requests.utils.default_user_agent())
|
||||
self.session = requests.Session()
|
||||
if self.auth_type == 'basic':
|
||||
authclass = requests.auth.HTTPBasicAuth
|
||||
elif self.auth_type == 'form':
|
||||
authclass = FormAuth
|
||||
else:
|
||||
authclass = requests.auth.HTTPDigestAuth
|
||||
self.auth = authclass(
|
||||
self.user, self.password)
|
||||
|
||||
def url(self, path):
|
||||
return self.baseurl + '/a/' + path
|
||||
|
||||
def post(self, path, data):
|
||||
url = self.url(path)
|
||||
self.log.debug('POST: %s' % (url,))
|
||||
self.log.debug('data: %s' % (data,))
|
||||
r = self.session.post(
|
||||
url, data=json.dumps(data).encode('utf8'),
|
||||
verify=self.verify_ssl,
|
||||
auth=self.auth, timeout=TIMEOUT,
|
||||
headers={'Content-Type': 'application/json;charset=UTF-8',
|
||||
'User-Agent': self.user_agent})
|
||||
self.log.debug('Received: %s %s' % (r.status_code, r.text,))
|
||||
if r.status_code != 200:
|
||||
raise Exception("Received response %s" % (r.status_code,))
|
||||
ret = None
|
||||
if r.text and len(r.text) > 4:
|
||||
try:
|
||||
ret = json.loads(r.text[4:])
|
||||
except Exception:
|
||||
self.log.exception(
|
||||
"Unable to parse result %s from post to %s" %
|
||||
(r.text, url))
|
||||
raise
|
||||
return ret
|
||||
|
||||
def getProject(self, name: str) -> Project:
|
||||
return self.projects.get(name)
|
||||
|
||||
|
@ -477,6 +530,7 @@ class GerritConnection(BaseConnection):
|
|||
if 'project' not in data:
|
||||
raise exceptions.ChangeNotFound(change.number, change.patchset)
|
||||
change.project = self.source.getProject(data['project'])
|
||||
change.id = data['id']
|
||||
change.branch = data['branch']
|
||||
change.url = data['url']
|
||||
urlparse = urllib.parse.urlparse(self.baseurl)
|
||||
|
@ -492,6 +546,7 @@ class GerritConnection(BaseConnection):
|
|||
for ps in data['patchSets']:
|
||||
if str(ps['number']) == change.patchset:
|
||||
change.ref = ps['ref']
|
||||
change.commit = ps['revision']
|
||||
for f in ps.get('files', []):
|
||||
files.append(f['file'])
|
||||
if int(ps['number']) > int(max_ps):
|
||||
|
@ -721,7 +776,15 @@ class GerritConnection(BaseConnection):
|
|||
def eventDone(self):
|
||||
self.event_queue.task_done()
|
||||
|
||||
def review(self, project, change, message, action={}):
|
||||
def review(self, change, message, action={}):
|
||||
if self.session:
|
||||
meth = self.review_http
|
||||
else:
|
||||
meth = self.review_ssh
|
||||
return meth(change, message, action)
|
||||
|
||||
def review_ssh(self, change, message, action={}):
|
||||
project = change.project.name
|
||||
cmd = 'gerrit review --project %s' % project
|
||||
if message:
|
||||
cmd += ' --message %s' % shlex.quote(message)
|
||||
|
@ -730,10 +793,51 @@ class GerritConnection(BaseConnection):
|
|||
cmd += ' --%s' % key
|
||||
else:
|
||||
cmd += ' --label %s=%s' % (key, val)
|
||||
cmd += ' %s' % change
|
||||
changeid = '%s,%s' % (change.number, change.patchset)
|
||||
cmd += ' %s' % changeid
|
||||
out, err = self._ssh(cmd)
|
||||
return err
|
||||
|
||||
def review_http(self, change, message, action={},
|
||||
file_comments={}):
|
||||
data = dict(message=message,
|
||||
strict_labels=False)
|
||||
submit = False
|
||||
labels = {}
|
||||
for key, val in action.items():
|
||||
if val is True:
|
||||
if key == 'submit':
|
||||
submit = True
|
||||
else:
|
||||
labels[key] = val
|
||||
if change.is_current_patchset:
|
||||
if labels:
|
||||
data['labels'] = labels
|
||||
if file_comments:
|
||||
data['comments'] = file_comments
|
||||
# { path: [
|
||||
# {line=42, message='foobar'},
|
||||
# {line=40, message='baz'},
|
||||
# ]
|
||||
# }
|
||||
for x in range(1, 4):
|
||||
try:
|
||||
self.post('changes/%s/revisions/%s/review' %
|
||||
(change.id, change.commit),
|
||||
data)
|
||||
break
|
||||
except Exception:
|
||||
self.log.exception(
|
||||
"Error submitting data to gerrit, attempt %s", x)
|
||||
time.sleep(x * 10)
|
||||
if change.is_current_patchset and submit:
|
||||
try:
|
||||
self.post('changes/%s/submit' % (change.id,), {})
|
||||
except Exception:
|
||||
self.log.exception(
|
||||
"Error submitting data to gerrit, attempt %s", x)
|
||||
time.sleep(x * 10)
|
||||
|
||||
def query(self, query):
|
||||
args = '--all-approvals --comments --commit-message'
|
||||
args += ' --current-patch-set --dependencies --files'
|
||||
|
|
|
@ -42,12 +42,10 @@ class GerritReporter(BaseReporter):
|
|||
|
||||
self.log.debug("Report change %s, params %s, message: %s" %
|
||||
(item.change, self.config, message))
|
||||
changeid = '%s,%s' % (item.change.number, item.change.patchset)
|
||||
item.change._ref_sha = item.change.project.source.getRefSha(
|
||||
item.change.project, 'refs/heads/' + item.change.branch)
|
||||
|
||||
return self.connection.review(item.change.project.name, changeid,
|
||||
message, self.config)
|
||||
return self.connection.review(item.change, message, self.config)
|
||||
|
||||
def getSubmitAllowNeeds(self):
|
||||
"""Get a list of code review labels that are allowed to be
|
||||
|
|
Loading…
Reference in New Issue