Support pagination in simplequery for Gerrit >=2.9
Starting with Gerrit 2.9 the pagination mechanism changed to using offsets instead of sortkeys. For details please check the release notes of 2.9: https://gerrit-documentation.storage.googleapis.com/ReleaseNotes/ReleaseNotes-2.9.html#_rest_api. In order to support both versions it is necessary to check for the presence of moreChanges in the status line and then process the next chunks either with the old or the new method. Change-Id: Ieb54052debd07d86db66ef0f1d7ce1bdb42964b4
This commit is contained in:
Tobias Henkel
parent
e8aa3c6842
commit
dd78ef2dcc
|
|
@ -11,3 +11,4 @@ python-swiftclient>=1.6
|
|||
testrepository>=0.0.17
|
||||
testtools>=0.9.32
|
||||
sphinxcontrib-programoutput
|
||||
mock
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"(detached","id":"I173251c8b1569755124b7cb1a48b6274bf38c94b","number":"202867","subject":"Report the per-job build wait time to graphite","owner":{"name":"Timothy R. Chavez","email":"timothy.chavez@hp.com","username":"timrchavez"},"url":"https://review.openstack.org/202867","commitMessage":"Report the per-job build wait time to graphite\n\nKnowing how long a job waits to build in aggregate can give useful\ninsights into the performance and capacity of the build system. This\nchange also uses the node labels sent back from the gearman worker to\nsubmit metrics within that context.\n\nChange-Id: I173251c8b1569755124b7cb1a48b6274bf38c94b\nDepends-On: Ibca938fcf8a65facd7e39dab4eb994dfc637722a\n","createdOn":1437104683,"lastUpdated":1440760891,"open":true,"status":"NEW"}
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"ignore-deletes","id":"Iea75d05ddcb49b0bf748b72b9d2d5472d077f0c6","number":"178833","subject":"Add option to ignore ref-updated events emitted by branch deletions","owner":{"name":"K Jonathan Harker","email":"code@gentlydownthe.net","username":"jesusaurus"},"url":"https://review.openstack.org/178833","commitMessage":"Add option to ignore ref-updated events emitted by branch deletions\n\nWhen a branch is deleted, gerrit emits a ref-updated event with a newrev\nvalue of all zeros. This adds a boolean field to optionally not trigger\non these ref-updated events.\n\nChange-Id: Iea75d05ddcb49b0bf748b72b9d2d5472d077f0c6\n","createdOn":1430339761,"lastUpdated":1440735750,"open":true,"status":"NEW"}
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"undefined-projects","id":"I7912197fb86c1a7becb7f43ca36078101f632715","number":"207094","subject":"Dependencies from undefined projects","owner":{"name":"Evgeny Antyshev","email":"eantyshev@virtuozzo.com","username":"eantyshev"},"url":"https://review.openstack.org/207094","commitMessage":"Dependencies from undefined projects\n\n3rd party CI layout usually has only a few projects defined,\nso it\u0027s possible that some changes depend on projects\nwhich are unknown to Zuul scheduler.\nThese items had None as a \"item.change.project\", which\nis not handled in many places, for ex. in reconfiguration.\n\nThese cases could be handled by defining these projects in layout\nas \"foreign\" projects: no jobs, no other non-standard attributes.\nChanges to those projects are also dropped, unless\nthey came as dependencies.\n\nChange-Id: I7912197fb86c1a7becb7f43ca36078101f632715\n","createdOn":1438183395,"lastUpdated":1440667433,"open":true,"status":"NEW"}
|
||||
{"type":"stats","rowCount":3,"runTimeMilliseconds":12,"moreChanges":true}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul -S 3
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"github-integration","id":"I95f41088ea160d4e33a507c4a413e3fa7f08906b","number":"192457","subject":"(WIP) Fix job hierarchy bug.","owner":{"name":"Wayne Warren","email":"waynr+launchpad@sdf.org","username":"waynr"},"url":"https://review.openstack.org/192457","commitMessage":"(WIP) Fix job hierarchy bug.\n\nJobTree.addJob may return \u0027None\u0027, this prevents that from happening.\n\nChange-Id: I95f41088ea160d4e33a507c4a413e3fa7f08906b\n","createdOn":1434498278,"lastUpdated":1440608984,"sortKey":"003763050002efc9","open":true,"status":"NEW"}
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"github-integration","id":"Ic5887d00ff302f67469df5154e9df10b99f1cfcd","number":"215642","subject":"(WIP) Allow using webapp from connections","owner":{"name":"Jan Hruban","email":"jan.hruban@gooddata.com","username":"hrubi"},"url":"https://review.openstack.org/215642","commitMessage":"(WIP) Allow using webapp from connections\n\nAllow connections to register their own handlers for HTTP URIs inside\nthe zuul\u0027s webapp HTTP server. That way, connections can listen for\nevents comming through HTTP.\n\nChange-Id: Ic5887d00ff302f67469df5154e9df10b99f1cfcd\n","createdOn":1440165019,"lastUpdated":1440602591,"sortKey":"0037629b00034a5a","open":true,"status":"NEW"}
|
||||
{"type":"stats","rowCount":2,"runTimeMilliseconds":12,"moreChanges":false}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"(detached","id":"I173251c8b1569755124b7cb1a48b6274bf38c94b","number":"202867","subject":"Report the per-job build wait time to graphite","owner":{"name":"Timothy R. Chavez","email":"timothy.chavez@hp.com","username":"timrchavez"},"url":"https://review.openstack.org/202867","commitMessage":"Report the per-job build wait time to graphite\n\nKnowing how long a job waits to build in aggregate can give useful\ninsights into the performance and capacity of the build system. This\nchange also uses the node labels sent back from the gearman worker to\nsubmit metrics within that context.\n\nChange-Id: I173251c8b1569755124b7cb1a48b6274bf38c94b\nDepends-On: Ibca938fcf8a65facd7e39dab4eb994dfc637722a\n","createdOn":1437104683,"lastUpdated":1440760891,"sortKey":"00376ce900031873","open":true,"status":"NEW"}
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"ignore-deletes","id":"Iea75d05ddcb49b0bf748b72b9d2d5472d077f0c6","number":"178833","subject":"Add option to ignore ref-updated events emitted by branch deletions","owner":{"name":"K Jonathan Harker","email":"code@gentlydownthe.net","username":"jesusaurus"},"url":"https://review.openstack.org/178833","commitMessage":"Add option to ignore ref-updated events emitted by branch deletions\n\nWhen a branch is deleted, gerrit emits a ref-updated event with a newrev\nvalue of all zeros. This adds a boolean field to optionally not trigger\non these ref-updated events.\n\nChange-Id: Iea75d05ddcb49b0bf748b72b9d2d5472d077f0c6\n","createdOn":1430339761,"lastUpdated":1440735750,"sortKey":"00376b460002ba91","open":true,"status":"NEW"}
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"undefined-projects","id":"I7912197fb86c1a7becb7f43ca36078101f632715","number":"207094","subject":"Dependencies from undefined projects","owner":{"name":"Evgeny Antyshev","email":"eantyshev@virtuozzo.com","username":"eantyshev"},"url":"https://review.openstack.org/207094","commitMessage":"Dependencies from undefined projects\n\n3rd party CI layout usually has only a few projects defined,\nso it\u0027s possible that some changes depend on projects\nwhich are unknown to Zuul scheduler.\nThese items had None as a \"item.change.project\", which\nis not handled in many places, for ex. in reconfiguration.\n\nThese cases could be handled by defining these projects in layout\nas \"foreign\" projects: no jobs, no other non-standard attributes.\nChanges to those projects are also dropped, unless\nthey came as dependencies.\n\nChange-Id: I7912197fb86c1a7becb7f43ca36078101f632715\n","createdOn":1438183395,"lastUpdated":1440667433,"sortKey":"003766d3000328f6","open":true,"status":"NEW"}
|
||||
{"type":"stats","rowCount":3,"runTimeMilliseconds":12}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul resume_sortkey:'003766d3000328f6'
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"github-integration","id":"I95f41088ea160d4e33a507c4a413e3fa7f08906b","number":"192457","subject":"(WIP) Fix job hierarchy bug.","owner":{"name":"Wayne Warren","email":"waynr+launchpad@sdf.org","username":"waynr"},"url":"https://review.openstack.org/192457","commitMessage":"(WIP) Fix job hierarchy bug.\n\nJobTree.addJob may return \u0027None\u0027, this prevents that from happening.\n\nChange-Id: I95f41088ea160d4e33a507c4a413e3fa7f08906b\n","createdOn":1434498278,"lastUpdated":1440608984,"sortKey":"003763050002efc9","open":true,"status":"NEW"}
|
||||
{"project":"openstack-infra/zuul","branch":"master","topic":"github-integration","id":"Ic5887d00ff302f67469df5154e9df10b99f1cfcd","number":"215642","subject":"(WIP) Allow using webapp from connections","owner":{"name":"Jan Hruban","email":"jan.hruban@gooddata.com","username":"hrubi"},"url":"https://review.openstack.org/215642","commitMessage":"(WIP) Allow using webapp from connections\n\nAllow connections to register their own handlers for HTTP URIs inside\nthe zuul\u0027s webapp HTTP server. That way, connections can listen for\nevents comming through HTTP.\n\nChange-Id: Ic5887d00ff302f67469df5154e9df10b99f1cfcd\n","createdOn":1440165019,"lastUpdated":1440602591,"sortKey":"0037629b00034a5a","open":true,"status":"NEW"}
|
||||
{"type":"stats","rowCount":2,"runTimeMilliseconds":12}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul resume_sortkey:'0037629b00034a5a'
|
||||
{"type":"stats","rowCount":0,"runTimeMilliseconds":12}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2015 BMW Car IT GmbH
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import os
|
||||
|
||||
try:
|
||||
from unittest import mock
|
||||
except ImportError:
|
||||
import mock
|
||||
|
||||
from tests.base import BaseTestCase
|
||||
from zuul.lib.gerrit import Gerrit
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'fixtures/gerrit')
|
||||
|
||||
|
||||
def read_fixture(file):
|
||||
with open('%s/%s' % (FIXTURE_DIR, file), 'r') as fixturefile:
|
||||
lines = fixturefile.readlines()
|
||||
command = lines[0].replace('\n', '')
|
||||
value = ''.join(lines[1:])
|
||||
return command, value
|
||||
|
||||
|
||||
def read_fixtures(files):
|
||||
calls = []
|
||||
values = []
|
||||
for fixture_file in files:
|
||||
command, value = read_fixture(fixture_file)
|
||||
calls.append(mock.call(command))
|
||||
values.append([value, ''])
|
||||
return calls, values
|
||||
|
||||
|
||||
class TestGerrit(BaseTestCase):
|
||||
|
||||
@mock.patch('zuul.lib.gerrit.Gerrit._ssh')
|
||||
def run_query(self, files, expected_patches, _ssh_mock):
|
||||
gerrit = Gerrit('localhost', 'user')
|
||||
|
||||
calls, values = read_fixtures(files)
|
||||
_ssh_mock.side_effect = values
|
||||
|
||||
result = gerrit.simpleQuery('project:openstack-infra/zuul')
|
||||
|
||||
_ssh_mock.assert_has_calls(calls)
|
||||
self.assertEquals(len(calls), _ssh_mock.call_count,
|
||||
'_ssh should be called %d times' % len(calls))
|
||||
self.assertIsNotNone(result, 'Result is not none')
|
||||
self.assertEquals(len(result), expected_patches,
|
||||
'There must be %d patches.' % expected_patches)
|
||||
|
||||
def test_simple_query_pagination_new(self):
|
||||
files = ['simple_query_pagination_new_1',
|
||||
'simple_query_pagination_new_2']
|
||||
expected_patches = 5
|
||||
self.run_query(files, expected_patches)
|
||||
|
||||
def test_simple_query_pagination_old(self):
|
||||
files = ['simple_query_pagination_old_1',
|
||||
'simple_query_pagination_old_2',
|
||||
'simple_query_pagination_old_3']
|
||||
expected_patches = 5
|
||||
self.run_query(files, expected_patches)
|
||||
|
|
@ -156,22 +156,42 @@ class Gerrit(object):
|
|||
lines = out.split('\n')
|
||||
if not lines:
|
||||
return False
|
||||
|
||||
# filter out blank lines
|
||||
data = [json.loads(line) for line in lines
|
||||
if "sortKey" in line]
|
||||
if line.startswith('{')]
|
||||
|
||||
# check last entry for more changes
|
||||
more_changes = None
|
||||
if 'moreChanges' in data[-1]:
|
||||
more_changes = data[-1]['moreChanges']
|
||||
|
||||
# we have to remove the statistics line
|
||||
del data[-1]
|
||||
|
||||
if not data:
|
||||
return False
|
||||
return False, more_changes
|
||||
self.log.debug("Received data from Gerrit query: \n%s" %
|
||||
(pprint.pformat(data)))
|
||||
return data
|
||||
return data, more_changes
|
||||
|
||||
# gerrit returns 500 results by default, so implement paging
|
||||
# for large projects like nova
|
||||
alldata = []
|
||||
chunk = _query_chunk(query)
|
||||
chunk, more_changes = _query_chunk(query)
|
||||
while(chunk):
|
||||
alldata.extend(chunk)
|
||||
sortkey = "resume_sortkey:'%s'" % chunk[-1]["sortKey"]
|
||||
chunk = _query_chunk("%s %s" % (query, sortkey))
|
||||
if more_changes is None:
|
||||
# continue sortKey based (before Gerrit 2.9)
|
||||
resume = "resume_sortkey:'%s'" % chunk[-1]["sortKey"]
|
||||
elif more_changes:
|
||||
# continue moreChanges based (since Gerrit 2.9)
|
||||
resume = "-S %d" % len(alldata)
|
||||
else:
|
||||
# no more changes
|
||||
break
|
||||
|
||||
chunk, more_changes = _query_chunk("%s %s" % (query, resume))
|
||||
return alldata
|
||||
|
||||
def _open(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue