Add an additional pass through project templates
In order to find project templates that need to be expanded from job matchers that are not project specific, we need to make a pass through all of the projects assuming all templates need to be expanded. We then look at the result of the expansion to see if anything was actually done. As part of this, we also collect same-expansions on a job basis to track if a given job always has the same matcher expansion. If it does, we can apply that to the job definition and not to the project-pipeline defintion, which could lower the number of templates that need to be expanded. This may be the ugliest code I've ever written. I'm sorry. Also fix a bash bug in the run-migration script that caused final to always get run regardless of flag setting. Whoops. Change-Id: I523909e5242e0db125b7560cbdcd9ac41ca6c72f
This commit is contained in:
Monty Taylor
parent
5d7d2c6439
commit
9dcb71cb87
|
|
@ -47,12 +47,12 @@ done
|
|||
|
||||
BASE_DIR=$(cd $(dirname $0)/../..; pwd)
|
||||
cd $BASE_DIR/project-config
|
||||
if [[ $FINAL ]] ; then
|
||||
if [[ $FINAL = 1 ]] ; then
|
||||
git reset --hard
|
||||
fi
|
||||
python3 $BASE_DIR/zuul/zuul/cmd/migrate.py --mapping=zuul/mapping.yaml \
|
||||
zuul/layout.yaml jenkins/jobs nodepool/nodepool.yaml . $VERBOSE
|
||||
if [[ $FINAL ]] ; then
|
||||
if [[ $FINAL = 1 ]] ; then
|
||||
find ../openstack-zuul-jobs/playbooks/legacy -maxdepth 1 -mindepth 1 \
|
||||
-type d | xargs rm -rf
|
||||
mv zuul.d/zuul-legacy-* ../openstack-zuul-jobs/zuul.d/
|
||||
|
|
|
|||
|
|
@ -42,6 +42,9 @@ from jenkins_jobs.parser import matches
|
|||
import jenkins_jobs.parser
|
||||
import yaml
|
||||
|
||||
JOB_MATCHERS = {} # type: Dict[str, Dict[str, Dict]]
|
||||
TEMPLATES_TO_EXPAND = {} # type: Dict[str, List]
|
||||
JOBS_FOR_EXPAND = collections.defaultdict(dict) # type: ignore
|
||||
JOBS_BY_ORIG_TEMPLATE = {} # type: ignore
|
||||
SUFFIXES = [] # type: ignore
|
||||
ENVIRONMENT = '{{ zuul | zuul_legacy_vars }}'
|
||||
|
|
@ -186,6 +189,37 @@ def merge_project_dict(project_dicts, name, project):
|
|||
return
|
||||
|
||||
|
||||
def normalize_project_expansions():
|
||||
remove_from_job_matchers = []
|
||||
template = None
|
||||
# First find the matchers that are the same for all jobs
|
||||
for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items():
|
||||
JOB_MATCHERS[job_name] = None
|
||||
for project_name, expansion in project.items():
|
||||
template = expansion['template']
|
||||
if not JOB_MATCHERS[job_name]:
|
||||
JOB_MATCHERS[job_name] = copy.deepcopy(expansion['info'])
|
||||
else:
|
||||
if JOB_MATCHERS[job_name] != expansion['info']:
|
||||
# We have different expansions for this job, it can't be
|
||||
# done at the job level
|
||||
remove_from_job_matchers.append(job_name)
|
||||
|
||||
for job_name in remove_from_job_matchers:
|
||||
JOB_MATCHERS.pop(job_name, None)
|
||||
|
||||
# Second, find out which projects need to expand a given template
|
||||
for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items():
|
||||
# There is a job-level expansion for this one
|
||||
if job_name in JOB_MATCHERS.keys():
|
||||
continue
|
||||
for project_name, expansion in project.items():
|
||||
TEMPLATES_TO_EXPAND[project_name] = []
|
||||
if expansion['info']:
|
||||
# There is an expansion for this project
|
||||
TEMPLATES_TO_EXPAND[project_name].append(expansion['template'])
|
||||
|
||||
|
||||
# from :
|
||||
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data flake8: noqa
|
||||
def should_use_block(value):
|
||||
|
|
@ -910,6 +944,14 @@ class Job:
|
|||
if expanded_projects:
|
||||
output['required-projects'] = sorted(list(set(expanded_projects)))
|
||||
|
||||
if self.name in JOB_MATCHERS:
|
||||
for k, v in JOB_MATCHERS[self.name].items():
|
||||
if k in output:
|
||||
self.log.error(
|
||||
'Job %s has attributes directly and from matchers',
|
||||
self.name)
|
||||
output[k] = v
|
||||
|
||||
return output
|
||||
|
||||
def toPipelineDict(self):
|
||||
|
|
@ -1345,7 +1387,7 @@ class ZuulMigrate:
|
|||
for pipeline, value in template.items():
|
||||
if pipeline == 'name':
|
||||
continue
|
||||
if pipeline not in project:
|
||||
if pipeline not in project or 'jobs' not in project[pipeline]:
|
||||
project[pipeline] = dict(jobs=[])
|
||||
project[pipeline]['jobs'].extend(value['jobs'])
|
||||
|
||||
|
|
@ -1355,7 +1397,7 @@ class ZuulMigrate:
|
|||
return job.orig
|
||||
return None
|
||||
|
||||
def applyProjectMatchers(self, matchers, project):
|
||||
def applyProjectMatchers(self, matchers, project, final=False):
|
||||
'''
|
||||
Apply per-project job matchers to the given project.
|
||||
|
||||
|
|
@ -1373,7 +1415,8 @@ class ZuulMigrate:
|
|||
self.log.debug(
|
||||
"Applied irrelevant-files to job %s in project %s",
|
||||
job, project['name'])
|
||||
job = {job: {'irrelevant-files': list(set(files))}}
|
||||
job = {job: {'irrelevant-files':
|
||||
sorted(list(set(files)))}}
|
||||
elif isinstance(job, dict):
|
||||
job = job.copy()
|
||||
job_name = get_single_key(job)
|
||||
|
|
@ -1387,8 +1430,8 @@ class ZuulMigrate:
|
|||
if 'irrelevant-files' not in extras:
|
||||
extras['irrelevant-files'] = []
|
||||
extras['irrelevant-files'].extend(files)
|
||||
extras['irrelevant-files'] = list(
|
||||
set(extras['irrelevant-files']))
|
||||
extras['irrelevant-files'] = sorted(list(
|
||||
set(extras['irrelevant-files'])))
|
||||
job[job_name] = extras
|
||||
new_jobs.append(job)
|
||||
return new_jobs
|
||||
|
|
@ -1398,17 +1441,61 @@ class ZuulMigrate:
|
|||
if k in ('templates', 'name'):
|
||||
continue
|
||||
project[k]['jobs'] = processPipeline(
|
||||
project[k]['jobs'], job_name_regex, files)
|
||||
project[k].get('jobs', []), job_name_regex, files)
|
||||
|
||||
for matcher in matchers:
|
||||
# find the project-specific section
|
||||
for skipper in matcher.get('skip-if', []):
|
||||
if skipper.get('project'):
|
||||
if re.search(skipper['project'], project['name']):
|
||||
if 'all-files-match-any' in skipper:
|
||||
applyIrrelevantFiles(
|
||||
matcher['name'],
|
||||
skipper['all-files-match-any'])
|
||||
if matchers:
|
||||
for matcher in matchers:
|
||||
# find the project-specific section
|
||||
for skipper in matcher.get('skip-if', []):
|
||||
if skipper.get('project'):
|
||||
if re.search(skipper['project'], project['name']):
|
||||
if 'all-files-match-any' in skipper:
|
||||
applyIrrelevantFiles(
|
||||
matcher['name'],
|
||||
skipper['all-files-match-any'])
|
||||
|
||||
if not final:
|
||||
return
|
||||
|
||||
for k, v in project.items():
|
||||
if k in ('templates', 'name'):
|
||||
continue
|
||||
jobs = []
|
||||
for job in project[k].get('jobs', []):
|
||||
if isinstance(job, dict):
|
||||
job_name = get_single_key(job)
|
||||
else:
|
||||
job_name = job
|
||||
if job_name in JOB_MATCHERS:
|
||||
jobs.append(job)
|
||||
continue
|
||||
orig_name = self.getOldJobName(job_name)
|
||||
if not orig_name:
|
||||
jobs.append(job)
|
||||
continue
|
||||
orig_name = orig_name.format(
|
||||
name=project['name'].split('/')[1])
|
||||
info = {}
|
||||
for layout_job in self.mapping.layout.get('jobs', []):
|
||||
if 'parameter-function' in layout_job:
|
||||
continue
|
||||
if 'skip-if' in layout_job:
|
||||
continue
|
||||
if re.search(layout_job['name'], orig_name):
|
||||
if not layout_job.get('voting', True):
|
||||
info['voting'] = False
|
||||
if layout_job.get('branch'):
|
||||
info['branches'] = layout_job['branch']
|
||||
if layout_job.get('files'):
|
||||
info['files'] = layout_job['files']
|
||||
if not isinstance(job, dict):
|
||||
job = {job: info}
|
||||
else:
|
||||
job[job_name].update(info)
|
||||
|
||||
jobs.append(job)
|
||||
if jobs:
|
||||
project[k]['jobs'] = jobs
|
||||
|
||||
def writeProject(self, project):
|
||||
'''
|
||||
|
|
@ -1423,12 +1510,7 @@ class ZuulMigrate:
|
|||
if 'name' in project:
|
||||
new_project['name'] = project['name']
|
||||
|
||||
job_matchers = self.scanForProjectMatchers(project['name'])
|
||||
if job_matchers:
|
||||
exp_template_names = self.findReferencedTemplateNames(
|
||||
job_matchers, project['name'])
|
||||
else:
|
||||
exp_template_names = []
|
||||
exp_template_names = TEMPLATES_TO_EXPAND.get(project['name'], [])
|
||||
|
||||
templates_to_expand = []
|
||||
if 'template' in project:
|
||||
|
|
@ -1440,6 +1522,51 @@ class ZuulMigrate:
|
|||
new_project['templates'].append(
|
||||
self.mapping.getNewTemplateName(template['name']))
|
||||
|
||||
for key, value in project.items():
|
||||
if key in ('name', 'template'):
|
||||
continue
|
||||
else:
|
||||
new_project[key] = collections.OrderedDict()
|
||||
if key == 'gate':
|
||||
for queue in self.change_queues:
|
||||
if (project['name'] not in queue.getProjects() or
|
||||
len(queue.getProjects()) == 1):
|
||||
continue
|
||||
new_project[key]['queue'] = queue.name
|
||||
tmp = [job for job in self.makeNewJobs(value)]
|
||||
# Don't insert into self.job_objects - that was done
|
||||
# in the speculative pass
|
||||
jobs = [job.toPipelineDict() for job in tmp]
|
||||
if jobs:
|
||||
new_project[key]['jobs'] = jobs
|
||||
if not new_project[key]:
|
||||
del new_project[key]
|
||||
|
||||
for name in templates_to_expand:
|
||||
self.expandTemplateIntoProject(name, new_project)
|
||||
|
||||
job_matchers = self.scanForProjectMatchers(project['name'])
|
||||
|
||||
# Need a deep copy after expansion, else our templates end up
|
||||
# also getting this change.
|
||||
new_project = copy.deepcopy(new_project)
|
||||
self.applyProjectMatchers(job_matchers, new_project, final=True)
|
||||
|
||||
return new_project
|
||||
|
||||
def checkSpeculativeProject(self, project):
|
||||
'''
|
||||
Create a new v3 project definition expanding all templates.
|
||||
'''
|
||||
new_project = collections.OrderedDict()
|
||||
if 'name' in project:
|
||||
new_project['name'] = project['name']
|
||||
|
||||
templates_to_expand = []
|
||||
for template in project.get('template', []):
|
||||
templates_to_expand.append(template['name'])
|
||||
|
||||
# We have to do this section to expand self.job_objects
|
||||
for key, value in project.items():
|
||||
if key in ('name', 'template'):
|
||||
continue
|
||||
|
|
@ -1454,18 +1581,60 @@ class ZuulMigrate:
|
|||
new_project[key]['queue'] = queue.name
|
||||
tmp = [job for job in self.makeNewJobs(value)]
|
||||
self.job_objects.extend(tmp)
|
||||
jobs = [job.toPipelineDict() for job in tmp]
|
||||
new_project[key]['jobs'] = jobs
|
||||
|
||||
for name in templates_to_expand:
|
||||
self.expandTemplateIntoProject(name, new_project)
|
||||
|
||||
# Need a deep copy after expansion, else our templates end up
|
||||
# also getting this change.
|
||||
new_project = copy.deepcopy(new_project)
|
||||
self.applyProjectMatchers(job_matchers, new_project)
|
||||
expand_project = copy.deepcopy(new_project)
|
||||
self.expandTemplateIntoProject(name, expand_project)
|
||||
|
||||
return new_project
|
||||
# Need a deep copy after expansion, else our templates end up
|
||||
# also getting this change.
|
||||
expand_project = copy.deepcopy(expand_project)
|
||||
job_matchers = self.scanForProjectMatchers(project['name'])
|
||||
self.applyProjectMatchers(job_matchers, expand_project)
|
||||
|
||||
# We should now have a project-pipeline with only the
|
||||
# jobs expanded from this one template
|
||||
for project_part in expand_project.values():
|
||||
# The pipelines are dicts - we only want pipelines
|
||||
if isinstance(project_part, dict):
|
||||
if 'jobs' not in project_part:
|
||||
continue
|
||||
self.processProjectTemplateExpansion(
|
||||
project_part, project, name)
|
||||
|
||||
def processProjectTemplateExpansion(self, project_part, project, template):
|
||||
# project_part should be {'jobs': []}
|
||||
job_list = project_part['jobs']
|
||||
for new_job in job_list:
|
||||
if isinstance(new_job, dict):
|
||||
new_job_name = get_single_key(new_job)
|
||||
info = new_job[new_job_name]
|
||||
else:
|
||||
new_job_name = new_job
|
||||
info = None
|
||||
orig_name = self.getOldJobName(new_job_name)
|
||||
if not orig_name:
|
||||
self.log.error("Job without old name: %s", new_job_name)
|
||||
continue
|
||||
orig_name = orig_name.format(name=project['name'].split('/')[1])
|
||||
|
||||
for layout_job in self.mapping.layout.get('jobs', []):
|
||||
if 'parameter-function' in layout_job:
|
||||
continue
|
||||
if re.search(layout_job['name'], orig_name):
|
||||
if not info:
|
||||
info = {}
|
||||
if not layout_job.get('voting', True):
|
||||
info['voting'] = False
|
||||
if layout_job.get('branch'):
|
||||
info['branches'] = layout_job['branch']
|
||||
if layout_job.get('files'):
|
||||
info['files'] = layout_job['files']
|
||||
|
||||
if info:
|
||||
expansion = dict(info=info, template=template)
|
||||
JOBS_FOR_EXPAND[new_job_name][project['name']] = expansion
|
||||
|
||||
def writeJobs(self):
|
||||
output_dir = self.setupDir()
|
||||
|
|
@ -1487,13 +1656,17 @@ class ZuulMigrate:
|
|||
template_config,
|
||||
key=lambda template: template['project-template']['name'])
|
||||
|
||||
for project in self.layout.get('projects', []):
|
||||
self.checkSpeculativeProject(project)
|
||||
normalize_project_expansions()
|
||||
|
||||
project_names = []
|
||||
for project in self.layout.get('projects', []):
|
||||
project_names.append(project['name'])
|
||||
project_dict = self.writeProject(project)
|
||||
merge_project_dict(
|
||||
project_dicts, project['name'],
|
||||
self.writeProject(project))
|
||||
project_dict)
|
||||
project_config = project_dicts_to_list(project_dicts)
|
||||
|
||||
seen_jobs = []
|
||||
|
|
|
|||
Loading…
Reference in New Issue