Emit shell instead of script tasks

Unmatched single quotes is what killed us before. Add an option which
does a test with ansible-playbook --syntax-check to validate whether a
playbook emitted with shell instead of script works.

Running that way takes around 90 minutes. So default to shell, but
provide --syntax-check as a way to check using shell and fallback to
script. This can then be used to fix pre-migration jobs if desired.

Also separate task entries with an extra '\n' for readability.

And only collect the generated playbooks - not all of them.

Change-Id: I147a7562e6578ab300ef56ae7b1c9d9d3f8f920f
This commit is contained in:
Monty Taylor 2017-09-18 11:22:51 -05:00
parent 661433409f
commit feec38afe6
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
2 changed files with 140 additions and 27 deletions

View File

@ -19,8 +19,8 @@
- name: Collect generated playbooks
synchronize:
dest: "{{ zuul.executor.log_root }}"
dest: "{{ zuul.executor.log_root }}/playbooks"
mode: pull
src: "src/git.openstack.org/openstack-infra/project-config/playbooks"
src: "src/git.openstack.org/openstack-infra/project-config/playbooks/legacy"
verify_host: true
no_log: true

View File

@ -26,8 +26,11 @@ import argparse
import collections
import copy
import itertools
import getopt
import logging
import os
import subprocess
import tempfile
import re
from typing import Any, Dict, List, Optional # flake8: noqa
@ -40,6 +43,7 @@ import yaml
JOBS_BY_ORIG_TEMPLATE = {} # type: ignore
SUFFIXES = [] # type: ignore
ENVIRONMENT = '{{ host_vars[inventory_hostname] | zuul_legacy_vars }}'
DESCRIPTION = """Migrate zuul v2 and Jenkins Job Builder to Zuul v3.
This program takes a zuul v2 layout.yaml and a collection of Jenkins Job
@ -48,6 +52,65 @@ optional mapping config can be given that defines how to map old jobs
to new jobs.
"""
def deal_with_shebang(data):
# Ansible shell blocks do not honor shebang lines. That's fine - but
# we do have a bunch of scripts that have either nothing, -x, -xe,
# -ex or -eux. Transform those into leading set commands
if not data.startswith('#!'):
return (None, data)
data_lines = data.split('\n')
data_lines.reverse()
shebang = data_lines.pop()
split_line = shebang.split()
# Strip the # and the !
executable = split_line[0][2:]
if executable == '/bin/sh':
# Ansible default
executable = None
if len(split_line) > 1:
flag_x = False
flag_e = False
flag_u = False
optlist, args = getopt.getopt(split_line[1:], 'uex')
for opt, _ in optlist:
if opt == '-x':
flag_x = True
elif opt == '-e':
flag_e = True
elif opt == '-u':
flag_u = True
if flag_x:
data_lines.append('set -x')
if flag_e:
data_lines.append('set -e')
if flag_u:
data_lines.append('set -u')
data_lines.reverse()
data = '\n'.join(data_lines)
return (executable, data)
# from:
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data flake8: noqa
def should_use_block(value):
for c in u"\u000a\u000d\u001c\u001d\u001e\u0085\u2028\u2029":
if c in value:
return True
return False
def my_represent_scalar(self, tag, value, style=None):
if style is None:
if should_use_block(value):
style='|'
else:
style = self.default_style
node = yaml.representer.ScalarNode(tag, value, style=style)
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
return node
def project_representer(dumper, data):
return dumper.represent_mapping('tag:yaml.org,2002:map',
@ -121,8 +184,14 @@ def ordered_dump(data, stream=None, *args, **kwargs):
# works. Without it, we end up with YAML references to the expanded jobs.
dumper.ignore_aliases = lambda self, data: True
return yaml.dump(data, stream=stream, default_flow_style=False,
Dumper=dumper, width=80, *args, **kwargs)
output = yaml.dump(
data, default_flow_style=False,
Dumper=dumper, width=80, *args, **kwargs).replace(
'\n -', '\n\n -')
if stream:
stream.write(output)
else:
return output
def get_single_key(var):
@ -496,24 +565,62 @@ class Job:
tasks.insert(0, ensure_task)
return dict(tasks=tasks, artifacts=artifacts, draft=draft)
def _makeBuilderTask(self, playbook_dir, builder, sequence):
script_fn = '%s-%02d.sh' % (self.short_name, sequence)
def _emitShellTask(self, data, syntax_check):
shell, data = deal_with_shebang(data)
task = collections.OrderedDict()
task['shell'] = data
if shell:
task['args'] = dict(executable=shell)
if syntax_check:
# Emit a test playbook with this shell task in it then run
# ansible-playbook --syntax-check on it. This will fail if there
# are embedding issues, such as with unbalanced single quotes
# The end result should be less scripts and more shell
play = dict(hosts='all', tasks=[task])
(fd, tmp_path) = tempfile.mkstemp()
try:
f = os.fdopen(fd, 'w')
ordered_dump([play], f)
f.close()
proc = subprocess.run(
['ansible-playbook', '--syntax-check', tmp_path],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if proc.returncode != 0:
# Return of None means we must emit a script
self.log.error(
"Job {name} had an ansible syntax error, using script"
" instead of shell".format(name=self.name))
return None
finally:
os.unlink(tmp_path)
return task
def _emitScriptContent(self, data, playbook_dir, seq):
script_fn = '%s-%02d.sh' % (self.short_name, seq)
script_path = os.path.join(playbook_dir, script_fn)
# Don't write a script to echo the template line
if builder['shell'].startswith('echo JJB template: '):
return
with open(script_path, 'w') as script:
data = builder['shell']
if not data.startswith('#!'):
data = '#!/bin/bash -x\n %s' % (data,)
script.write(data)
task = collections.OrderedDict()
task['name'] = 'Builder script {seq} translated from {old}'.format(
seq=sequence, old=self.orig)
task['name'] = 'Running playbooks/legacy/{playbook}'.format(
playbook=script_fn)
task['script'] = script_fn
task['environment'] = (
'{{ host_vars[inventory_hostname] | zuul_legacy_vars }}')
return task
def _makeBuilderTask(self, playbook_dir, builder, sequence, syntax_check):
# Don't write a script to echo the template line
if builder['shell'].startswith('echo JJB template: '):
return
task = self._emitShellTask(builder['shell'], syntax_check)
if not task:
task = self._emitScriptContent(
builder['shell'], playbook_dir, sequence)
task['environment'] = ENVIRONMENT
return task
def _transformPublishers(self, jjb_job):
@ -544,7 +651,7 @@ class Job:
self.log.debug("Transformed job publishers")
return early_publishers, late_publishers
def emitPlaybooks(self, jobsdir):
def emitPlaybooks(self, jobsdir, syntax_check=False):
has_artifacts = False
has_draft = False
if not self.jjb_job:
@ -561,19 +668,20 @@ class Job:
post_playbook = os.path.join(self.job_path, 'post.yaml')
tasks = []
sequence = 0
tasks.append(collections.OrderedDict(
debug=collections.OrderedDict(
msg='Autoconverted job {name} from old job {old}'.format(
name=self.name, old=self.orig))))
for builder in self.jjb_job.get('builders', []):
if 'shell' in builder:
task = self._makeBuilderTask(playbook_dir, builder, sequence)
task = self._makeBuilderTask(
playbook_dir, builder, sequence, syntax_check)
if task:
sequence += 1
if 'script' in task:
sequence += 1
tasks.append(task)
play = collections.OrderedDict(
hosts='all',
tasks=tasks)
play = collections.OrderedDict()
play['hosts'] = 'all'
play['name'] = 'Autoconverted job {name} from old job {old}'.format(
name=self.name, old=self.orig)
play['tasks'] = tasks
with open(run_playbook, 'w') as run_playbook_out:
ordered_dump([play], run_playbook_out)
@ -825,12 +933,13 @@ class ZuulMigrate:
log = logging.getLogger("zuul.Migrate")
def __init__(self, layout, job_config, nodepool_config,
outdir, mapping, move):
outdir, mapping, move, syntax_check):
self.layout = ordered_load(open(layout, 'r'))
self.job_config = job_config
self.outdir = outdir
self.mapping = JobMapping(nodepool_config, self.layout, mapping)
self.move = move
self.syntax_check = syntax_check
self.jobs = {}
self.old_jobs = {}
@ -1199,7 +1308,7 @@ class ZuulMigrate:
job.name not in self.mapping.seen_new_jobs and
job.emit):
has_artifacts, has_post, has_draft = job.emitPlaybooks(
self.outdir)
self.outdir, self.syntax_check)
job_config.append({'job': job.toJobDict(
has_artifacts, has_post, has_draft)})
seen_jobs.append(job.name)
@ -1219,6 +1328,7 @@ def main():
yaml.add_representer(collections.OrderedDict, project_representer,
Dumper=IndentedDumper)
yaml.representer.BaseRepresenter.represent_scalar = my_represent_scalar
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument(
@ -1239,6 +1349,9 @@ def main():
help="A filename with a yaml mapping of old name to new name.")
parser.add_argument(
'-v', dest='verbose', action='store_true', help='verbose output')
parser.add_argument(
'--syntax-check', dest='syntax_check', action='store_true',
help='Run ansible-playbook --syntax-check on generated playbooks')
parser.add_argument(
'-m', dest='move', action='store_true',
help='Move zuul.yaml to zuul.d if it exists')
@ -1250,7 +1363,7 @@ def main():
logging.basicConfig(level=logging.INFO)
ZuulMigrate(args.layout, args.job_config, args.nodepool_config,
args.outdir, args.mapping, args.move).run()
args.outdir, args.mapping, args.move, args.syntax_check).run()
if __name__ == '__main__':