zuul
/
zuul-jobs
Code Issues Proposed changes

Enable options for generating the parent links 

This change restores the --no-parent-links option that was dropped at some
point.

It also adds a new --create-topdir-parent-link option which defaults to
False. This is a separate option to hte parent links as it creates a
relative link to '../' from the results of the logs. This makes sense if it
is going back to some kind of global index of all the job results, but in
most cases if logs are in containers behind some kind of CDN it's likely
the entrypoint into the logs is going to be from the zuul dashboard (or
comments in the review system). Thus default to False.

Change-Id: I3b5ef4ac81bd7dbc9e65a5d22d81198b6642fa58
This commit is contained in:
Joshua Hesketh 2018-07-31 18:46:50 +10:00 committed by James E. Blair
parent 6de6e20609
commit b6756077c8
2 changed files with 122 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
roles/wip-upload-logs-swift/library/test_zuul_swift_upload.py
View File

@ -131,12 +131,10 @@ class TestFileList(unittest.TestCase):
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0].find('a').get('href'), '../')
self.assertEqual(rows[0].find('a').text, '../')
self.assertEqual(len(rows), 1)
self.assertEqual(rows[1].find('a').get('href'), 'logs/')
self.assertEqual(rows[1].find('a').text, 'logs/')
self.assertEqual(rows[0].find('a').get('href'), 'logs/')
self.assertEqual(rows[0].find('a').text, 'logs/')
subdir_index = self.find_file(fl, 'logs/controller/subdir/index.html')
page = open(subdir_index.full_path).read()
@ -178,6 +176,59 @@ class TestFileList(unittest.TestCase):
page = open(top_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0].find('a').get('href'), 'controller/')
self.assertEqual(rows[0].find('a').text, 'controller/')
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
page = open(subdir_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
self.assertEqual(rows[0].find('a').get('href'), '../')
self.assertEqual(rows[0].find('a').text, '../')
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
def test_topdir_parent_link(self):
'''Test index generation creates topdir parent link'''
fl = FileList()
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
ix = Indexer(create_parent_links=True,
create_topdir_parent_link=True)
fl = ix.make_indexes(fl)
self.assert_files(fl, [
('', 'application/directory', None),
('controller', 'application/directory', None),
('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None),
('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/journal.xz', 'text/plain', 'xz'),
('controller/service_log.txt', 'text/plain', None),
('controller/syslog', 'text/plain', None),
('controller/index.html', 'text/html', None),
('controller/subdir/subdir.txt', 'text/plain', None),
('controller/subdir/index.html', 'text/html', None),
('zuul-info/inventory.yaml', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
('zuul-info/index.html', 'text/html', None),
])
top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0].find('a').get('href'), '../')
self.assertEqual(rows[0].find('a').text, '../')
@ -196,3 +247,51 @@ class TestFileList(unittest.TestCase):
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
def test_no_parent_links(self):
'''Test index generation creates topdir parent link'''
fl = FileList()
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
ix = Indexer(create_parent_links=False,
create_topdir_parent_link=False)
fl = ix.make_indexes(fl)
self.assert_files(fl, [
('', 'application/directory', None),
('controller', 'application/directory', None),
('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None),
('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/journal.xz', 'text/plain', 'xz'),
('controller/service_log.txt', 'text/plain', None),
('controller/syslog', 'text/plain', None),
('controller/index.html', 'text/html', None),
('controller/subdir/subdir.txt', 'text/plain', None),
('controller/subdir/index.html', 'text/html', None),
('zuul-info/inventory.yaml', 'text/plain', None),
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
('zuul-info/index.html', 'text/html', None),
])
top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0].find('a').get('href'), 'controller/')
self.assertEqual(rows[0].find('a').text, 'controller/')
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
page = open(subdir_index.full_path).read()
page = BeautifulSoup(page, 'html.parser')
rows = page.find_all('tr')[1:]
self.assertEqual(rows[0].find('a').get('href'), 'subdir.txt')
self.assertEqual(rows[0].find('a').text, 'subdir.txt')

24
roles/wip-upload-logs-swift/library/zuul_swift_upload.py
View File

@ -179,8 +179,10 @@ class Indexer(object):
"""generates index.html files if requested."""
def __init__(self, create_parent_links=True,
create_topdir_parent_link=False,
append_footer='index_footer.html'):
self.create_parent_links = create_parent_links
self.create_topdir_parent_link = create_topdir_parent_link
self.append_footer = append_footer
self.index_filename = 'index.html'
@ -203,10 +205,13 @@ class Indexer(object):
# Don't add the pseudo-top-directory
if files and files[0].full_path is None:
files = files[1:]
if self.create_topdir_parent_link:
files = [parent_file_detail] + files
elif self.create_parent_links:
files = [parent_file_detail] + files
# Do generate a link to the parent directory
full_path = self.make_index_file([parent_file_detail] + files,
'Index of %s' % (folder,))
full_path = self.make_index_file(files, 'Index of %s' % (folder,))
if full_path:
filename = os.path.basename(full_path)
@ -449,10 +454,9 @@ class Uploader(object):
def run(cloud, container, files,
indexes=True, parent_links=True, partition=False,
footer='index_footer.html',
delete_after=15552000, prefix=None,
public=True, dry_run=False):
indexes=True, parent_links=True, topdir_parent_link=False,
partition=False, footer='index_footer.html', delete_after=15552000,
prefix=None, public=True, dry_run=False):
if prefix:
prefix = prefix.lstrip('/')
@ -465,6 +469,7 @@ def run(cloud, container, files,
# Create the objects to make sure the arguments are sound.
file_list = FileList()
indexer = Indexer(create_parent_links=parent_links,
create_topdir_parent_link=topdir_parent_link,
append_footer=footer)
# Scan the files.
@ -500,6 +505,7 @@ def ansible_main():
partition=dict(type='bool', default=False),
indexes=dict(type='bool', default=True),
parent_links=dict(type='bool', default=True),
topdir_parent_link=dict(type='bool', default=False),
public=dict(type='bool', default=True),
footer=dict(type='str'),
delete_after=dict(type='int'),
@ -511,6 +517,7 @@ def ansible_main():
url = run(p.get('cloud'), p.get('container'), p.get('files'),
indexes=p.get('indexes'),
parent_links=p.get('parent_links'),
topdir_parent_link=p.get('topdir_parent_link'),
partition=p.get('partition'),
footer=p.get('footer'),
delete_after=p.get('delete_after', 15552000),
@ -531,6 +538,10 @@ def cli_main():
help='do not generate any indexes at all')
parser.add_argument('--no-parent-links', action='store_true',
help='do not include links back to a parent dir')
parser.add_argument('--create-topdir-parent-link', action='store_true',
help='include a link in the root directory of the '
'files to the parent directory which may be the '
'index of all results')
parser.add_argument('--no-public', action='store_true',
help='do not create the container as public')
parser.add_argument('--partition', action='store_true',
@ -573,6 +584,7 @@ def cli_main():
url = run(args.cloud, args.container, args.files,
indexes=not args.no_indexes,
parent_links=not args.no_parent_links,
topdir_parent_link=args.create_topdir_parent_link,
partition=args.partition,
footer=append_footer,
delete_after=args.delete_after,