Merge "upload-logs-swift: Make indexer more generic"

This commit is contained in:
Zuul 2018-11-30 04:50:11 +00:00 committed by Gerrit Code Review
commit 2ea7a1b6bd
2 changed files with 99 additions and 91 deletions

View File

@ -257,10 +257,10 @@ class TestFileList(testtools.TestCase):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
ix = Indexer(fl,
create_parent_links=True,
create_topdir_parent_link=True)
ix.make_indexes()
ix = Indexer(fl)
ix.make_indexes(
create_parent_links=True,
create_topdir_parent_link=True)
self.assert_files(fl, [
('', 'application/directory', None),
@ -312,10 +312,10 @@ class TestFileList(testtools.TestCase):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
ix = Indexer(fl,
create_parent_links=False,
create_topdir_parent_link=False)
ix.make_indexes()
ix = Indexer(fl)
ix.make_indexes(
create_parent_links=False,
create_topdir_parent_link=False)
self.assert_files(fl, [
('', 'application/directory', None),

View File

@ -282,87 +282,23 @@ class FileList(Sequence):
class Indexer():
"""generates index.html files if requested."""
"""Index a FileList
def __init__(self, file_list, create_parent_links=True,
create_topdir_parent_link=False,
append_footer='index_footer.html'):
Functions to generate indexes and other collated data for a
FileList
- make_indexes() : make index.html in folders
"""
def __init__(self, file_list):
'''
Args:
file_list (FileList): A FileList object to be updated
with index files for each directory.
create_parent_links (bool):
create_topdir_parent_link (bool):
append_footer (str):
file_list (FileList): A FileList object with all files
to be indexed.
'''
assert isinstance(file_list, FileList)
self.file_list = file_list
self.create_parent_links = create_parent_links
self.create_topdir_parent_link = create_topdir_parent_link
self.append_footer = append_footer
self.index_filename = 'index.html'
def make_indexes(self):
'''Make index files
Return:
No value, the self.file_list will be updated
'''
folders = collections.OrderedDict()
for f in self.file_list:
if f.folder:
folders[f.relative_path] = []
folder = os.path.dirname(os.path.dirname(
f.relative_path + '/'))
if folder == '/':
folder = ''
else:
folder = os.path.dirname(f.relative_path)
folders[folder].append(f)
indexes = {}
parent_file_detail = FileDetail(None, '..', '..')
for folder, files in folders.items():
# Don't add the pseudo-top-directory
if files and files[0].full_path is None:
files = files[1:]
if self.create_topdir_parent_link:
files = [parent_file_detail] + files
elif self.create_parent_links:
files = [parent_file_detail] + files
# Do generate a link to the parent directory
full_path = self.make_index_file(files, 'Index of %s' % (folder,),
self.file_list.get_tempdir())
if full_path:
filename = os.path.basename(full_path)
relative_name = os.path.join(folder, filename)
indexes[folder] = FileDetail(full_path, relative_name)
# This appends the index file at the end of the group of files
# for each directory.
new_list = []
last_dirname = None
for f in reversed(list(self.file_list)):
if f.folder:
relative_path = f.relative_path + '/'
else:
relative_path = f.relative_path
dirname = os.path.dirname(relative_path)
if dirname == '/':
dirname = ''
if dirname != last_dirname:
index = indexes.pop(dirname, None)
if index:
new_list.append(index)
last_dirname = dirname
new_list.append(f)
new_list.reverse()
self.file_list.file_list = new_list
def make_index_file(self, folder_links, title, tempdir):
def _make_index_file(self, folder_links, title, tempdir, append_footer):
"""Writes an index into a file for pushing"""
for file_details in folder_links:
# Do not generate an index file if one exists already.
@ -370,12 +306,13 @@ class Indexer():
# content like python coverage info.
if self.index_filename == file_details.filename:
return
index_content = self.generate_log_index(folder_links, title)
index_content = self._generate_log_index(
folder_links, title, append_footer)
fd = open(os.path.join(tempdir, self.index_filename), 'w')
fd.write(index_content)
return os.path.join(tempdir, self.index_filename)
def generate_log_index(self, folder_links, title):
def _generate_log_index(self, folder_links, title, append_footer):
"""Create an index of logfiles and links to them"""
output = '<html><head><title>%s</title></head><body>\n' % title
@ -403,8 +340,8 @@ class Indexer():
output += '<td style="text-align: right">%s</td>' % size
output += '</tr>\n'
if (self.append_footer and
self.append_footer in file_details.filename):
if (append_footer and
append_footer in file_details.filename):
file_details_to_append = file_details
output += '</table>'
@ -420,6 +357,78 @@ class Indexer():
output += '</body></html>\n'
return output
def make_indexes(self, create_parent_links=True,
create_topdir_parent_link=False,
append_footer='index_footer.html'):
'''Make index.html files
Iterate the file list and crete index.html files for folders
Args:
create_parent_links (bool): Create parent links
create_topdir_parent_link (bool): Create topdir parent link
append_footer (str): Filename of a footer to append to each
generated page
Return:
No value, the self.file_list will be updated
'''
self.index_filename = 'index.html'
folders = collections.OrderedDict()
for f in self.file_list:
if f.folder:
folders[f.relative_path] = []
folder = os.path.dirname(os.path.dirname(
f.relative_path + '/'))
if folder == '/':
folder = ''
else:
folder = os.path.dirname(f.relative_path)
folders[folder].append(f)
indexes = {}
parent_file_detail = FileDetail(None, '..', '..')
for folder, files in folders.items():
# Don't add the pseudo-top-directory
if files and files[0].full_path is None:
files = files[1:]
if create_topdir_parent_link:
files = [parent_file_detail] + files
elif create_parent_links:
files = [parent_file_detail] + files
# Do generate a link to the parent directory
full_path = self._make_index_file(files, 'Index of %s' % (folder,),
self.file_list.get_tempdir(),
append_footer)
if full_path:
filename = os.path.basename(full_path)
relative_name = os.path.join(folder, filename)
indexes[folder] = FileDetail(full_path, relative_name)
# This appends the index file at the end of the group of files
# for each directory.
new_list = []
last_dirname = None
for f in reversed(list(self.file_list)):
if f.folder:
relative_path = f.relative_path + '/'
else:
relative_path = f.relative_path
dirname = os.path.dirname(relative_path)
if dirname == '/':
dirname = ''
if dirname != last_dirname:
index = indexes.pop(dirname, None)
if index:
new_list.append(index)
last_dirname = dirname
new_list.append(f)
new_list.reverse()
self.file_list.file_list = new_list
class DeflateFilter():
chunk_size = 16384
@ -585,18 +594,17 @@ def run(cloud, container, files,
# Create the objects to make sure the arguments are sound.
with FileList() as file_list:
indexer = Indexer(file_list,
create_parent_links=parent_links,
create_topdir_parent_link=topdir_parent_link,
append_footer=footer)
# Scan the files.
for file_path in files:
file_list.add(file_path)
indexer = Indexer(file_list)
# (Possibly) make indexes.
if indexes:
indexer.make_indexes()
indexer.make_indexes(create_parent_links=parent_links,
create_topdir_parent_link=topdir_parent_link,
append_footer=footer)
logging.debug("List of files prepared to upload:")
for x in file_list: