summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorIan Wienand <iwienand@redhat.com>2018-08-17 14:17:20 +1000
committerIan Wienand <iwienand@redhat.com>2018-11-28 15:47:48 +1100
commit2d61e9d4fc32b85a30afc05c12e4fd3a4e40c1ed (patch)
treebb85b29cbf4f3a7ff4e291e6041ddef0f75458ef
parent928d0bb051b02538f2c97f3357f4c9b3b1b454f7 (diff)
upload-logs-swift: Make indexer more generic
This converts the Indexer class from something that strictly generates index.html files for folders to a more generic class that will be able to hold other types of transformations we might like to do on the total collections of uploaded files. The index.html specific arguments are moved into make_indexes() and the two helper functions that should not be called externally are renamed private. Change-Id: I388042ffb6a74c3200d92fb3a084369fcf2cf3a9
Notes
Notes (review): Code-Review+2: Joshua Hesketh <josh@nitrotech.org> Code-Review+2: Andreas Jaeger <jaegerandi@gmail.com> Workflow+1: Andreas Jaeger <jaegerandi@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Fri, 30 Nov 2018 04:50:11 +0000 Reviewed-on: https://review.openstack.org/592852 Project: openstack-infra/zuul-jobs Branch: refs/heads/master
-rw-r--r--roles/upload-logs-swift/library/test_zuul_swift_upload.py16
-rwxr-xr-xroles/upload-logs-swift/library/zuul_swift_upload.py174
2 files changed, 99 insertions, 91 deletions
diff --git a/roles/upload-logs-swift/library/test_zuul_swift_upload.py b/roles/upload-logs-swift/library/test_zuul_swift_upload.py
index be69c78..39f23b4 100644
--- a/roles/upload-logs-swift/library/test_zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/test_zuul_swift_upload.py
@@ -257,10 +257,10 @@ class TestFileList(testtools.TestCase):
257 '''Test index generation creates topdir parent link''' 257 '''Test index generation creates topdir parent link'''
258 with FileList() as fl: 258 with FileList() as fl:
259 fl.add(os.path.join(FIXTURE_DIR, 'logs/')) 259 fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
260 ix = Indexer(fl, 260 ix = Indexer(fl)
261 create_parent_links=True, 261 ix.make_indexes(
262 create_topdir_parent_link=True) 262 create_parent_links=True,
263 ix.make_indexes() 263 create_topdir_parent_link=True)
264 264
265 self.assert_files(fl, [ 265 self.assert_files(fl, [
266 ('', 'application/directory', None), 266 ('', 'application/directory', None),
@@ -312,10 +312,10 @@ class TestFileList(testtools.TestCase):
312 '''Test index generation creates topdir parent link''' 312 '''Test index generation creates topdir parent link'''
313 with FileList() as fl: 313 with FileList() as fl:
314 fl.add(os.path.join(FIXTURE_DIR, 'logs/')) 314 fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
315 ix = Indexer(fl, 315 ix = Indexer(fl)
316 create_parent_links=False, 316 ix.make_indexes(
317 create_topdir_parent_link=False) 317 create_parent_links=False,
318 ix.make_indexes() 318 create_topdir_parent_link=False)
319 319
320 self.assert_files(fl, [ 320 self.assert_files(fl, [
321 ('', 'application/directory', None), 321 ('', 'application/directory', None),
diff --git a/roles/upload-logs-swift/library/zuul_swift_upload.py b/roles/upload-logs-swift/library/zuul_swift_upload.py
index 04d9070..ffa7398 100755
--- a/roles/upload-logs-swift/library/zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/zuul_swift_upload.py
@@ -282,87 +282,23 @@ class FileList(Sequence):
282 282
283 283
284class Indexer(): 284class Indexer():
285 """generates index.html files if requested.""" 285 """Index a FileList
286 286
287 def __init__(self, file_list, create_parent_links=True, 287 Functions to generate indexes and other collated data for a
288 create_topdir_parent_link=False, 288 FileList
289 append_footer='index_footer.html'):
290 289
290 - make_indexes() : make index.html in folders
291 """
292 def __init__(self, file_list):
291 ''' 293 '''
292 Args: 294 Args:
293 file_list (FileList): A FileList object to be updated 295 file_list (FileList): A FileList object with all files
294 with index files for each directory. 296 to be indexed.
295 create_parent_links (bool):
296 create_topdir_parent_link (bool):
297 append_footer (str):
298 ''' 297 '''
299 assert isinstance(file_list, FileList) 298 assert isinstance(file_list, FileList)
300 self.file_list = file_list 299 self.file_list = file_list
301 self.create_parent_links = create_parent_links
302 self.create_topdir_parent_link = create_topdir_parent_link
303 self.append_footer = append_footer
304 self.index_filename = 'index.html'
305
306 def make_indexes(self):
307 '''Make index files
308
309 Return:
310 No value, the self.file_list will be updated
311 '''
312 folders = collections.OrderedDict()
313 for f in self.file_list:
314 if f.folder:
315 folders[f.relative_path] = []
316 folder = os.path.dirname(os.path.dirname(
317 f.relative_path + '/'))
318 if folder == '/':
319 folder = ''
320 else:
321 folder = os.path.dirname(f.relative_path)
322 folders[folder].append(f)
323
324 indexes = {}
325 parent_file_detail = FileDetail(None, '..', '..')
326 for folder, files in folders.items():
327 # Don't add the pseudo-top-directory
328 if files and files[0].full_path is None:
329 files = files[1:]
330 if self.create_topdir_parent_link:
331 files = [parent_file_detail] + files
332 elif self.create_parent_links:
333 files = [parent_file_detail] + files
334
335 # Do generate a link to the parent directory
336 full_path = self.make_index_file(files, 'Index of %s' % (folder,),
337 self.file_list.get_tempdir())
338 300
339 if full_path: 301 def _make_index_file(self, folder_links, title, tempdir, append_footer):
340 filename = os.path.basename(full_path)
341 relative_name = os.path.join(folder, filename)
342 indexes[folder] = FileDetail(full_path, relative_name)
343
344 # This appends the index file at the end of the group of files
345 # for each directory.
346 new_list = []
347 last_dirname = None
348 for f in reversed(list(self.file_list)):
349 if f.folder:
350 relative_path = f.relative_path + '/'
351 else:
352 relative_path = f.relative_path
353 dirname = os.path.dirname(relative_path)
354 if dirname == '/':
355 dirname = ''
356 if dirname != last_dirname:
357 index = indexes.pop(dirname, None)
358 if index:
359 new_list.append(index)
360 last_dirname = dirname
361 new_list.append(f)
362 new_list.reverse()
363 self.file_list.file_list = new_list
364
365 def make_index_file(self, folder_links, title, tempdir):
366 """Writes an index into a file for pushing""" 302 """Writes an index into a file for pushing"""
367 for file_details in folder_links: 303 for file_details in folder_links:
368 # Do not generate an index file if one exists already. 304 # Do not generate an index file if one exists already.
@@ -370,12 +306,13 @@ class Indexer():
370 # content like python coverage info. 306 # content like python coverage info.
371 if self.index_filename == file_details.filename: 307 if self.index_filename == file_details.filename:
372 return 308 return
373 index_content = self.generate_log_index(folder_links, title) 309 index_content = self._generate_log_index(
310 folder_links, title, append_footer)
374 fd = open(os.path.join(tempdir, self.index_filename), 'w') 311 fd = open(os.path.join(tempdir, self.index_filename), 'w')
375 fd.write(index_content) 312 fd.write(index_content)
376 return os.path.join(tempdir, self.index_filename) 313 return os.path.join(tempdir, self.index_filename)
377 314
378 def generate_log_index(self, folder_links, title): 315 def _generate_log_index(self, folder_links, title, append_footer):
379 """Create an index of logfiles and links to them""" 316 """Create an index of logfiles and links to them"""
380 317
381 output = '<html><head><title>%s</title></head><body>\n' % title 318 output = '<html><head><title>%s</title></head><body>\n' % title
@@ -403,8 +340,8 @@ class Indexer():
403 output += '<td style="text-align: right">%s</td>' % size 340 output += '<td style="text-align: right">%s</td>' % size
404 output += '</tr>\n' 341 output += '</tr>\n'
405 342
406 if (self.append_footer and 343 if (append_footer and
407 self.append_footer in file_details.filename): 344 append_footer in file_details.filename):
408 file_details_to_append = file_details 345 file_details_to_append = file_details
409 346
410 output += '</table>' 347 output += '</table>'
@@ -420,6 +357,78 @@ class Indexer():
420 output += '</body></html>\n' 357 output += '</body></html>\n'
421 return output 358 return output
422 359
360 def make_indexes(self, create_parent_links=True,
361 create_topdir_parent_link=False,
362 append_footer='index_footer.html'):
363 '''Make index.html files
364
365 Iterate the file list and crete index.html files for folders
366
367 Args:
368 create_parent_links (bool): Create parent links
369 create_topdir_parent_link (bool): Create topdir parent link
370 append_footer (str): Filename of a footer to append to each
371 generated page
372
373 Return:
374 No value, the self.file_list will be updated
375 '''
376 self.index_filename = 'index.html'
377
378 folders = collections.OrderedDict()
379 for f in self.file_list:
380 if f.folder:
381 folders[f.relative_path] = []
382 folder = os.path.dirname(os.path.dirname(
383 f.relative_path + '/'))
384 if folder == '/':
385 folder = ''
386 else:
387 folder = os.path.dirname(f.relative_path)
388 folders[folder].append(f)
389
390 indexes = {}
391 parent_file_detail = FileDetail(None, '..', '..')
392 for folder, files in folders.items():
393 # Don't add the pseudo-top-directory
394 if files and files[0].full_path is None:
395 files = files[1:]
396 if create_topdir_parent_link:
397 files = [parent_file_detail] + files
398 elif create_parent_links:
399 files = [parent_file_detail] + files
400
401 # Do generate a link to the parent directory
402 full_path = self._make_index_file(files, 'Index of %s' % (folder,),
403 self.file_list.get_tempdir(),
404 append_footer)
405
406 if full_path:
407 filename = os.path.basename(full_path)
408 relative_name = os.path.join(folder, filename)
409 indexes[folder] = FileDetail(full_path, relative_name)
410
411 # This appends the index file at the end of the group of files
412 # for each directory.
413 new_list = []
414 last_dirname = None
415 for f in reversed(list(self.file_list)):
416 if f.folder:
417 relative_path = f.relative_path + '/'
418 else:
419 relative_path = f.relative_path
420 dirname = os.path.dirname(relative_path)
421 if dirname == '/':
422 dirname = ''
423 if dirname != last_dirname:
424 index = indexes.pop(dirname, None)
425 if index:
426 new_list.append(index)
427 last_dirname = dirname
428 new_list.append(f)
429 new_list.reverse()
430 self.file_list.file_list = new_list
431
423 432
424class DeflateFilter(): 433class DeflateFilter():
425 chunk_size = 16384 434 chunk_size = 16384
@@ -585,18 +594,17 @@ def run(cloud, container, files,
585 594
586 # Create the objects to make sure the arguments are sound. 595 # Create the objects to make sure the arguments are sound.
587 with FileList() as file_list: 596 with FileList() as file_list:
588 indexer = Indexer(file_list,
589 create_parent_links=parent_links,
590 create_topdir_parent_link=topdir_parent_link,
591 append_footer=footer)
592
593 # Scan the files. 597 # Scan the files.
594 for file_path in files: 598 for file_path in files:
595 file_list.add(file_path) 599 file_list.add(file_path)
596 600
601 indexer = Indexer(file_list)
602
597 # (Possibly) make indexes. 603 # (Possibly) make indexes.
598 if indexes: 604 if indexes:
599 indexer.make_indexes() 605 indexer.make_indexes(create_parent_links=parent_links,
606 create_topdir_parent_link=topdir_parent_link,
607 append_footer=footer)
600 608
601 logging.debug("List of files prepared to upload:") 609 logging.debug("List of files prepared to upload:")
602 for x in file_list: 610 for x in file_list: