summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-11-30 04:50:11 +0000
committerGerrit Code Review <review@openstack.org>2018-11-30 04:50:11 +0000
commit2ea7a1b6bdf69d5a79658aea04a1f6d596e95d0a (patch)
tree3b173ed1faf8324314d25c2ca562682137b5ed8b
parent947e49a270707274dbf328ddf694cb95794029c7 (diff)
parent2d61e9d4fc32b85a30afc05c12e4fd3a4e40c1ed (diff)
Merge "upload-logs-swift: Make indexer more generic"
-rw-r--r--roles/upload-logs-swift/library/test_zuul_swift_upload.py16
-rwxr-xr-xroles/upload-logs-swift/library/zuul_swift_upload.py174
2 files changed, 99 insertions, 91 deletions
diff --git a/roles/upload-logs-swift/library/test_zuul_swift_upload.py b/roles/upload-logs-swift/library/test_zuul_swift_upload.py
index be69c78..39f23b4 100644
--- a/roles/upload-logs-swift/library/test_zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/test_zuul_swift_upload.py
@@ -257,10 +257,10 @@ class TestFileList(testtools.TestCase):
257 '''Test index generation creates topdir parent link''' 257 '''Test index generation creates topdir parent link'''
258 with FileList() as fl: 258 with FileList() as fl:
259 fl.add(os.path.join(FIXTURE_DIR, 'logs/')) 259 fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
260 ix = Indexer(fl, 260 ix = Indexer(fl)
261 create_parent_links=True, 261 ix.make_indexes(
262 create_topdir_parent_link=True) 262 create_parent_links=True,
263 ix.make_indexes() 263 create_topdir_parent_link=True)
264 264
265 self.assert_files(fl, [ 265 self.assert_files(fl, [
266 ('', 'application/directory', None), 266 ('', 'application/directory', None),
@@ -312,10 +312,10 @@ class TestFileList(testtools.TestCase):
312 '''Test index generation creates topdir parent link''' 312 '''Test index generation creates topdir parent link'''
313 with FileList() as fl: 313 with FileList() as fl:
314 fl.add(os.path.join(FIXTURE_DIR, 'logs/')) 314 fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
315 ix = Indexer(fl, 315 ix = Indexer(fl)
316 create_parent_links=False, 316 ix.make_indexes(
317 create_topdir_parent_link=False) 317 create_parent_links=False,
318 ix.make_indexes() 318 create_topdir_parent_link=False)
319 319
320 self.assert_files(fl, [ 320 self.assert_files(fl, [
321 ('', 'application/directory', None), 321 ('', 'application/directory', None),
diff --git a/roles/upload-logs-swift/library/zuul_swift_upload.py b/roles/upload-logs-swift/library/zuul_swift_upload.py
index 04d9070..ffa7398 100755
--- a/roles/upload-logs-swift/library/zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/zuul_swift_upload.py
@@ -282,87 +282,23 @@ class FileList(Sequence):
282 282
283 283
284class Indexer(): 284class Indexer():
285 """generates index.html files if requested.""" 285 """Index a FileList
286 286
287 def __init__(self, file_list, create_parent_links=True, 287 Functions to generate indexes and other collated data for a
288 create_topdir_parent_link=False, 288 FileList
289 append_footer='index_footer.html'):
290 289
290 - make_indexes() : make index.html in folders
291 """
292 def __init__(self, file_list):
291 ''' 293 '''
292 Args: 294 Args:
293 file_list (FileList): A FileList object to be updated 295 file_list (FileList): A FileList object with all files
294 with index files for each directory. 296 to be indexed.
295 create_parent_links (bool):
296 create_topdir_parent_link (bool):
297 append_footer (str):
298 ''' 297 '''
299 assert isinstance(file_list, FileList) 298 assert isinstance(file_list, FileList)
300 self.file_list = file_list 299 self.file_list = file_list
301 self.create_parent_links = create_parent_links
302 self.create_topdir_parent_link = create_topdir_parent_link
303 self.append_footer = append_footer
304 self.index_filename = 'index.html'
305
306 def make_indexes(self):
307 '''Make index files
308
309 Return:
310 No value, the self.file_list will be updated
311 '''
312 folders = collections.OrderedDict()
313 for f in self.file_list:
314 if f.folder:
315 folders[f.relative_path] = []
316 folder = os.path.dirname(os.path.dirname(
317 f.relative_path + '/'))
318 if folder == '/':
319 folder = ''
320 else:
321 folder = os.path.dirname(f.relative_path)
322 folders[folder].append(f)
323
324 indexes = {}
325 parent_file_detail = FileDetail(None, '..', '..')
326 for folder, files in folders.items():
327 # Don't add the pseudo-top-directory
328 if files and files[0].full_path is None:
329 files = files[1:]
330 if self.create_topdir_parent_link:
331 files = [parent_file_detail] + files
332 elif self.create_parent_links:
333 files = [parent_file_detail] + files
334
335 # Do generate a link to the parent directory
336 full_path = self.make_index_file(files, 'Index of %s' % (folder,),
337 self.file_list.get_tempdir())
338 300
339 if full_path: 301 def _make_index_file(self, folder_links, title, tempdir, append_footer):
340 filename = os.path.basename(full_path)
341 relative_name = os.path.join(folder, filename)
342 indexes[folder] = FileDetail(full_path, relative_name)
343
344 # This appends the index file at the end of the group of files
345 # for each directory.
346 new_list = []
347 last_dirname = None
348 for f in reversed(list(self.file_list)):
349 if f.folder:
350 relative_path = f.relative_path + '/'
351 else:
352 relative_path = f.relative_path
353 dirname = os.path.dirname(relative_path)
354 if dirname == '/':
355 dirname = ''
356 if dirname != last_dirname:
357 index = indexes.pop(dirname, None)
358 if index:
359 new_list.append(index)
360 last_dirname = dirname
361 new_list.append(f)
362 new_list.reverse()
363 self.file_list.file_list = new_list
364
365 def make_index_file(self, folder_links, title, tempdir):
366 """Writes an index into a file for pushing""" 302 """Writes an index into a file for pushing"""
367 for file_details in folder_links: 303 for file_details in folder_links:
368 # Do not generate an index file if one exists already. 304 # Do not generate an index file if one exists already.
@@ -370,12 +306,13 @@ class Indexer():
370 # content like python coverage info. 306 # content like python coverage info.
371 if self.index_filename == file_details.filename: 307 if self.index_filename == file_details.filename:
372 return 308 return
373 index_content = self.generate_log_index(folder_links, title) 309 index_content = self._generate_log_index(
310 folder_links, title, append_footer)
374 fd = open(os.path.join(tempdir, self.index_filename), 'w') 311 fd = open(os.path.join(tempdir, self.index_filename), 'w')
375 fd.write(index_content) 312 fd.write(index_content)
376 return os.path.join(tempdir, self.index_filename) 313 return os.path.join(tempdir, self.index_filename)
377 314
378 def generate_log_index(self, folder_links, title): 315 def _generate_log_index(self, folder_links, title, append_footer):
379 """Create an index of logfiles and links to them""" 316 """Create an index of logfiles and links to them"""
380 317
381 output = '<html><head><title>%s</title></head><body>\n' % title 318 output = '<html><head><title>%s</title></head><body>\n' % title
@@ -403,8 +340,8 @@ class Indexer():
403 output += '<td style="text-align: right">%s</td>' % size 340 output += '<td style="text-align: right">%s</td>' % size
404 output += '</tr>\n' 341 output += '</tr>\n'
405 342
406 if (self.append_footer and 343 if (append_footer and
407 self.append_footer in file_details.filename): 344 append_footer in file_details.filename):
408 file_details_to_append = file_details 345 file_details_to_append = file_details
409 346
410 output += '</table>' 347 output += '</table>'
@@ -420,6 +357,78 @@ class Indexer():
420 output += '</body></html>\n' 357 output += '</body></html>\n'
421 return output 358 return output
422 359
360 def make_indexes(self, create_parent_links=True,
361 create_topdir_parent_link=False,
362 append_footer='index_footer.html'):
363 '''Make index.html files
364
365 Iterate the file list and crete index.html files for folders
366
367 Args:
368 create_parent_links (bool): Create parent links
369 create_topdir_parent_link (bool): Create topdir parent link
370 append_footer (str): Filename of a footer to append to each
371 generated page
372
373 Return:
374 No value, the self.file_list will be updated
375 '''
376 self.index_filename = 'index.html'
377
378 folders = collections.OrderedDict()
379 for f in self.file_list:
380 if f.folder:
381 folders[f.relative_path] = []
382 folder = os.path.dirname(os.path.dirname(
383 f.relative_path + '/'))
384 if folder == '/':
385 folder = ''
386 else:
387 folder = os.path.dirname(f.relative_path)
388 folders[folder].append(f)
389
390 indexes = {}
391 parent_file_detail = FileDetail(None, '..', '..')
392 for folder, files in folders.items():
393 # Don't add the pseudo-top-directory
394 if files and files[0].full_path is None:
395 files = files[1:]
396 if create_topdir_parent_link:
397 files = [parent_file_detail] + files
398 elif create_parent_links:
399 files = [parent_file_detail] + files
400
401 # Do generate a link to the parent directory
402 full_path = self._make_index_file(files, 'Index of %s' % (folder,),
403 self.file_list.get_tempdir(),
404 append_footer)
405
406 if full_path:
407 filename = os.path.basename(full_path)
408 relative_name = os.path.join(folder, filename)
409 indexes[folder] = FileDetail(full_path, relative_name)
410
411 # This appends the index file at the end of the group of files
412 # for each directory.
413 new_list = []
414 last_dirname = None
415 for f in reversed(list(self.file_list)):
416 if f.folder:
417 relative_path = f.relative_path + '/'
418 else:
419 relative_path = f.relative_path
420 dirname = os.path.dirname(relative_path)
421 if dirname == '/':
422 dirname = ''
423 if dirname != last_dirname:
424 index = indexes.pop(dirname, None)
425 if index:
426 new_list.append(index)
427 last_dirname = dirname
428 new_list.append(f)
429 new_list.reverse()
430 self.file_list.file_list = new_list
431
423 432
424class DeflateFilter(): 433class DeflateFilter():
425 chunk_size = 16384 434 chunk_size = 16384
@@ -585,18 +594,17 @@ def run(cloud, container, files,
585 594
586 # Create the objects to make sure the arguments are sound. 595 # Create the objects to make sure the arguments are sound.
587 with FileList() as file_list: 596 with FileList() as file_list:
588 indexer = Indexer(file_list,
589 create_parent_links=parent_links,
590 create_topdir_parent_link=topdir_parent_link,
591 append_footer=footer)
592
593 # Scan the files. 597 # Scan the files.
594 for file_path in files: 598 for file_path in files:
595 file_list.add(file_path) 599 file_list.add(file_path)
596 600
601 indexer = Indexer(file_list)
602
597 # (Possibly) make indexes. 603 # (Possibly) make indexes.
598 if indexes: 604 if indexes:
599 indexer.make_indexes() 605 indexer.make_indexes(create_parent_links=parent_links,
606 create_topdir_parent_link=topdir_parent_link,
607 append_footer=footer)
600 608
601 logging.debug("List of files prepared to upload:") 609 logging.debug("List of files prepared to upload:")
602 for x in file_list: 610 for x in file_list: