summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-11-22 20:41:05 +0000
committerGerrit Code Review <review@openstack.org>2018-11-22 20:41:05 +0000
commit92c5348052cb726f6b3931b35ed2d06e369006a1 (patch)
tree9d91b5b9cb1c78a4e9f0b48d417e303f918608d2
parent89e386449db5bf96d2a3ffb190a9fc1fea65f248 (diff)
parent0fb160ec9510ef9f7da53445601fc890f55ced08 (diff)
Merge "upload-logs-swift: Turn FileList into a context manager"
-rw-r--r--roles/upload-logs-swift/library/test_zuul_swift_upload.py500
-rwxr-xr-xroles/upload-logs-swift/library/zuul_swift_upload.py88
2 files changed, 305 insertions, 283 deletions
diff --git a/roles/upload-logs-swift/library/test_zuul_swift_upload.py b/roles/upload-logs-swift/library/test_zuul_swift_upload.py
index 8e9179d..24daf6e 100644
--- a/roles/upload-logs-swift/library/test_zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/test_zuul_swift_upload.py
@@ -82,272 +82,276 @@ class TestFileList(testtools.TestCase):
82 82
83 def test_single_dir_trailing_slash(self): 83 def test_single_dir_trailing_slash(self):
84 '''Test a single directory with a trailing slash''' 84 '''Test a single directory with a trailing slash'''
85 fl = FileList() 85
86 fl.add(os.path.join(FIXTURE_DIR, 'logs/')) 86 with FileList() as fl:
87 self.assert_files(fl, [ 87 fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
88 ('', 'application/directory', None), 88 self.assert_files(fl, [
89 ('controller', 'application/directory', None), 89 ('', 'application/directory', None),
90 ('zuul-info', 'application/directory', None), 90 ('controller', 'application/directory', None),
91 ('job-output.json', 'application/json', None), 91 ('zuul-info', 'application/directory', None),
92 ('controller/subdir', 'application/directory', None), 92 ('job-output.json', 'application/json', None),
93 ('controller/compressed.gz', 'text/plain', 'gzip'), 93 ('controller/subdir', 'application/directory', None),
94 ('controller/cpu-load.svg', 'image/svg+xml', None), 94 ('controller/compressed.gz', 'text/plain', 'gzip'),
95 ('controller/journal.xz', 'text/plain', 'xz'), 95 ('controller/cpu-load.svg', 'image/svg+xml', None),
96 ('controller/service_log.txt', 'text/plain', None), 96 ('controller/journal.xz', 'text/plain', 'xz'),
97 ('controller/syslog', 'text/plain', None), 97 ('controller/service_log.txt', 'text/plain', None),
98 ('controller/subdir/subdir.txt', 'text/plain', None), 98 ('controller/syslog', 'text/plain', None),
99 ('zuul-info/inventory.yaml', 'text/plain', None), 99 ('controller/subdir/subdir.txt', 'text/plain', None),
100 ('zuul-info/zuul-info.controller.txt', 'text/plain', None), 100 ('zuul-info/inventory.yaml', 'text/plain', None),
101 ]) 101 ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
102 ])
102 103
103 def test_single_dir(self): 104 def test_single_dir(self):
104 '''Test a single directory without a trailing slash''' 105 '''Test a single directory without a trailing slash'''
105 fl = FileList() 106 with FileList() as fl:
106 fl.add(os.path.join(FIXTURE_DIR, 'logs')) 107 fl.add(os.path.join(FIXTURE_DIR, 'logs'))
107 self.assert_files(fl, [ 108 self.assert_files(fl, [
108 ('', 'application/directory', None), 109 ('', 'application/directory', None),
109 ('logs', 'application/directory', None), 110 ('logs', 'application/directory', None),
110 ('logs/controller', 'application/directory', None), 111 ('logs/controller', 'application/directory', None),
111 ('logs/zuul-info', 'application/directory', None), 112 ('logs/zuul-info', 'application/directory', None),
112 ('logs/job-output.json', 'application/json', None), 113 ('logs/job-output.json', 'application/json', None),
113 ('logs/controller/subdir', 'application/directory', None), 114 ('logs/controller/subdir', 'application/directory', None),
114 ('logs/controller/compressed.gz', 'text/plain', 'gzip'), 115 ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
115 ('logs/controller/cpu-load.svg', 'image/svg+xml', None), 116 ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
116 ('logs/controller/journal.xz', 'text/plain', 'xz'), 117 ('logs/controller/journal.xz', 'text/plain', 'xz'),
117 ('logs/controller/service_log.txt', 'text/plain', None), 118 ('logs/controller/service_log.txt', 'text/plain', None),
118 ('logs/controller/syslog', 'text/plain', None), 119 ('logs/controller/syslog', 'text/plain', None),
119 ('logs/controller/subdir/subdir.txt', 'text/plain', None), 120 ('logs/controller/subdir/subdir.txt', 'text/plain', None),
120 ('logs/zuul-info/inventory.yaml', 'text/plain', None), 121 ('logs/zuul-info/inventory.yaml', 'text/plain', None),
121 ('logs/zuul-info/zuul-info.controller.txt', 'text/plain', None), 122 ('logs/zuul-info/zuul-info.controller.txt',
122 ]) 123 'text/plain', None),
124 ])
123 125
124 def test_single_file(self): 126 def test_single_file(self):
125 '''Test a single file''' 127 '''Test a single file'''
126 fl = FileList() 128 with FileList() as fl:
127 fl.add(os.path.join(FIXTURE_DIR, 129 fl.add(os.path.join(FIXTURE_DIR,
128 'logs/zuul-info/inventory.yaml')) 130 'logs/zuul-info/inventory.yaml'))
129 self.assert_files(fl, [ 131 self.assert_files(fl, [
130 ('', 'application/directory', None), 132 ('', 'application/directory', None),
131 ('inventory.yaml', 'text/plain', None), 133 ('inventory.yaml', 'text/plain', None),
132 ]) 134 ])
133 135
134 def test_symlinks(self): 136 def test_symlinks(self):
135 '''Test symlinks''' 137 '''Test symlinks'''
136 fl = FileList() 138 with FileList() as fl:
137 self.useFixture(SymlinkFixture()) 139 self.useFixture(SymlinkFixture())
138 fl.add(os.path.join(FIXTURE_DIR, 'links/')) 140 fl.add(os.path.join(FIXTURE_DIR, 'links/'))
139 self.assert_files(fl, [ 141 self.assert_files(fl, [
140 ('', 'application/directory', None), 142 ('', 'application/directory', None),
141 ('controller', 'application/directory', None), 143 ('controller', 'application/directory', None),
142 ('good_symlink', 'application/directory', None), 144 ('good_symlink', 'application/directory', None),
143 ('recursive_symlink', 'application/directory', None), 145 ('recursive_symlink', 'application/directory', None),
144 ('symlink_loop', 'application/directory', None), 146 ('symlink_loop', 'application/directory', None),
145 ('symlink_loop_a', 'application/directory', None), 147 ('symlink_loop_a', 'application/directory', None),
146 ('job-output.json', 'application/json', None), 148 ('job-output.json', 'application/json', None),
147 ('symlink_file', 'text/plain', None), 149 ('symlink_file', 'text/plain', None),
148 ('controller/service_log.txt', 'text/plain', None), 150 ('controller/service_log.txt', 'text/plain', None),
149 ('symlink_loop/symlink_loop_b', 'application/directory', None), 151 ('symlink_loop/symlink_loop_b', 'application/directory', None),
150 ('symlink_loop/placeholder', 'text/plain', None), 152 ('symlink_loop/placeholder', 'text/plain', None),
151 ]) 153 ])
152 154
153 def test_index_files(self): 155 def test_index_files(self):
154 '''Test index generation''' 156 '''Test index generation'''
155 fl = FileList() 157 with FileList() as fl:
156 fl.add(os.path.join(FIXTURE_DIR, 'logs')) 158 fl.add(os.path.join(FIXTURE_DIR, 'logs'))
157 ix = Indexer() 159 ix = Indexer()
158 fl = ix.make_indexes(fl) 160 ix.make_indexes(fl)
159 161
160 self.assert_files(fl, [ 162 self.assert_files(fl, [
161 ('', 'application/directory', None), 163 ('', 'application/directory', None),
162 ('index.html', 'text/html', None), 164 ('index.html', 'text/html', None),
163 ('logs', 'application/directory', None), 165 ('logs', 'application/directory', None),
164 ('logs/controller', 'application/directory', None), 166 ('logs/controller', 'application/directory', None),
165 ('logs/zuul-info', 'application/directory', None), 167 ('logs/zuul-info', 'application/directory', None),
166 ('logs/job-output.json', 'application/json', None), 168 ('logs/job-output.json', 'application/json', None),
167 ('logs/index.html', 'text/html', None), 169 ('logs/index.html', 'text/html', None),
168 ('logs/controller/subdir', 'application/directory', None), 170 ('logs/controller/subdir', 'application/directory', None),
169 ('logs/controller/compressed.gz', 'text/plain', 'gzip'), 171 ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
170 ('logs/controller/cpu-load.svg', 'image/svg+xml', None), 172 ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
171 ('logs/controller/journal.xz', 'text/plain', 'xz'), 173 ('logs/controller/journal.xz', 'text/plain', 'xz'),
172 ('logs/controller/service_log.txt', 'text/plain', None), 174 ('logs/controller/service_log.txt', 'text/plain', None),
173 ('logs/controller/syslog', 'text/plain', None), 175 ('logs/controller/syslog', 'text/plain', None),
174 ('logs/controller/index.html', 'text/html', None), 176 ('logs/controller/index.html', 'text/html', None),
175 ('logs/controller/subdir/subdir.txt', 'text/plain', None), 177 ('logs/controller/subdir/subdir.txt', 'text/plain', None),
176 ('logs/controller/subdir/index.html', 'text/html', None), 178 ('logs/controller/subdir/index.html', 'text/html', None),
177 ('logs/zuul-info/inventory.yaml', 'text/plain', None), 179 ('logs/zuul-info/inventory.yaml', 'text/plain', None),
178 ('logs/zuul-info/zuul-info.controller.txt', 'text/plain', None), 180 ('logs/zuul-info/zuul-info.controller.txt',
179 ('logs/zuul-info/index.html', 'text/html', None), 181 'text/plain', None),
180 ]) 182 ('logs/zuul-info/index.html', 'text/html', None),
181 183 ])
182 top_index = self.find_file(fl, 'index.html') 184
183 page = open(top_index.full_path).read() 185 top_index = self.find_file(fl, 'index.html')
184 page = BeautifulSoup(page, 'html.parser') 186 page = open(top_index.full_path).read()
185 rows = page.find_all('tr')[1:] 187 page = BeautifulSoup(page, 'html.parser')
186 188 rows = page.find_all('tr')[1:]
187 self.assertEqual(len(rows), 1) 189
188 190 self.assertEqual(len(rows), 1)
189 self.assertEqual(rows[0].find('a').get('href'), 'logs/') 191
190 self.assertEqual(rows[0].find('a').text, 'logs/') 192 self.assertEqual(rows[0].find('a').get('href'), 'logs/')
191 193 self.assertEqual(rows[0].find('a').text, 'logs/')
192 subdir_index = self.find_file(fl, 'logs/controller/subdir/index.html') 194
193 page = open(subdir_index.full_path).read() 195 subdir_index = self.find_file(
194 page = BeautifulSoup(page, 'html.parser') 196 fl, 'logs/controller/subdir/index.html')
195 rows = page.find_all('tr')[1:] 197 page = open(subdir_index.full_path).read()
196 self.assertEqual(rows[0].find('a').get('href'), '../') 198 page = BeautifulSoup(page, 'html.parser')
197 self.assertEqual(rows[0].find('a').text, '../') 199 rows = page.find_all('tr')[1:]
198 200 self.assertEqual(rows[0].find('a').get('href'), '../')
199 self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') 201 self.assertEqual(rows[0].find('a').text, '../')
200 self.assertEqual(rows[1].find('a').text, 'subdir.txt') 202
203 self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
204 self.assertEqual(rows[1].find('a').text, 'subdir.txt')
201 205
202 def test_index_files_trailing_slash(self): 206 def test_index_files_trailing_slash(self):
203 '''Test index generation with a trailing slash''' 207 '''Test index generation with a trailing slash'''
204 fl = FileList() 208 with FileList() as fl:
205 fl.add(os.path.join(FIXTURE_DIR, 'logs/')) 209 fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
206 ix = Indexer() 210 ix = Indexer()
207 fl = ix.make_indexes(fl) 211 ix.make_indexes(fl)
208 212
209 self.assert_files(fl, [ 213 self.assert_files(fl, [
210 ('', 'application/directory', None), 214 ('', 'application/directory', None),
211 ('controller', 'application/directory', None), 215 ('controller', 'application/directory', None),
212 ('zuul-info', 'application/directory', None), 216 ('zuul-info', 'application/directory', None),
213 ('job-output.json', 'application/json', None), 217 ('job-output.json', 'application/json', None),
214 ('index.html', 'text/html', None), 218 ('index.html', 'text/html', None),
215 ('controller/subdir', 'application/directory', None), 219 ('controller/subdir', 'application/directory', None),
216 ('controller/compressed.gz', 'text/plain', 'gzip'), 220 ('controller/compressed.gz', 'text/plain', 'gzip'),
217 ('controller/cpu-load.svg', 'image/svg+xml', None), 221 ('controller/cpu-load.svg', 'image/svg+xml', None),
218 ('controller/journal.xz', 'text/plain', 'xz'), 222 ('controller/journal.xz', 'text/plain', 'xz'),
219 ('controller/service_log.txt', 'text/plain', None), 223 ('controller/service_log.txt', 'text/plain', None),
220 ('controller/syslog', 'text/plain', None), 224 ('controller/syslog', 'text/plain', None),
221 ('controller/index.html', 'text/html', None), 225 ('controller/index.html', 'text/html', None),
222 ('controller/subdir/subdir.txt', 'text/plain', None), 226 ('controller/subdir/subdir.txt', 'text/plain', None),
223 ('controller/subdir/index.html', 'text/html', None), 227 ('controller/subdir/index.html', 'text/html', None),
224 ('zuul-info/inventory.yaml', 'text/plain', None), 228 ('zuul-info/inventory.yaml', 'text/plain', None),
225 ('zuul-info/zuul-info.controller.txt', 'text/plain', None), 229 ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
226 ('zuul-info/index.html', 'text/html', None), 230 ('zuul-info/index.html', 'text/html', None),
227 ]) 231 ])
228 232
229 top_index = self.find_file(fl, 'index.html') 233 top_index = self.find_file(fl, 'index.html')
230 page = open(top_index.full_path).read() 234 page = open(top_index.full_path).read()
231 page = BeautifulSoup(page, 'html.parser') 235 page = BeautifulSoup(page, 'html.parser')
232 rows = page.find_all('tr')[1:] 236 rows = page.find_all('tr')[1:]
233 237
234 self.assertEqual(len(rows), 3) 238 self.assertEqual(len(rows), 3)
235 239
236 self.assertEqual(rows[0].find('a').get('href'), 'controller/') 240 self.assertEqual(rows[0].find('a').get('href'), 'controller/')
237 self.assertEqual(rows[0].find('a').text, 'controller/') 241 self.assertEqual(rows[0].find('a').text, 'controller/')
238 242
239 self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/') 243 self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
240 self.assertEqual(rows[1].find('a').text, 'zuul-info/') 244 self.assertEqual(rows[1].find('a').text, 'zuul-info/')
241 245
242 subdir_index = self.find_file(fl, 'controller/subdir/index.html') 246 subdir_index = self.find_file(fl, 'controller/subdir/index.html')
243 page = open(subdir_index.full_path).read() 247 page = open(subdir_index.full_path).read()
244 page = BeautifulSoup(page, 'html.parser') 248 page = BeautifulSoup(page, 'html.parser')
245 rows = page.find_all('tr')[1:] 249 rows = page.find_all('tr')[1:]
246 self.assertEqual(rows[0].find('a').get('href'), '../') 250 self.assertEqual(rows[0].find('a').get('href'), '../')
247 self.assertEqual(rows[0].find('a').text, '../') 251 self.assertEqual(rows[0].find('a').text, '../')
248 252
249 self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') 253 self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
250 self.assertEqual(rows[1].find('a').text, 'subdir.txt') 254 self.assertEqual(rows[1].find('a').text, 'subdir.txt')
251 255
252 def test_topdir_parent_link(self): 256 def test_topdir_parent_link(self):
253 '''Test index generation creates topdir parent link''' 257 '''Test index generation creates topdir parent link'''
254 fl = FileList() 258 with FileList() as fl:
255 fl.add(os.path.join(FIXTURE_DIR, 'logs/')) 259 fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
256 ix = Indexer(create_parent_links=True, 260 ix = Indexer(create_parent_links=True,
257 create_topdir_parent_link=True) 261 create_topdir_parent_link=True)
258 fl = ix.make_indexes(fl) 262 ix.make_indexes(fl)
259 263
260 self.assert_files(fl, [ 264 self.assert_files(fl, [
261 ('', 'application/directory', None), 265 ('', 'application/directory', None),
262 ('controller', 'application/directory', None), 266 ('controller', 'application/directory', None),
263 ('zuul-info', 'application/directory', None), 267 ('zuul-info', 'application/directory', None),
264 ('job-output.json', 'application/json', None), 268 ('job-output.json', 'application/json', None),
265 ('index.html', 'text/html', None), 269 ('index.html', 'text/html', None),
266 ('controller/subdir', 'application/directory', None), 270 ('controller/subdir', 'application/directory', None),
267 ('controller/compressed.gz', 'text/plain', 'gzip'), 271 ('controller/compressed.gz', 'text/plain', 'gzip'),
268 ('controller/cpu-load.svg', 'image/svg+xml', None), 272 ('controller/cpu-load.svg', 'image/svg+xml', None),
269 ('controller/journal.xz', 'text/plain', 'xz'), 273 ('controller/journal.xz', 'text/plain', 'xz'),
270 ('controller/service_log.txt', 'text/plain', None), 274 ('controller/service_log.txt', 'text/plain', None),
271 ('controller/syslog', 'text/plain', None), 275 ('controller/syslog', 'text/plain', None),
272 ('controller/index.html', 'text/html', None), 276 ('controller/index.html', 'text/html', None),
273 ('controller/subdir/subdir.txt', 'text/plain', None), 277 ('controller/subdir/subdir.txt', 'text/plain', None),
274 ('controller/subdir/index.html', 'text/html', None), 278 ('controller/subdir/index.html', 'text/html', None),
275 ('zuul-info/inventory.yaml', 'text/plain', None), 279 ('zuul-info/inventory.yaml', 'text/plain', None),
276 ('zuul-info/zuul-info.controller.txt', 'text/plain', None), 280 ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
277 ('zuul-info/index.html', 'text/html', None), 281 ('zuul-info/index.html', 'text/html', None),
278 ]) 282 ])
279 283
280 top_index = self.find_file(fl, 'index.html') 284 top_index = self.find_file(fl, 'index.html')
281 page = open(top_index.full_path).read() 285 page = open(top_index.full_path).read()
282 page = BeautifulSoup(page, 'html.parser') 286 page = BeautifulSoup(page, 'html.parser')
283 rows = page.find_all('tr')[1:] 287 rows = page.find_all('tr')[1:]
284 288
285 self.assertEqual(len(rows), 4) 289 self.assertEqual(len(rows), 4)
286 290
287 self.assertEqual(rows[0].find('a').get('href'), '../') 291 self.assertEqual(rows[0].find('a').get('href'), '../')
288 self.assertEqual(rows[0].find('a').text, '../') 292 self.assertEqual(rows[0].find('a').text, '../')
289 293
290 self.assertEqual(rows[1].find('a').get('href'), 'controller/') 294 self.assertEqual(rows[1].find('a').get('href'), 'controller/')
291 self.assertEqual(rows[1].find('a').text, 'controller/') 295 self.assertEqual(rows[1].find('a').text, 'controller/')
292 296
293 self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/') 297 self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
294 self.assertEqual(rows[2].find('a').text, 'zuul-info/') 298 self.assertEqual(rows[2].find('a').text, 'zuul-info/')
295 299
296 subdir_index = self.find_file(fl, 'controller/subdir/index.html') 300 subdir_index = self.find_file(fl, 'controller/subdir/index.html')
297 page = open(subdir_index.full_path).read() 301 page = open(subdir_index.full_path).read()
298 page = BeautifulSoup(page, 'html.parser') 302 page = BeautifulSoup(page, 'html.parser')
299 rows = page.find_all('tr')[1:] 303 rows = page.find_all('tr')[1:]
300 self.assertEqual(rows[0].find('a').get('href'), '../') 304 self.assertEqual(rows[0].find('a').get('href'), '../')
301 self.assertEqual(rows[0].find('a').text, '../') 305 self.assertEqual(rows[0].find('a').text, '../')
302 306
303 self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') 307 self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
304 self.assertEqual(rows[1].find('a').text, 'subdir.txt') 308 self.assertEqual(rows[1].find('a').text, 'subdir.txt')
305 309
306 def test_no_parent_links(self): 310 def test_no_parent_links(self):
307 '''Test index generation creates topdir parent link''' 311 '''Test index generation creates topdir parent link'''
308 fl = FileList() 312 with FileList() as fl:
309 fl.add(os.path.join(FIXTURE_DIR, 'logs/')) 313 fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
310 ix = Indexer(create_parent_links=False, 314 ix = Indexer(create_parent_links=False,
311 create_topdir_parent_link=False) 315 create_topdir_parent_link=False)
312 fl = ix.make_indexes(fl) 316 ix.make_indexes(fl)
313 317
314 self.assert_files(fl, [ 318 self.assert_files(fl, [
315 ('', 'application/directory', None), 319 ('', 'application/directory', None),
316 ('controller', 'application/directory', None), 320 ('controller', 'application/directory', None),
317 ('zuul-info', 'application/directory', None), 321 ('zuul-info', 'application/directory', None),
318 ('job-output.json', 'application/json', None), 322 ('job-output.json', 'application/json', None),
319 ('index.html', 'text/html', None), 323 ('index.html', 'text/html', None),
320 ('controller/subdir', 'application/directory', None), 324 ('controller/subdir', 'application/directory', None),
321 ('controller/compressed.gz', 'text/plain', 'gzip'), 325 ('controller/compressed.gz', 'text/plain', 'gzip'),
322 ('controller/cpu-load.svg', 'image/svg+xml', None), 326 ('controller/cpu-load.svg', 'image/svg+xml', None),
323 ('controller/journal.xz', 'text/plain', 'xz'), 327 ('controller/journal.xz', 'text/plain', 'xz'),
324 ('controller/service_log.txt', 'text/plain', None), 328 ('controller/service_log.txt', 'text/plain', None),
325 ('controller/syslog', 'text/plain', None), 329 ('controller/syslog', 'text/plain', None),
326 ('controller/index.html', 'text/html', None), 330 ('controller/index.html', 'text/html', None),
327 ('controller/subdir/subdir.txt', 'text/plain', None), 331 ('controller/subdir/subdir.txt', 'text/plain', None),
328 ('controller/subdir/index.html', 'text/html', None), 332 ('controller/subdir/index.html', 'text/html', None),
329 ('zuul-info/inventory.yaml', 'text/plain', None), 333 ('zuul-info/inventory.yaml', 'text/plain', None),
330 ('zuul-info/zuul-info.controller.txt', 'text/plain', None), 334 ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
331 ('zuul-info/index.html', 'text/html', None), 335 ('zuul-info/index.html', 'text/html', None),
332 ]) 336 ])
333 337
334 top_index = self.find_file(fl, 'index.html') 338 top_index = self.find_file(fl, 'index.html')
335 page = open(top_index.full_path).read() 339 page = open(top_index.full_path).read()
336 page = BeautifulSoup(page, 'html.parser') 340 page = BeautifulSoup(page, 'html.parser')
337 rows = page.find_all('tr')[1:] 341 rows = page.find_all('tr')[1:]
338 342
339 self.assertEqual(len(rows), 3) 343 self.assertEqual(len(rows), 3)
340 344
341 self.assertEqual(rows[0].find('a').get('href'), 'controller/') 345 self.assertEqual(rows[0].find('a').get('href'), 'controller/')
342 self.assertEqual(rows[0].find('a').text, 'controller/') 346 self.assertEqual(rows[0].find('a').text, 'controller/')
343 347
344 self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/') 348 self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
345 self.assertEqual(rows[1].find('a').text, 'zuul-info/') 349 self.assertEqual(rows[1].find('a').text, 'zuul-info/')
346 350
347 subdir_index = self.find_file(fl, 'controller/subdir/index.html') 351 subdir_index = self.find_file(fl, 'controller/subdir/index.html')
348 page = open(subdir_index.full_path).read() 352 page = open(subdir_index.full_path).read()
349 page = BeautifulSoup(page, 'html.parser') 353 page = BeautifulSoup(page, 'html.parser')
350 rows = page.find_all('tr')[1:] 354 rows = page.find_all('tr')[1:]
351 355
352 self.assertEqual(rows[0].find('a').get('href'), 'subdir.txt') 356 self.assertEqual(rows[0].find('a').get('href'), 'subdir.txt')
353 self.assertEqual(rows[0].find('a').text, 'subdir.txt') 357 self.assertEqual(rows[0].find('a').text, 'subdir.txt')
diff --git a/roles/upload-logs-swift/library/zuul_swift_upload.py b/roles/upload-logs-swift/library/zuul_swift_upload.py
index e2aa46c..305a5d3 100755
--- a/roles/upload-logs-swift/library/zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/zuul_swift_upload.py
@@ -185,10 +185,21 @@ class FileDetail():
185 185
186 186
187class FileList(Sequence): 187class FileList(Sequence):
188 '''A collection of FileDetail objects
189
190 This is a list-like group of FileDetail objects, intended to be
191 used as a context manager around the upload process.
192 '''
188 def __init__(self): 193 def __init__(self):
189 self.file_list = [] 194 self.file_list = []
190 self.file_list.append(FileDetail(None, '', '')) 195 self.file_list.append(FileDetail(None, '', ''))
191 196
197 def __enter__(self):
198 return self
199
200 def __exit__(self, type, value, traceback):
201 pass
202
192 def __getitem__(self, item): 203 def __getitem__(self, item):
193 return self.file_list.__getitem__(item) 204 return self.file_list.__getitem__(item)
194 205
@@ -269,6 +280,16 @@ class Indexer():
269 self.index_filename = 'index.html' 280 self.index_filename = 'index.html'
270 281
271 def make_indexes(self, file_list): 282 def make_indexes(self, file_list):
283 '''Make index files
284
285 Args:
286 file_list (FileList): A FileList object to be updated
287 with index files for each directory.
288 Return:
289 No value, the file_list will be updated
290 '''
291 assert isinstance(file_list, FileList)
292
272 folders = collections.OrderedDict() 293 folders = collections.OrderedDict()
273 for f in file_list: 294 for f in file_list:
274 if f.folder: 295 if f.folder:
@@ -302,8 +323,7 @@ class Indexer():
302 323
303 # This appends the index file at the end of the group of files 324 # This appends the index file at the end of the group of files
304 # for each directory. 325 # for each directory.
305 ret_file_list = FileList() 326 new_list = []
306 newlist = []
307 last_dirname = None 327 last_dirname = None
308 for f in reversed(list(file_list)): 328 for f in reversed(list(file_list)):
309 if f.folder: 329 if f.folder:
@@ -316,13 +336,11 @@ class Indexer():
316 if dirname != last_dirname: 336 if dirname != last_dirname:
317 index = indexes.pop(dirname, None) 337 index = indexes.pop(dirname, None)
318 if index: 338 if index:
319 newlist.append(index) 339 new_list.append(index)
320 last_dirname = dirname 340 last_dirname = dirname
321 newlist.append(f) 341 new_list.append(f)
322 newlist.reverse() 342 new_list.reverse()
323 ret_file_list.file_list = newlist 343 file_list.file_list = new_list
324
325 return ret_file_list
326 344
327 def make_index_file(self, folder_links, title): 345 def make_index_file(self, folder_links, title):
328 """Writes an index into a file for pushing""" 346 """Writes an index into a file for pushing"""
@@ -547,33 +565,33 @@ def run(cloud, container, files,
547 prefix = '/'.join(parts[1:]) 565 prefix = '/'.join(parts[1:])
548 566
549 # Create the objects to make sure the arguments are sound. 567 # Create the objects to make sure the arguments are sound.
550 file_list = FileList() 568 with FileList() as file_list:
551 indexer = Indexer(create_parent_links=parent_links, 569 indexer = Indexer(create_parent_links=parent_links,
552 create_topdir_parent_link=topdir_parent_link, 570 create_topdir_parent_link=topdir_parent_link,
553 append_footer=footer) 571 append_footer=footer)
554 572
555 # Scan the files. 573 # Scan the files.
556 for file_path in files: 574 for file_path in files:
557 file_list.add(file_path) 575 file_list.add(file_path)
558 576
559 # (Possibly) make indexes. 577 # (Possibly) make indexes.
560 if indexes: 578 if indexes:
561 file_list = indexer.make_indexes(file_list) 579 indexer.make_indexes(file_list)
562 580
563 logging.debug("List of files prepared to upload:") 581 logging.debug("List of files prepared to upload:")
564 for x in file_list: 582 for x in file_list:
565 logging.debug(x) 583 logging.debug(x)
566 584
567 # Do no connect to swift or do any uploading in a dry run 585 # Do no connect to swift or do any uploading in a dry run
568 if dry_run: 586 if dry_run:
569 # No URL is known, so return nothing 587 # No URL is known, so return nothing
570 return 588 return
571 589
572 # Upload. 590 # Upload.
573 uploader = Uploader(cloud, container, prefix, delete_after, 591 uploader = Uploader(cloud, container, prefix, delete_after,
574 public) 592 public)
575 uploader.upload(file_list) 593 uploader.upload(file_list)
576 return uploader.url 594 return uploader.url
577 595
578 596
579def ansible_main(): 597def ansible_main():