Merge "Add HTMLify logs role"

2018-08-03 23:09:21 +00:00 · 2018-08-03 23:09:21 +00:00 · ad44415d0b
parent 8bee69d67a 3f6190cb65
commit ad44415d0b
8 changed files with 372 additions and 0 deletions
--- a/roles/htmlify-logs/README.rst
+++ b/roles/htmlify-logs/README.rst
@ -0,0 +1,5 @@
+HTMLify text logs
+
+This makes an HTML version of every file in the executor log directory
+with a ``.txt`` or ``.txt.gz`` extension.  If the original was
+gzipped, the HTML version will be as well.
--- a/roles/htmlify-logs/library/init.py
+++ b/roles/htmlify-logs/library/init.py
--- a/roles/htmlify-logs/library/htmlify.py
+++ b/roles/htmlify-logs/library/htmlify.py
@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# Copyright 2018 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import cgi
+import argparse
+import gzip
+import sys
+
+from ansible.module_utils.basic import AnsibleModule
+
+HEADER = '''<html>
+<head>
+<style>
+a {color: #000; text-decoration: none}
+a:hover {text-decoration: underline}
+#selector, #selector a {color: #888}
+#selector a:hover {color: #c00}
+.highlight {
+    background-color: rgb(255, 255, 204);
+}
+</style>
+</head>
+<body>
+<pre>
+'''
+
+FOOTER = '''</pre>
+</body>
+<script>
+var old_highlight;
+
+function remove_highlight() {
+    if (old_highlight) {
+        items = document.getElementsByClassName(old_highlight);
+        for (var i = 0; i < items.length; i++) {
+            items[i].className = items[i].className.replace('highlight','');
+        }
+    }
+}
+
+function update_selector(highlight) {
+    var selector = document.getElementById('selector');
+    if (selector) {
+        var links = selector.getElementsByTagName('a');
+        for (var i = 0; i < links.length; i++) {
+            links[i].hash = "#" + highlight;
+        }
+    }
+}
+
+function highlight_by_hash(event) {
+    var highlight = window.location.hash.substr(1);
+    // handle changes to highlighting separate from reload
+    if (event) {
+         highlight = event.target.hash.substr(1);
+    }
+    remove_highlight();
+    if (highlight) {
+        elements = document.getElementsByClassName(highlight);
+        for (var i = 0; i < elements.length; i++) {
+            elements[i].className += " highlight";
+        }
+        update_selector(highlight);
+        old_highlight = highlight;
+    }
+}
+document.onclick = highlight_by_hash;
+highlight_by_hash();
+</script>
+</html>
+'''
+
+
+def run(inpath, outpath):
+    if inpath.endswith('.gz'):
+        infile = gzip.open(inpath, 'r')
+        outfile = gzip.open(outpath, 'w')
+    else:
+        infile = open(inpath, 'r')
+        outfile = open(outpath, 'w')
+
+    outfile.write(HEADER)
+
+    for i, line in enumerate(infile):
+        i = i + 1
+        line = cgi.escape(line.rstrip('\n'))
+        outfile.write('<a name="l%s" class="l%s" href="#l%s">' % (i, i, i))
+        outfile.write(line.rstrip('\n'))
+        outfile.write("</a>\n")
+
+    outfile.write(FOOTER)
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            input=dict(required=True, type='path'),
+            output=dict(required=True, type='path'),
+        )
+    )
+
+    p = module.params
+    run(p.get('input'), p.get('output'))
+
+    module.exit_json(changed=True)
+
+
+def cli_main():
+    parser = argparse.ArgumentParser(
+        description="HTMLify text files"
+    )
+    parser.add_argument('input',
+                        help='Input path')
+    parser.add_argument('output',
+                        help='Output path')
+
+    args = parser.parse_args()
+
+    run(args.input, args.output)
+
+
+if __name__ == '__main__':
+
+    if sys.stdin.isatty():
+        cli_main()
+    else:
+        ansible_main()
--- a/roles/htmlify-logs/library/test-fixtures/in/job-output.txt
+++ b/roles/htmlify-logs/library/test-fixtures/in/job-output.txt
@ -0,0 +1,54 @@
+2018-08-01 00:43:51.328884 | Job console starting...
+2018-08-01 00:44:01.742989 | PRE-RUN START: [trusted : git.openstack.org/openstack-infra/project-config/playbooks/base/pre.yaml@master]
+2018-08-01 00:44:04.693529 | 
+2018-08-01 00:44:04.693765 | PLAY [localhost]
+2018-08-01 00:44:04.736664 | 
+2018-08-01 00:44:04.736863 | TASK [emit-job-header : Setup log path fact]
+2018-08-01 00:44:04.804077 | localhost | ok
+2018-08-01 00:44:04.870027 | 
+2018-08-01 00:44:04.870264 | TASK [set-zuul-log-path-fact : Set log path for a change]
+2018-08-01 00:44:04.942854 | localhost | skipping: Conditional result was False
+2018-08-01 00:44:04.970178 | 
+2018-08-01 00:44:04.970405 | TASK [set-zuul-log-path-fact : Set log path for a ref update]
+2018-08-01 00:44:05.067095 | localhost | ok
+2018-08-01 00:44:05.094186 | 
+2018-08-01 00:44:05.094430 | TASK [set-zuul-log-path-fact : Set log path for a periodic job]
+2018-08-01 00:44:05.152841 | localhost | skipping: Conditional result was False
+2018-08-01 00:44:05.188754 | 
+2018-08-01 00:44:05.189030 | TASK [emit-job-header : Print job information]
+2018-08-01 00:44:05.317656 | # Job Information
+2018-08-01 00:44:05.318162 | Ansible Version: 2.5.7
+2018-08-01 00:44:05.318269 | Job: trigger-readthedocs-webhook
+2018-08-01 00:44:05.318371 | Pipeline: post
+2018-08-01 00:44:05.318469 | Executor: ze09.openstack.org
+2018-08-01 00:44:05.318567 | Triggered by: https://git.openstack.org/cgit/openstack/gerrit-dash-creator/commit/?id=ee5167a518de07c325f326df212c92e5c1e786a9
+2018-08-01 00:44:05.318663 | Log URL (when completed): http://logs.openstack.org/ee/ee5167a518de07c325f326df212c92e5c1e786a9/post/trigger-readthedocs-webhook/f677f03
+2018-08-01 00:44:05.349043 | 
+2018-08-01 00:44:05.349250 | PLAY [all]
+2018-08-01 00:44:05.385083 | 
+2018-08-01 00:44:05.385308 | TASK [Gather network facts]
+2018-08-01 00:44:07.104842 | ubuntu-xenial | ok
+2018-08-01 00:44:07.167964 | 
+2018-08-01 00:44:07.168193 | TASK [add-build-sshkey : Check to see if ssh key was already created for this build]
+2018-08-01 00:44:07.760495 | ubuntu-xenial -> localhost | ok
+2018-08-01 00:44:07.799900 | 
+2018-08-01 00:44:07.800163 | TASK [add-build-sshkey : Create Temp SSH key]
+2018-08-01 00:44:08.445211 | ubuntu-xenial -> localhost | Generating public/private rsa key pair.
+2018-08-01 00:44:08.445859 | ubuntu-xenial -> localhost | Your identification has been saved in /var/lib/zuul/builds/f677f0312811438a8db3fa38953649c7/work/f677f0312811438a8db3fa38953649c7_id_rsa.
+2018-08-01 00:44:08.445989 | ubuntu-xenial -> localhost | Your public key has been saved in /var/lib/zuul/builds/f677f0312811438a8db3fa38953649c7/work/f677f0312811438a8db3fa38953649c7_id_rsa.pub.
+2018-08-01 00:44:08.446099 | ubuntu-xenial -> localhost | The key fingerprint is:
+2018-08-01 00:44:08.446205 | ubuntu-xenial -> localhost | SHA256:ZNugwiDo1mjkhNQt/GnNoH3XQgo+uU/jA8NtHUKCPiY zuul@ze09
+2018-08-01 00:44:08.446320 | ubuntu-xenial -> localhost | The key's randomart image is:
+2018-08-01 00:44:08.446424 | ubuntu-xenial -> localhost | +---[RSA 1024]----+
+2018-08-01 00:44:08.446527 | ubuntu-xenial -> localhost | | .o o            |
+2018-08-01 00:44:08.446630 | ubuntu-xenial -> localhost | |+  = = . .       |
+2018-08-01 00:44:08.446732 | ubuntu-xenial -> localhost | |+oo * X * .      |
+2018-08-01 00:44:08.446834 | ubuntu-xenial -> localhost | |=EoB O X B .     |
+2018-08-01 00:44:08.446936 | ubuntu-xenial -> localhost | | *o.* * S +      |
+2018-08-01 00:44:08.447037 | ubuntu-xenial -> localhost | |o    * = .       |
+2018-08-01 00:44:08.447138 | ubuntu-xenial -> localhost | |      B .        |
+2018-08-01 00:44:08.447239 | ubuntu-xenial -> localhost | |       +         |
+2018-08-01 00:44:08.447339 | ubuntu-xenial -> localhost | |        .        |
+2018-08-01 00:44:08.447440 | ubuntu-xenial -> localhost | +----[SHA256]-----+
+2018-08-01 00:44:08.447643 | ubuntu-xenial -> localhost | ok: Runtime: 0:00:00.022230
+here is a line with some <escapes>
--- a/roles/htmlify-logs/library/test-fixtures/in/job-output.txt.gz
+++ b/roles/htmlify-logs/library/test-fixtures/in/job-output.txt.gz
--- a/roles/htmlify-logs/library/test-fixtures/reference/job-output.txt
+++ b/roles/htmlify-logs/library/test-fixtures/reference/job-output.txt
@ -0,0 +1,112 @@
+<html>
+<head>
+<style>
+a {color: #000; text-decoration: none}
+a:hover {text-decoration: underline}
+#selector, #selector a {color: #888}
+#selector a:hover {color: #c00}
+.highlight {
+    background-color: rgb(255, 255, 204);
+}
+</style>
+</head>
+<body>
+<pre>
+<a name="l1" class="l1" href="#l1">2018-08-01 00:43:51.328884 | Job console starting...</a>
+<a name="l2" class="l2" href="#l2">2018-08-01 00:44:01.742989 | PRE-RUN START: [trusted : git.openstack.org/openstack-infra/project-config/playbooks/base/pre.yaml@master]</a>
+<a name="l3" class="l3" href="#l3">2018-08-01 00:44:04.693529 | </a>
+<a name="l4" class="l4" href="#l4">2018-08-01 00:44:04.693765 | PLAY [localhost]</a>
+<a name="l5" class="l5" href="#l5">2018-08-01 00:44:04.736664 | </a>
+<a name="l6" class="l6" href="#l6">2018-08-01 00:44:04.736863 | TASK [emit-job-header : Setup log path fact]</a>
+<a name="l7" class="l7" href="#l7">2018-08-01 00:44:04.804077 | localhost | ok</a>
+<a name="l8" class="l8" href="#l8">2018-08-01 00:44:04.870027 | </a>
+<a name="l9" class="l9" href="#l9">2018-08-01 00:44:04.870264 | TASK [set-zuul-log-path-fact : Set log path for a change]</a>
+<a name="l10" class="l10" href="#l10">2018-08-01 00:44:04.942854 | localhost | skipping: Conditional result was False</a>
+<a name="l11" class="l11" href="#l11">2018-08-01 00:44:04.970178 | </a>
+<a name="l12" class="l12" href="#l12">2018-08-01 00:44:04.970405 | TASK [set-zuul-log-path-fact : Set log path for a ref update]</a>
+<a name="l13" class="l13" href="#l13">2018-08-01 00:44:05.067095 | localhost | ok</a>
+<a name="l14" class="l14" href="#l14">2018-08-01 00:44:05.094186 | </a>
+<a name="l15" class="l15" href="#l15">2018-08-01 00:44:05.094430 | TASK [set-zuul-log-path-fact : Set log path for a periodic job]</a>
+<a name="l16" class="l16" href="#l16">2018-08-01 00:44:05.152841 | localhost | skipping: Conditional result was False</a>
+<a name="l17" class="l17" href="#l17">2018-08-01 00:44:05.188754 | </a>
+<a name="l18" class="l18" href="#l18">2018-08-01 00:44:05.189030 | TASK [emit-job-header : Print job information]</a>
+<a name="l19" class="l19" href="#l19">2018-08-01 00:44:05.317656 | # Job Information</a>
+<a name="l20" class="l20" href="#l20">2018-08-01 00:44:05.318162 | Ansible Version: 2.5.7</a>
+<a name="l21" class="l21" href="#l21">2018-08-01 00:44:05.318269 | Job: trigger-readthedocs-webhook</a>
+<a name="l22" class="l22" href="#l22">2018-08-01 00:44:05.318371 | Pipeline: post</a>
+<a name="l23" class="l23" href="#l23">2018-08-01 00:44:05.318469 | Executor: ze09.openstack.org</a>
+<a name="l24" class="l24" href="#l24">2018-08-01 00:44:05.318567 | Triggered by: https://git.openstack.org/cgit/openstack/gerrit-dash-creator/commit/?id=ee5167a518de07c325f326df212c92e5c1e786a9</a>
+<a name="l25" class="l25" href="#l25">2018-08-01 00:44:05.318663 | Log URL (when completed): http://logs.openstack.org/ee/ee5167a518de07c325f326df212c92e5c1e786a9/post/trigger-readthedocs-webhook/f677f03</a>
+<a name="l26" class="l26" href="#l26">2018-08-01 00:44:05.349043 | </a>
+<a name="l27" class="l27" href="#l27">2018-08-01 00:44:05.349250 | PLAY [all]</a>
+<a name="l28" class="l28" href="#l28">2018-08-01 00:44:05.385083 | </a>
+<a name="l29" class="l29" href="#l29">2018-08-01 00:44:05.385308 | TASK [Gather network facts]</a>
+<a name="l30" class="l30" href="#l30">2018-08-01 00:44:07.104842 | ubuntu-xenial | ok</a>
+<a name="l31" class="l31" href="#l31">2018-08-01 00:44:07.167964 | </a>
+<a name="l32" class="l32" href="#l32">2018-08-01 00:44:07.168193 | TASK [add-build-sshkey : Check to see if ssh key was already created for this build]</a>
+<a name="l33" class="l33" href="#l33">2018-08-01 00:44:07.760495 | ubuntu-xenial -&gt; localhost | ok</a>
+<a name="l34" class="l34" href="#l34">2018-08-01 00:44:07.799900 | </a>
+<a name="l35" class="l35" href="#l35">2018-08-01 00:44:07.800163 | TASK [add-build-sshkey : Create Temp SSH key]</a>
+<a name="l36" class="l36" href="#l36">2018-08-01 00:44:08.445211 | ubuntu-xenial -&gt; localhost | Generating public/private rsa key pair.</a>
+<a name="l37" class="l37" href="#l37">2018-08-01 00:44:08.445859 | ubuntu-xenial -&gt; localhost | Your identification has been saved in /var/lib/zuul/builds/f677f0312811438a8db3fa38953649c7/work/f677f0312811438a8db3fa38953649c7_id_rsa.</a>
+<a name="l38" class="l38" href="#l38">2018-08-01 00:44:08.445989 | ubuntu-xenial -&gt; localhost | Your public key has been saved in /var/lib/zuul/builds/f677f0312811438a8db3fa38953649c7/work/f677f0312811438a8db3fa38953649c7_id_rsa.pub.</a>
+<a name="l39" class="l39" href="#l39">2018-08-01 00:44:08.446099 | ubuntu-xenial -&gt; localhost | The key fingerprint is:</a>
+<a name="l40" class="l40" href="#l40">2018-08-01 00:44:08.446205 | ubuntu-xenial -&gt; localhost | SHA256:ZNugwiDo1mjkhNQt/GnNoH3XQgo+uU/jA8NtHUKCPiY zuul@ze09</a>
+<a name="l41" class="l41" href="#l41">2018-08-01 00:44:08.446320 | ubuntu-xenial -&gt; localhost | The key's randomart image is:</a>
+<a name="l42" class="l42" href="#l42">2018-08-01 00:44:08.446424 | ubuntu-xenial -&gt; localhost | +---[RSA 1024]----+</a>
+<a name="l43" class="l43" href="#l43">2018-08-01 00:44:08.446527 | ubuntu-xenial -&gt; localhost | | .o o            |</a>
+<a name="l44" class="l44" href="#l44">2018-08-01 00:44:08.446630 | ubuntu-xenial -&gt; localhost | |+  = = . .       |</a>
+<a name="l45" class="l45" href="#l45">2018-08-01 00:44:08.446732 | ubuntu-xenial -&gt; localhost | |+oo * X * .      |</a>
+<a name="l46" class="l46" href="#l46">2018-08-01 00:44:08.446834 | ubuntu-xenial -&gt; localhost | |=EoB O X B .     |</a>
+<a name="l47" class="l47" href="#l47">2018-08-01 00:44:08.446936 | ubuntu-xenial -&gt; localhost | | *o.* * S +      |</a>
+<a name="l48" class="l48" href="#l48">2018-08-01 00:44:08.447037 | ubuntu-xenial -&gt; localhost | |o    * = .       |</a>
+<a name="l49" class="l49" href="#l49">2018-08-01 00:44:08.447138 | ubuntu-xenial -&gt; localhost | |      B .        |</a>
+<a name="l50" class="l50" href="#l50">2018-08-01 00:44:08.447239 | ubuntu-xenial -&gt; localhost | |       +         |</a>
+<a name="l51" class="l51" href="#l51">2018-08-01 00:44:08.447339 | ubuntu-xenial -&gt; localhost | |        .        |</a>
+<a name="l52" class="l52" href="#l52">2018-08-01 00:44:08.447440 | ubuntu-xenial -&gt; localhost | +----[SHA256]-----+</a>
+<a name="l53" class="l53" href="#l53">2018-08-01 00:44:08.447643 | ubuntu-xenial -&gt; localhost | ok: Runtime: 0:00:00.022230</a>
+<a name="l54" class="l54" href="#l54">here is a line with some &lt;escapes&gt;</a>
+</pre>
+</body>
+<script>
+var old_highlight;
+
+function remove_highlight() {
+    if (old_highlight) {
+        items = document.getElementsByClassName(old_highlight);
+        for (var i = 0; i < items.length; i++) {
+            items[i].className = items[i].className.replace('highlight','');
+        }
+    }
+}
+
+function update_selector(highlight) {
+    var selector = document.getElementById('selector');
+    if (selector) {
+        var links = selector.getElementsByTagName('a');
+        for (var i = 0; i < links.length; i++) {
+            links[i].hash = "#" + highlight;
+        }
+    }
+}
+
+function highlight_by_hash(event) {
+    var highlight = window.location.hash.substr(1);
+    // handle changes to highlighting separate from reload
+    if (event) {
+         highlight = event.target.hash.substr(1);
+    }
+    remove_highlight();
+    if (highlight) {
+        elements = document.getElementsByClassName(highlight);
+        for (var i = 0; i < elements.length; i++) {
+            elements[i].className += " highlight";
+        }
+        update_selector(highlight);
+        old_highlight = highlight;
+    }
+}
+document.onclick = highlight_by_hash;
+highlight_by_hash();
+</script>
+</html>
--- a/roles/htmlify-logs/library/test_htmlify.py
+++ b/roles/htmlify-logs/library/test_htmlify.py
@ -0,0 +1,48 @@
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gzip
+import os
+
+import testtools
+import fixtures
+
+from .htmlify import run
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+                           'test-fixtures')
+
+
+class TestHTMLify(testtools.TestCase):
+
+    def _test_file(self, fn):
+        in_path = os.path.join(FIXTURE_DIR, 'in', fn)
+        ref_path = os.path.join(FIXTURE_DIR, 'reference', fn)
+        out_root = self.useFixture(fixtures.TempDir()).path
+        out_path = os.path.join(out_root, fn)
+        run(in_path, out_path)
+
+        if fn.endswith('gz'):
+            out = gzip.open(out_path, 'rb')
+        else:
+            out = open(out_path, 'rb')
+
+        generated_data = out.read()
+        reference_data = open(ref_path, 'rb').read()
+        self.assertEqual(reference_data, generated_data)
+
+    def test_htmlify(self):
+        self._test_file('job-output.txt')
--- a/roles/htmlify-logs/tasks/main.yaml
+++ b/roles/htmlify-logs/tasks/main.yaml
@ -0,0 +1,14 @@
+- name: Find text files to HTMLify
+  find:
+    paths: "{{ zuul.executor.log_root }}"
+    recurse: true
+    patterns:
+      - "*.txt"
+      - "*.txt.gz"
+  register: htmlify_files
+
+- name: HTMLify text files
+  htmlify:
+    input: "{{ item.path }}"
+    output: "{{ item.path | regex_replace('\\.txt', '.html') }}"
+  loop: "{{ htmlify_files.files }}"