From 3d8387c22bbb371aad3d14ec3e4cbf78aff169ba Mon Sep 17 00:00:00 2001
From: Angelo Nakos <3038292+angelonakos@users.noreply.github.com>
Date: Fri, 28 Jan 2022 15:10:01 -0500
Subject: [PATCH 1/3] Arbitrary text, HTML content can be rendered as the
literad stdout of job
This commit introduces the `literal-stdout` tag inside `litani add-job`. This
allows text, HTML content, SVGs to be rendered as the output of a Litani job
within the pipeline page. A relevant example is included within:
examples/rich-outputs/run-1.sh
Fixes: #81
---
doc/src/man/litani-add-job.scdoc | 3 +-
examples/rich-output/README.md | 10 +++++++
examples/rich-output/assumptions.html | 41 +++++++++++++++++++++++++++
examples/rich-output/file.dat | 7 +++++
examples/rich-output/histogram.dat | 7 +++++
examples/rich-output/run-1.sh | 41 +++++++++++++++++++++++++++
examples/rich-output/run-all.py | 34 ++++++++++++++++++++++
templates/pipeline.jinja.html | 28 ++++++++++++++----
8 files changed, 164 insertions(+), 7 deletions(-)
create mode 100644 examples/rich-output/README.md
create mode 100644 examples/rich-output/assumptions.html
create mode 100644 examples/rich-output/file.dat
create mode 100644 examples/rich-output/histogram.dat
create mode 100755 examples/rich-output/run-1.sh
create mode 100755 examples/rich-output/run-all.py
diff --git a/doc/src/man/litani-add-job.scdoc b/doc/src/man/litani-add-job.scdoc
index 9647a33f..1fd541bb 100644
--- a/doc/src/man/litani-add-job.scdoc
+++ b/doc/src/man/litani-add-job.scdoc
@@ -123,7 +123,8 @@ configure times.
dashboard generator does use some of them). Each tag can be in whatever format
you prefer, e.g. a plain string, key-value pair, or even arbitrary JSON. A
job's list of tags is included in its _run.json_ and is intended to help with
- analyzing run data.
+ analyzing run data. If `literal-stdout` is provided as a tag, then the
+ report will render the literal standard output generated by the job.
*--pipeline-name* _P_
The 'pipeline' that this job is part of. A pipeline is a subgraph of the
diff --git a/examples/rich-output/README.md b/examples/rich-output/README.md
new file mode 100644
index 00000000..daed1dd5
--- /dev/null
+++ b/examples/rich-output/README.md
@@ -0,0 +1,10 @@
+README
+
+The run-all.py script will run examples that demonstrate how jobs in Litani can
+display "rich" output either on the dasboard or pipeline pages
+
+To try this out, run
+
+ LITANI=../../litani ./run-all.py
+
+in this directory.
diff --git a/examples/rich-output/assumptions.html b/examples/rich-output/assumptions.html
new file mode 100644
index 00000000..800f4d32
--- /dev/null
+++ b/examples/rich-output/assumptions.html
@@ -0,0 +1,41 @@
+
+
+
+
+
+
+ Assumption to check |
+ Difficulty |
+
+
+ Bound of input size function |
+ Easy |
+
+
+ Harness |
+ Medium |
+
+
+ Function specifications |
+ Hard |
+
+
+
+
+
\ No newline at end of file
diff --git a/examples/rich-output/file.dat b/examples/rich-output/file.dat
new file mode 100644
index 00000000..b8b47cee
--- /dev/null
+++ b/examples/rich-output/file.dat
@@ -0,0 +1,7 @@
+pipeline #Assumptions
+"Class\\\_A" 5
+"Class\\\_B" 1
+"Class\\\_C" 2
+"Class\\\_D" 8
+"Class\\\_E" 4
+"Class\\\_F" 0
diff --git a/examples/rich-output/histogram.dat b/examples/rich-output/histogram.dat
new file mode 100644
index 00000000..e6c81e3e
--- /dev/null
+++ b/examples/rich-output/histogram.dat
@@ -0,0 +1,7 @@
+set terminal svg noenhanced size 565,1096
+set style data histogram
+set style fill solid border
+set style histogram clustered
+set xtics rotate out
+set yrange [ 0 : 10 ]
+plot for [COL=2:2] 'file.dat' using COL:xticlabels(1) title columnheader
diff --git a/examples/rich-output/run-1.sh b/examples/rich-output/run-1.sh
new file mode 100755
index 00000000..f1938a3e
--- /dev/null
+++ b/examples/rich-output/run-1.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+LITANI=${LITANI:-litani}
+
+if [ "$1" != "--no-standalone" ]; then
+ "${LITANI}" init --project-name "literal-stdout-examples"
+fi
+
+"${LITANI}" add-job \
+ --pipeline-name "verify assumptions" \
+ --description "list out assumptions" \
+ --command "cat assumptions.html" \
+ --ci-stage test \
+ --phony-outputs assumptions.html \
+ --tags literal-stdout
+
+"${LITANI}" add-job \
+ --pipeline-name "conduct analysis" \
+ --description "generate the histogram" \
+ --command "gnuplot -c histogram.dat" \
+ --ci-stage report \
+ --inputs file.dat \
+ --phony-outputs histogram.svg \
+ --tags literal-stdout
+
+if [ "$1" != "--no-standalone" ]; then
+ "${LITANI}" run-build
+fi
diff --git a/examples/rich-output/run-all.py b/examples/rich-output/run-all.py
new file mode 100755
index 00000000..6998a5ca
--- /dev/null
+++ b/examples/rich-output/run-all.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+
+import os
+import pathlib
+import re
+import subprocess
+
+
+litani = os.getenv("LITANI", "litani")
+this_dir = pathlib.Path(__file__).parent
+
+subprocess.run(
+ [litani, "init", "--project-name", "literal-stdout-examples"], check=True)
+
+pat = re.compile(r"run-\d.sh")
+for fyle in this_dir.iterdir():
+ if pat.match(fyle.name):
+ subprocess.run([fyle, "--no-standalone"], check=True)
+
+subprocess.run([litani, "run-build"], check=True)
diff --git a/templates/pipeline.jinja.html b/templates/pipeline.jinja.html
index fe04f7f2..cc537687 100644
--- a/templates/pipeline.jinja.html
+++ b/templates/pipeline.jinja.html
@@ -194,6 +194,14 @@
text-shadow: 0 0 2px #82ada944;
font-weight: bold;
}
+.literal-stdout {
+ margin-top: 0.4em;
+ margin-bottom: 0.2em;
+ padding: 1em;
+ border-radius: 0.5em;
+ overflow-x: auto;
+ overflow-y: auto;
+}
.output-box {
margin-top: 0.4em;
margin-bottom: 0.2em;
@@ -773,12 +781,20 @@
{% if job["stdout"] %}
-
stdout:
- {% for line in job["stdout"] %}
-{{ line.strip() }}
- {%- endfor %}{# line in job["stdout"] #}
-
-
+ {% if job["wrapper_arguments"]["tags"] is not none and "literal-stdout" in job["wrapper_arguments"]["tags"] %}
+
+ {% for line in job["stdout"] %}
+ {{ line.strip() }}
+ {%- endfor %}{# line in job["stdout"] #}
+
+ {% else %}
+ stdout:
+ {% for line in job["stdout"] %}
+ {{ line.strip() }}
+ {%- endfor %}{# line in job["stdout"] #}
+
+
+ {% endif %}{# is literal-stdout in tags #}
{% endif %}{# job["stdout"] #}
{% if job["stderr"] %}
From 9b92bac5689f2e2583e3382747c02de82d59a516 Mon Sep 17 00:00:00 2001
From: Nicholas Rodgers
Date: Thu, 3 Feb 2022 16:27:03 -0500
Subject: [PATCH 2/3] Add front-page-text tag functionality
When "front-page-text" is passed as a tag to litani add-job, render the
stdout of the job on the dashboard page
---
doc/src/man/litani-add-job.scdoc | 4 +-
lib/litani_report.py | 26 ++++++++-----
templates/dashboard.jinja.html | 67 ++++++++++++++++++++++++++++++++
3 files changed, 87 insertions(+), 10 deletions(-)
diff --git a/doc/src/man/litani-add-job.scdoc b/doc/src/man/litani-add-job.scdoc
index 1fd541bb..49557ad6 100644
--- a/doc/src/man/litani-add-job.scdoc
+++ b/doc/src/man/litani-add-job.scdoc
@@ -124,7 +124,9 @@ configure times.
you prefer, e.g. a plain string, key-value pair, or even arbitrary JSON. A
job's list of tags is included in its _run.json_ and is intended to help with
analyzing run data. If `literal-stdout` is provided as a tag, then the
- report will render the literal standard output generated by the job.
+ report will render the literal standard output generated by the job. If
+ 'front-page-output' is provided as a tag, then the report will render the
+ literal standard out generated by the job on Litani's dashboard.
*--pipeline-name* _P_
The 'pipeline' that this job is part of. A pipeline is a subgraph of the
diff --git a/lib/litani_report.py b/lib/litani_report.py
index 7b335dcf..f24b4940 100644
--- a/lib/litani_report.py
+++ b/lib/litani_report.py
@@ -544,18 +544,10 @@ def render(run, report_dir, pipeline_depgraph_renderer):
litani_report_archive_path = os.getenv("LITANI_REPORT_ARCHIVE_PATH")
- dash_templ = env.get_template("dashboard.jinja.html")
- page = dash_templ.render(
- run=run, svgs=svgs,
- litani_version=litani.VERSION,
- litani_report_archive_path=litani_report_archive_path,
- summary=get_summary(run))
- with litani.atomic_write(temporary_report_dir / "index.html") as handle:
- print(page, file=handle)
-
with litani.atomic_write(temporary_report_dir / litani.RUN_FILE) as handle:
print(json.dumps(run, indent=2), file=handle)
+ front_page_outputs = {}
pipe_templ = env.get_template("pipeline.jinja.html")
for pipe in run["pipelines"]:
pipeline_depgraph_renderer.render(
@@ -570,11 +562,27 @@ def render(run, report_dir, pipeline_depgraph_renderer):
MemoryTraceRenderer.render(
temporary_report_dir / pipe["url"], env, job, gnuplot)
+ tags = job["wrapper_arguments"]["tags"]
+ description = job["wrapper_arguments"]["description"]
+ if tags and "front-page-text" in tags:
+ if "stdout" in job and job["stdout"]:
+ front_page_outputs[description] = job
+
pipe_page = pipe_templ.render(run=run, pipe=pipe)
with litani.atomic_write(
temporary_report_dir / pipe["url"] / "index.html") as handle:
print(pipe_page, file=handle)
+ dash_templ = env.get_template("dashboard.jinja.html")
+ page = dash_templ.render(
+ run=run, svgs=svgs,
+ litani_version=litani.VERSION,
+ litani_report_archive_path=litani_report_archive_path,
+ summary=get_summary(run), front_page_outputs=front_page_outputs)
+
+ with litani.atomic_write(temporary_report_dir / "index.html") as handle:
+ print(page, file=handle)
+
temp_symlink_dir = report_dir.with_name(report_dir.name + str(uuid.uuid4()))
os.symlink(temporary_report_dir, temp_symlink_dir)
diff --git a/templates/dashboard.jinja.html b/templates/dashboard.jinja.html
index 52bdf22d..95bf570b 100644
--- a/templates/dashboard.jinja.html
+++ b/templates/dashboard.jinja.html
@@ -251,6 +251,42 @@
font-weight: bold;
}
+.job-outputs-header {
+ color: #263238;
+}
+.job-output-content {
+ background-color: #eceff1;
+ padding: 0.3em 1em;
+ border-radius: 0.3em;
+ margin-bottom: 4em;
+}
+.job-output-title {
+ margin: 0.3em 0 1em;
+}
+.job-output-title a {
+ text-decoration: none;
+ color: #29b6f6;
+}
+.job-output-title a:link, .job-output-title a:visited {
+ color: #29b6f6;
+}
+.job-output-title a:hover, .job-output-title a:visited:hover {
+ color: #ec407a;
+ text-decoration: none;
+}
+.job-output-title a:active, .job-output-title a:visited:active {
+ color: #ffeb3b;
+ text-decoration: none;
+}
+.job-output-box {
+ margin-top: 0.4em;
+ margin-bottom: 0.2em;
+ padding: 1em;
+ border-radius: 0.5em;
+ overflow-x: auto;
+ overflow-y: auto;
+}
+
@media (max-width: 640px){
.pipeline-progress {
width: 10em;
@@ -279,6 +315,15 @@
.downloads a {
color: #29b6f6;
}
+ .job-outputs-header {
+ color: #eceff1;
+ }
+ .job-output-content {
+ background-color: #37474f;
+ }
+ .job-output-title {
+ color: #eceff1;
+ }
.pipeline-row {
color: #eceff1;
background-color: #37474f;
@@ -445,6 +490,28 @@
{% endfor %}{# pipe in run["pipelines"] #}
+ {% if front_page_outputs %}
+
+
+ {% for job_desc, job in front_page_outputs.items() %}
+
+
+
+ {% for line in job["stdout"] %}
+ {{ line.strip() }}
+ {%- endfor %}{# line in job["stdout"] #}
+
+
+ {% endfor %}{# job_desc, job in front_page_outputs.items() #}
+
+ {% endif %}{# front_page_outputs #}
{% for title, svg_list in svgs.items() %}
{% if svg_list %}
From 86ef71df6cdabbff726ab3eece57927c7856b354 Mon Sep 17 00:00:00 2001
From: Nicholas Rodgers
Date: Thu, 3 Feb 2022 16:34:31 -0500
Subject: [PATCH 3/3] Add rich-output examples
Add examples demonstrating the functionality of the front-page-text tag
as well as the front-page-text tag and literal-stdout tag together
---
.gitignore | 2 +
doc/src/man/litani-add-job.scdoc | 12 +++--
examples/rich-output/README.md | 2 +-
examples/rich-output/assumptions.html | 3 +-
examples/rich-output/file.dat | 14 +++---
examples/rich-output/run-2.sh | 47 +++++++++++++++++++
examples/rich-output/run-3.sh | 41 ++++++++++++++++
examples/rich-output/run-all.py | 2 +-
examples/rich-output/scripts/fib-table.py | 40 ++++++++++++++++
examples/rich-output/scripts/fib.plt | 5 ++
examples/rich-output/scripts/fib.py | 32 +++++++++++++
examples/rich-output/scripts/sin-output.py | 40 ++++++++++++++++
examples/rich-output/scripts/sin.plt | 5 ++
examples/rich-output/scripts/sin.py | 30 ++++++++++++
.../templates/fib-table.jinja.html | 45 ++++++++++++++++++
.../templates/sin-output.jinja.html | 30 ++++++++++++
templates/pipeline.jinja.html | 4 +-
17 files changed, 338 insertions(+), 16 deletions(-)
create mode 100755 examples/rich-output/run-2.sh
create mode 100755 examples/rich-output/run-3.sh
create mode 100644 examples/rich-output/scripts/fib-table.py
create mode 100644 examples/rich-output/scripts/fib.plt
create mode 100644 examples/rich-output/scripts/fib.py
create mode 100644 examples/rich-output/scripts/sin-output.py
create mode 100644 examples/rich-output/scripts/sin.plt
create mode 100644 examples/rich-output/scripts/sin.py
create mode 100644 examples/rich-output/templates/fib-table.jinja.html
create mode 100644 examples/rich-output/templates/sin-output.jinja.html
diff --git a/.gitignore b/.gitignore
index bcc5487c..8a85e71c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,8 @@ test/output
doc/tmp
doc/out
+examples/rich-output/*.csv
+
__pycache__/
build.ninja
diff --git a/doc/src/man/litani-add-job.scdoc b/doc/src/man/litani-add-job.scdoc
index 49557ad6..1661d481 100644
--- a/doc/src/man/litani-add-job.scdoc
+++ b/doc/src/man/litani-add-job.scdoc
@@ -123,10 +123,14 @@ configure times.
dashboard generator does use some of them). Each tag can be in whatever format
you prefer, e.g. a plain string, key-value pair, or even arbitrary JSON. A
job's list of tags is included in its _run.json_ and is intended to help with
- analyzing run data. If `literal-stdout` is provided as a tag, then the
- report will render the literal standard output generated by the job. If
- 'front-page-output' is provided as a tag, then the report will render the
- literal standard out generated by the job on Litani's dashboard.
+ analyzing run data.
+
+ If 'literal-stdout' is provided as a tag, then the report will render the
+ job's stdout underneath the boxes that indicate the command and its
+ statistics within the associated pipeline's page. If 'front-page-text' is
+ provided as a tag, then the report will render the job's stdout on the
+ front page of the report. These 2 tags allow users to render custom HTML
+ data, e.g. tables, graphs. Examples can be found at 'examples/rich-output'.
*--pipeline-name* _P_
The 'pipeline' that this job is part of. A pipeline is a subgraph of the
diff --git a/examples/rich-output/README.md b/examples/rich-output/README.md
index daed1dd5..2d73841e 100644
--- a/examples/rich-output/README.md
+++ b/examples/rich-output/README.md
@@ -1,7 +1,7 @@
README
The run-all.py script will run examples that demonstrate how jobs in Litani can
-display "rich" output either on the dasboard or pipeline pages
+display rich output either on the dashboard or pipeline pages
To try this out, run
diff --git a/examples/rich-output/assumptions.html b/examples/rich-output/assumptions.html
index 800f4d32..46321e79 100644
--- a/examples/rich-output/assumptions.html
+++ b/examples/rich-output/assumptions.html
@@ -38,4 +38,5 @@