[uss_qualifier/reports] Add timing report coloring and fix artifact generation pipeline

github_pages/static/index.md

@@ -10,6 +10,7 @@ These reports were generated during continuous integration for the most recent P
 
 * [Sequence view](./artifacts/uss_qualifier/reports/uspace/sequence)
 * [Tested requirements](./artifacts/uss_qualifier/reports/uspace/requirements)
+* [Timing report](./artifacts/uss_qualifier/reports/uspace/timing)
 * [Demonstrated capabilities](./artifacts/uss_qualifier/reports/uspace/capabilities.html)
 * [Raw report](./artifacts/uss_qualifier/reports/uspace/report.json) (large)
 
@@ -18,27 +19,32 @@ These reports were generated during continuous integration for the most recent P
 * [Raw report](./artifacts/uss_qualifier/reports/noop/report.json) (indented to be human-readable)
 * [Interactive report](./artifacts/uss_qualifier/reports/noop/report.html)
 * [Sequence view](./artifacts/uss_qualifier/reports/noop/sequence)
+* [Timing report](./artifacts/uss_qualifier/reports/noop/timing)
 
 ### [ASTM F3548-21 test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml)
 
 * [Sequence view](./artifacts/uss_qualifier/reports/f3548_self_contained/sequence)
 * [Tested requirements](./artifacts/uss_qualifier/reports/f3548_self_contained/gate3)
 * [Globally-expanded report](./artifacts/uss_qualifier/reports/f3548_self_contained/globally_expanded/report.html)
+* [Timing report](./artifacts/uss_qualifier/reports/f3548_self_contained/timing)
 
 ### [US UTM Implementation test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/utm_implementation_us/environments/local/test_1.jsonnet)
 
 * [Sequence view](./artifacts/uss_qualifier/reports/test_1/sequence)
 * [Tested requirements](./artifacts/uss_qualifier/reports/test_1/scd)
+* [Timing report](./artifacts/uss_qualifier/reports/test_1/timing)
 
 ### [ASTM F3411-22a test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml)
 
 * [Sequence view](./artifacts/uss_qualifier/reports/netrid_v22a/sequence)
 * [Tested requirements](./artifacts/uss_qualifier/reports/netrid_v22a/requirements)
+* [Timing report](./artifacts/uss_qualifier/reports/netrid_v22a/timing)
 
 ### [DSS integration test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml)
 
 * [Sequence view](./artifacts/uss_qualifier/reports/dss_probing/sequence)
 * [Tested requirements](./artifacts/uss_qualifier/reports/dss_probing/requirements)
+* [Timing report](./artifacts/uss_qualifier/reports/dss_probing/timing)
 
 ### [General flight authorization configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml)
 

monitoring/uss_qualifier/configurations/dev/dss_probing.yaml

@@ -57,6 +57,7 @@ v1:
         participant_requirements:
           uss1: null
           uss2: all_astm_dss_requirements
+    timing_report: { }
   validation:
     criteria:
       - $ref: ./library/validation.yaml#/execution_error_none

monitoring/uss_qualifier/make_artifacts.sh

@@ -3,11 +3,14 @@
 set -eo pipefail
 
 if [[ $# -lt 1 ]]; then
-  echo "Usage: $0 <CONFIG_NAME(s)> [<REPORT_NAME(s)> [<OUTPUT_PATH>]]"
+  echo "Usage: $0 <REPORT_PATH> [<CONFIG_NAME(s)> [<OUTPUT_PATH>]]"
   echo "Generates artifacts according to the specified configuration(s) using the specified report(s)"
-  echo "<REPORT_NAME>: Location of the report file (or multiple locations separated by commas).  Relative paths are relative to this folder.  Use file:// prefix to explicitly specify file-based location."
-  echo "<CONFIG_NAME>: Location of the configuration file (or multiple locations separated by commas)."
-  echo "<OUTPUT_PATH>: Location to which artifacts should be written (defaults to output/<SIMPLE_CONFIG_NAME>)"
+  echo "  <REPORT_PATH>: Location (on the host machine) of the report file.  Relative paths are RELATIVE TO THE REPO ROOT."
+  echo "  <CONFIG_NAME>: Location of the configuration file describing what artifacts to make.  Must be built into the monitoring image (in configurations/personal, for instance).  If not specified, use artifacts configuration from report."
+  echo "  <OUTPUT_PATH>: Location (on the host machine) to which artifacts should be written.  Defaults to folder containing the report."
+  echo "Examples:"
+  echo "  ./monitoring/uss_qualifier/make_artifacts.sh ~/Downloads/be0bbe7c-4670-43f5-906a-594be69087f4/report.json configurations.dev.f3548_self_contained"
+  echo "  ./make_artifacts.sh monitoring/uss_qualifier/output/f3548_self_contained/report.json configurations.personal.custom_artifacts"
   exit 1
 fi
 
@@ -26,23 +29,31 @@ cd monitoring || exit 1
 make image
 )
 
-REPORT_NAME="${1}"
-echo "Reading report(s) from: ${REPORT_NAME}"
-MAKE_ARTIFACTS_OPTIONS="--report $REPORT_NAME"
+REPORT_LOCATION="${1}"
+
+# TODO: Retrieve local copy of report if location starts with "http"
+
+REPORT_PATH=$(realpath "${REPORT_LOCATION}")
+echo "Reading report from (host machine): ${REPORT_PATH}"
+REPORT_FILENAME=$(basename "${REPORT_PATH}")
+MAKE_ARTIFACTS_OPTIONS="--report file:///input/${REPORT_FILENAME}"
 
 if [ "$#" -gt 1 ]; then
   CONFIG_NAME="${2}"
-  echo "Generating artifacts from configuration(s): ${CONFIG_NAME}"
+  echo "Generating artifacts from configuration (in image): ${CONFIG_NAME}"
   MAKE_ARTIFACTS_OPTIONS="$MAKE_ARTIFACTS_OPTIONS --config $CONFIG_NAME"
 fi
 
 if [ "$#" -gt 2 ]; then
-  OUTPUT_PATH="${3}"
-  MAKE_ARTIFACTS_OPTIONS="$MAKE_ARTIFACTS_OPTIONS --output-path $OUTPUT_PATH"
+  OUTPUT_PATH=$(realpath "${3}")
+else
+  OUTPUT_PATH=$(dirname "${REPORT_PATH}")
 fi
+OUTPUT_FOLDERNAME=$(basename "${OUTPUT_PATH}")
+echo "Writing artifacts to (host machine): ${OUTPUT_PATH}"
+MAKE_ARTIFACTS_OPTIONS="$MAKE_ARTIFACTS_OPTIONS --output-path output/${OUTPUT_FOLDERNAME}"
 
-OUTPUT_DIR="monitoring/uss_qualifier/output"
-mkdir -p "$OUTPUT_DIR"
+mkdir -p "$OUTPUT_PATH"
 
 CACHE_DIR="monitoring/uss_qualifier/.templates_cache"
 mkdir -p "$CACHE_DIR"
@@ -53,7 +64,8 @@ docker run --name uss_qualifier \
   -u "$(id -u):$(id -g)" \
   -e PYTHONBUFFERED=1 \
   -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \
-  -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \
+  -v "${REPORT_PATH}:/input/${REPORT_FILENAME}" \
+  -v "${OUTPUT_PATH}:/app/monitoring/uss_qualifier/output/${OUTPUT_FOLDERNAME}" \
   -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \
   -w /app/monitoring/uss_qualifier \
   interuss/monitoring \

monitoring/uss_qualifier/reports/artifacts.py

@@ -37,7 +37,10 @@ def generate_artifacts(
     disallow_unredacted: bool,
 ):
     logger.debug(f"Writing artifacts to {os.path.abspath(output_path)}")
-    os.makedirs(output_path, exist_ok=True)
+    try:
+        os.makedirs(output_path, exist_ok=True)
+    except PermissionError:
+        pass  # This may be ok if writing directly to a single specific output folder provided to a container
 
     def _should_redact(cfg) -> bool:
         result = "redact_access_tokens" in cfg and cfg.redact_access_tokens

monitoring/uss_qualifier/reports/report.py

@@ -87,6 +87,23 @@ class IntentionalDelay(ImplicitDict):
     """Reason given for this delay"""
 
 
+class _TimestampAccumulator:
+    result: datetime | None
+    _f_accum: Callable[[datetime, datetime], datetime]
+
+    def __init__(self, f_accum: Callable[[datetime, datetime], datetime]):
+        self.result = None
+        self._f_accum = f_accum
+
+    def accumulate(self, new_timestamp: datetime | None) -> None:
+        if new_timestamp is None:
+            return
+        if self.result is None:
+            self.result = new_timestamp
+        else:
+            self.result = self._f_accum(self.result, new_timestamp)
+
+
 class TestStepReport(ImplicitDict):
     name: str
     """Name of this test step"""
@@ -157,6 +174,25 @@ def participant_ids(self) -> set[ParticipantID]:
             ids.update(fc.participants)
         return ids
 
+    @property
+    def latest_timestamp(self) -> datetime | None:
+        timestamp = _TimestampAccumulator(max)
+        if "end_time" in self and self.end_time:
+            timestamp.accumulate(self.end_time.datetime)
+        if "queries" in self and self.queries:
+            for query in self.queries:
+                timestamp.accumulate(query.response.reported.datetime)
+        if "delays" in self and self.delays:
+            for delay in self.delays:
+                timestamp.accumulate(
+                    delay.start_time.datetime + delay.duration.timedelta
+                )
+        for check in self.failed_checks:
+            timestamp.accumulate(check.timestamp.datetime)
+        for check in self.passed_checks:
+            timestamp.accumulate(check.timestamp.datetime)
+        return timestamp.result
+
 
 class TestCaseReport(ImplicitDict):
     name: str
@@ -203,6 +239,16 @@ def participant_ids(self) -> set[ParticipantID]:
             ids.update(step.participant_ids())
         return ids
 
+    @property
+    def latest_timestamp(self) -> datetime | None:
+        timestamp = _TimestampAccumulator(max)
+        if "end_time" in self and self.end_time:
+            timestamp.accumulate(self.end_time.datetime)
+        if "steps" in self and self.steps:
+            for step in self.steps:
+                timestamp.accumulate(step.latest_timestamp)
+        return timestamp.result
+
 
 class ErrorReport(ImplicitDict):
     type: str
@@ -328,6 +374,28 @@ def participant_ids(self) -> set[ParticipantID]:
             ids.update(self.cleanup.participant_ids())
         return ids
 
+    @property
+    def latest_timestamp(self) -> datetime | None:
+        timestamp = _TimestampAccumulator(max)
+        if "end_time" in self and self.end_time:
+            timestamp.accumulate(self.end_time.datetime)
+        if "notes" in self and self.notes:
+            for note in self.notes.values():
+                timestamp.accumulate(note.timestamp.datetime)
+        if "delays" in self and self.delays:
+            for delay in self.delays:
+                timestamp.accumulate(
+                    delay.start_time.datetime + delay.duration.timedelta
+                )
+        if "cases" in self and self.cases:
+            for case in self.cases:
+                timestamp.accumulate(case.latest_timestamp)
+        if "cleanup" in self and self.cleanup:
+            timestamp.accumulate(self.cleanup.latest_timestamp)
+        if "execution_error" in self and self.execution_error:
+            timestamp.accumulate(self.execution_error.timestamp.datetime)
+        return timestamp.result
+
 
 class ActionGeneratorReport(ImplicitDict):
     generator_type: GeneratorTypeName
@@ -380,6 +448,15 @@ def participant_ids(self) -> set[ParticipantID]:
             ids.update(action.participant_ids())
         return ids
 
+    @property
+    def latest_timestamp(self) -> datetime | None:
+        timestamp = _TimestampAccumulator(max)
+        if "end_time" in self and self.end_time:
+            timestamp.accumulate(self.end_time.datetime)
+        for action in self.actions:
+            timestamp.accumulate(action.latest_timestamp)
+        return timestamp.result
+
 
 class TestSuiteActionReport(ImplicitDict):
     test_suite: Optional[TestSuiteReport]
@@ -526,6 +603,10 @@ def end_time(self) -> StringBasedDateTime | None:
             lambda report: report.end_time if "end_time" in report else None
         )
 
+    @property
+    def latest_timestamp(self) -> datetime | None:
+        return self._conditional(lambda report: report.latest_time)
+
 
 class AllConditionsEvaluationReport(ImplicitDict):
     """Result of an evaluation of AllConditions determined by whether all the subconditions are satisfied."""

monitoring/uss_qualifier/reports/templates/timing/report.html

@@ -73,6 +73,14 @@
           font-style: italic;
           font-size: 10px;
         }
+        .server_name_container {
+          text-align: center;
+        }
+        .server_name {
+          writing-mode: vertical-rl;
+          transform: rotate(180deg);
+          white-space: nowrap;
+        }
     </style>
 </head>
 <body>
@@ -133,18 +141,21 @@ <h2>Breakdown by scenario type</h2>
             {% for row in scenario_breakdown %}
             <tr>
                 <td>{{ row.scenario }}</td>
-                <td>{{ format_time(row.total_time) }}</td>
-                <td>{{ format_time(row.average_time) }}</td>
-                <td>{{ round(row.query_fraction * 100, 1) }}%
+                <td style="background-color: #{{ color_of(row.total_time.total_seconds() / max_total_seconds_scenario) }};">{{ format_time(row.total_time) }}</td>
+                <td style="background-color: #{{ color_of(row.average_time.total_seconds() / max_average_seconds_scenario) }};">{{ format_time(row.average_time) }}</td>
+                <td style="background-color: #{{ color_of(1 - row.query_fraction) }};">{{ round(row.query_fraction * 100, 1) }}%
                 {% if row.query_fraction > 1 %} (concurrent queries){% endif %}
                 </td>
-                <td>{{ round(row.delay_fraction * 100, 1) }}%</td>
+                <td style="background-color: #{{ color_of(row.delay_fraction) }};">{{ round(row.delay_fraction * 100, 1) }}%</td>
             </tr>
             {% endfor %}
         </table>
     </div>
     <div>
         <h2>Breakdown by query</h2>
+        <div class="minor_note">
+            Note that differing performance between servers may be due to differing queries sent to the servers (and/or conditions at the times of those queries) in addition to, or instead of, fundamental server performance differences.
+        </div>
         <table>
             <tr>
                 <th rowspan="2">Query type</th>
@@ -154,16 +165,24 @@ <h2>Breakdown by query</h2>
             <tr>
                 <th>Overall</th>
                 {% for server in servers %}
-                <th>{{ server }}</th>
+                <th class="server_name_container"><span class="server_name">{{ server }}</span></th>
                 {% endfor %}
             </tr>
             {% for row in query_breakdown %}
             <tr>
                 <td>{{ row.query_type }}</td>
-                <td>{{ format_time(row.total_time) }}</td>
-                <td>{{ format_time(row.average_time) }}</td>
+                <td style="background-color: #{{ color_of(row.total_time.total_seconds() / max_total_seconds_query) }};">{{ format_time(row.total_time) }}</td>
+                <td style="background-color: #{{ color_of(row.average_time.total_seconds() / max_average_seconds_query) }};">{{ format_time(row.average_time) }}</td>
+                {% set max_seconds = row.max_average_server_time().total_seconds() %}
                 {% for server in servers %}
-                <td>{% if server in row.times_per_server %}{{ format_time(sum(row.times_per_server[server]) / len(row.times_per_server[server])) }}{% endif %}</td>
+                    {% if server in row.times_per_server %}
+                        {% set server_time = sum(row.times_per_server[server]) / len(row.times_per_server[server]) %}
+                        <td style="background-color:#{{ color_of(server_time.total_seconds() / max_seconds) }};">
+                            {{ format_time(server_time) }}
+                        </td>
+                    {% else %}
+                        <td></td>
+                    {% endif %}
                 {% endfor %}
             </tr>
             {% endfor %}
@@ -182,8 +201,8 @@ <h2>Breakdown by intentional delay</h2>
             <tr>
                 <td>{{ row.scenario_type }}</td>
                 <td>{{ row.reason }}</td>
-                <td>{{ format_time(row.total_time) }}</td>
-                <td>{{ format_time(row.average_time) }}</td>
+                <td style="background-color: #{{ color_of(row.total_time.total_seconds() / max_total_seconds_delay) }};">{{ format_time(row.total_time) }}</td>
+                <td style="background-color: #{{ color_of(row.average_time.total_seconds() / max_average_seconds_delay) }};">{{ format_time(row.average_time) }}</td>
             </tr>
             {% endfor %}
         </table>