diff --git a/github_pages/static/index.md b/github_pages/static/index.md index fed202461c..2b7fdd0224 100644 --- a/github_pages/static/index.md +++ b/github_pages/static/index.md @@ -10,6 +10,7 @@ These reports were generated during continuous integration for the most recent P * [Sequence view](./artifacts/uss_qualifier/reports/uspace/sequence) * [Tested requirements](./artifacts/uss_qualifier/reports/uspace/requirements) +* [Timing report](./artifacts/uss_qualifier/reports/uspace/timing) * [Demonstrated capabilities](./artifacts/uss_qualifier/reports/uspace/capabilities.html) * [Raw report](./artifacts/uss_qualifier/reports/uspace/report.json) (large) @@ -18,27 +19,32 @@ These reports were generated during continuous integration for the most recent P * [Raw report](./artifacts/uss_qualifier/reports/noop/report.json) (indented to be human-readable) * [Interactive report](./artifacts/uss_qualifier/reports/noop/report.html) * [Sequence view](./artifacts/uss_qualifier/reports/noop/sequence) +* [Timing report](./artifacts/uss_qualifier/reports/noop/timing) ### [ASTM F3548-21 test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml) * [Sequence view](./artifacts/uss_qualifier/reports/f3548_self_contained/sequence) * [Tested requirements](./artifacts/uss_qualifier/reports/f3548_self_contained/gate3) * [Globally-expanded report](./artifacts/uss_qualifier/reports/f3548_self_contained/globally_expanded/report.html) +* [Timing report](./artifacts/uss_qualifier/reports/f3548_self_contained/timing) ### [US UTM Implementation test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/utm_implementation_us/environments/local/test_1.jsonnet) * [Sequence view](./artifacts/uss_qualifier/reports/test_1/sequence) * [Tested requirements](./artifacts/uss_qualifier/reports/test_1/scd) +* [Timing report](./artifacts/uss_qualifier/reports/test_1/timing) ### [ASTM F3411-22a test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml) * [Sequence view](./artifacts/uss_qualifier/reports/netrid_v22a/sequence) * [Tested requirements](./artifacts/uss_qualifier/reports/netrid_v22a/requirements) +* [Timing report](./artifacts/uss_qualifier/reports/netrid_v22a/timing) ### [DSS integration test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml) * [Sequence view](./artifacts/uss_qualifier/reports/dss_probing/sequence) * [Tested requirements](./artifacts/uss_qualifier/reports/dss_probing/requirements) +* [Timing report](./artifacts/uss_qualifier/reports/dss_probing/timing) ### [General flight authorization configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml) diff --git a/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml b/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml index 1f92bc6319..8bf309e2fb 100644 --- a/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml @@ -57,6 +57,7 @@ v1: participant_requirements: uss1: null uss2: all_astm_dss_requirements + timing_report: { } validation: criteria: - $ref: ./library/validation.yaml#/execution_error_none diff --git a/monitoring/uss_qualifier/make_artifacts.sh b/monitoring/uss_qualifier/make_artifacts.sh index 1e8adcff16..8acac43388 100755 --- a/monitoring/uss_qualifier/make_artifacts.sh +++ b/monitoring/uss_qualifier/make_artifacts.sh @@ -3,11 +3,14 @@ set -eo pipefail if [[ $# -lt 1 ]]; then - echo "Usage: $0 [ []]" + echo "Usage: $0 [ []]" echo "Generates artifacts according to the specified configuration(s) using the specified report(s)" - echo ": Location of the report file (or multiple locations separated by commas). Relative paths are relative to this folder. Use file:// prefix to explicitly specify file-based location." - echo ": Location of the configuration file (or multiple locations separated by commas)." - echo ": Location to which artifacts should be written (defaults to output/)" + echo " : Location (on the host machine) of the report file. Relative paths are RELATIVE TO THE REPO ROOT." + echo " : Location of the configuration file describing what artifacts to make. Must be built into the monitoring image (in configurations/personal, for instance). If not specified, use artifacts configuration from report." + echo " : Location (on the host machine) to which artifacts should be written. Defaults to folder containing the report." + echo "Examples:" + echo " ./monitoring/uss_qualifier/make_artifacts.sh ~/Downloads/be0bbe7c-4670-43f5-906a-594be69087f4/report.json configurations.dev.f3548_self_contained" + echo " ./make_artifacts.sh monitoring/uss_qualifier/output/f3548_self_contained/report.json configurations.personal.custom_artifacts" exit 1 fi @@ -26,23 +29,31 @@ cd monitoring || exit 1 make image ) -REPORT_NAME="${1}" -echo "Reading report(s) from: ${REPORT_NAME}" -MAKE_ARTIFACTS_OPTIONS="--report $REPORT_NAME" +REPORT_LOCATION="${1}" + +# TODO: Retrieve local copy of report if location starts with "http" + +REPORT_PATH=$(realpath "${REPORT_LOCATION}") +echo "Reading report from (host machine): ${REPORT_PATH}" +REPORT_FILENAME=$(basename "${REPORT_PATH}") +MAKE_ARTIFACTS_OPTIONS="--report file:///input/${REPORT_FILENAME}" if [ "$#" -gt 1 ]; then CONFIG_NAME="${2}" - echo "Generating artifacts from configuration(s): ${CONFIG_NAME}" + echo "Generating artifacts from configuration (in image): ${CONFIG_NAME}" MAKE_ARTIFACTS_OPTIONS="$MAKE_ARTIFACTS_OPTIONS --config $CONFIG_NAME" fi if [ "$#" -gt 2 ]; then - OUTPUT_PATH="${3}" - MAKE_ARTIFACTS_OPTIONS="$MAKE_ARTIFACTS_OPTIONS --output-path $OUTPUT_PATH" + OUTPUT_PATH=$(realpath "${3}") +else + OUTPUT_PATH=$(dirname "${REPORT_PATH}") fi +OUTPUT_FOLDERNAME=$(basename "${OUTPUT_PATH}") +echo "Writing artifacts to (host machine): ${OUTPUT_PATH}" +MAKE_ARTIFACTS_OPTIONS="$MAKE_ARTIFACTS_OPTIONS --output-path output/${OUTPUT_FOLDERNAME}" -OUTPUT_DIR="monitoring/uss_qualifier/output" -mkdir -p "$OUTPUT_DIR" +mkdir -p "$OUTPUT_PATH" CACHE_DIR="monitoring/uss_qualifier/.templates_cache" mkdir -p "$CACHE_DIR" @@ -53,7 +64,8 @@ docker run --name uss_qualifier \ -u "$(id -u):$(id -g)" \ -e PYTHONBUFFERED=1 \ -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \ - -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \ + -v "${REPORT_PATH}:/input/${REPORT_FILENAME}" \ + -v "${OUTPUT_PATH}:/app/monitoring/uss_qualifier/output/${OUTPUT_FOLDERNAME}" \ -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \ -w /app/monitoring/uss_qualifier \ interuss/monitoring \ diff --git a/monitoring/uss_qualifier/reports/artifacts.py b/monitoring/uss_qualifier/reports/artifacts.py index 2ec683a887..6b97f5c140 100644 --- a/monitoring/uss_qualifier/reports/artifacts.py +++ b/monitoring/uss_qualifier/reports/artifacts.py @@ -37,7 +37,10 @@ def generate_artifacts( disallow_unredacted: bool, ): logger.debug(f"Writing artifacts to {os.path.abspath(output_path)}") - os.makedirs(output_path, exist_ok=True) + try: + os.makedirs(output_path, exist_ok=True) + except PermissionError: + pass # This may be ok if writing directly to a single specific output folder provided to a container def _should_redact(cfg) -> bool: result = "redact_access_tokens" in cfg and cfg.redact_access_tokens diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index 007953563b..e377221515 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -87,6 +87,23 @@ class IntentionalDelay(ImplicitDict): """Reason given for this delay""" +class _TimestampAccumulator: + result: datetime | None + _f_accum: Callable[[datetime, datetime], datetime] + + def __init__(self, f_accum: Callable[[datetime, datetime], datetime]): + self.result = None + self._f_accum = f_accum + + def accumulate(self, new_timestamp: datetime | None) -> None: + if new_timestamp is None: + return + if self.result is None: + self.result = new_timestamp + else: + self.result = self._f_accum(self.result, new_timestamp) + + class TestStepReport(ImplicitDict): name: str """Name of this test step""" @@ -157,6 +174,25 @@ def participant_ids(self) -> set[ParticipantID]: ids.update(fc.participants) return ids + @property + def latest_timestamp(self) -> datetime | None: + timestamp = _TimestampAccumulator(max) + if "end_time" in self and self.end_time: + timestamp.accumulate(self.end_time.datetime) + if "queries" in self and self.queries: + for query in self.queries: + timestamp.accumulate(query.response.reported.datetime) + if "delays" in self and self.delays: + for delay in self.delays: + timestamp.accumulate( + delay.start_time.datetime + delay.duration.timedelta + ) + for check in self.failed_checks: + timestamp.accumulate(check.timestamp.datetime) + for check in self.passed_checks: + timestamp.accumulate(check.timestamp.datetime) + return timestamp.result + class TestCaseReport(ImplicitDict): name: str @@ -203,6 +239,16 @@ def participant_ids(self) -> set[ParticipantID]: ids.update(step.participant_ids()) return ids + @property + def latest_timestamp(self) -> datetime | None: + timestamp = _TimestampAccumulator(max) + if "end_time" in self and self.end_time: + timestamp.accumulate(self.end_time.datetime) + if "steps" in self and self.steps: + for step in self.steps: + timestamp.accumulate(step.latest_timestamp) + return timestamp.result + class ErrorReport(ImplicitDict): type: str @@ -328,6 +374,28 @@ def participant_ids(self) -> set[ParticipantID]: ids.update(self.cleanup.participant_ids()) return ids + @property + def latest_timestamp(self) -> datetime | None: + timestamp = _TimestampAccumulator(max) + if "end_time" in self and self.end_time: + timestamp.accumulate(self.end_time.datetime) + if "notes" in self and self.notes: + for note in self.notes.values(): + timestamp.accumulate(note.timestamp.datetime) + if "delays" in self and self.delays: + for delay in self.delays: + timestamp.accumulate( + delay.start_time.datetime + delay.duration.timedelta + ) + if "cases" in self and self.cases: + for case in self.cases: + timestamp.accumulate(case.latest_timestamp) + if "cleanup" in self and self.cleanup: + timestamp.accumulate(self.cleanup.latest_timestamp) + if "execution_error" in self and self.execution_error: + timestamp.accumulate(self.execution_error.timestamp.datetime) + return timestamp.result + class ActionGeneratorReport(ImplicitDict): generator_type: GeneratorTypeName @@ -380,6 +448,15 @@ def participant_ids(self) -> set[ParticipantID]: ids.update(action.participant_ids()) return ids + @property + def latest_timestamp(self) -> datetime | None: + timestamp = _TimestampAccumulator(max) + if "end_time" in self and self.end_time: + timestamp.accumulate(self.end_time.datetime) + for action in self.actions: + timestamp.accumulate(action.latest_timestamp) + return timestamp.result + class TestSuiteActionReport(ImplicitDict): test_suite: Optional[TestSuiteReport] @@ -526,6 +603,10 @@ def end_time(self) -> StringBasedDateTime | None: lambda report: report.end_time if "end_time" in report else None ) + @property + def latest_timestamp(self) -> datetime | None: + return self._conditional(lambda report: report.latest_time) + class AllConditionsEvaluationReport(ImplicitDict): """Result of an evaluation of AllConditions determined by whether all the subconditions are satisfied.""" diff --git a/monitoring/uss_qualifier/reports/templates/timing/report.html b/monitoring/uss_qualifier/reports/templates/timing/report.html index d2f927d2ab..22d926c375 100644 --- a/monitoring/uss_qualifier/reports/templates/timing/report.html +++ b/monitoring/uss_qualifier/reports/templates/timing/report.html @@ -73,6 +73,14 @@ font-style: italic; font-size: 10px; } + .server_name_container { + text-align: center; + } + .server_name { + writing-mode: vertical-rl; + transform: rotate(180deg); + white-space: nowrap; + } @@ -133,18 +141,21 @@

Breakdown by scenario type

{% for row in scenario_breakdown %} {{ row.scenario }} - {{ format_time(row.total_time) }} - {{ format_time(row.average_time) }} - {{ round(row.query_fraction * 100, 1) }}% + {{ format_time(row.total_time) }} + {{ format_time(row.average_time) }} + {{ round(row.query_fraction * 100, 1) }}% {% if row.query_fraction > 1 %} (concurrent queries){% endif %} - {{ round(row.delay_fraction * 100, 1) }}% + {{ round(row.delay_fraction * 100, 1) }}% {% endfor %}

Breakdown by query

+
+ Note that differing performance between servers may be due to differing queries sent to the servers (and/or conditions at the times of those queries) in addition to, or instead of, fundamental server performance differences. +
@@ -154,16 +165,24 @@

Breakdown by query

{% for server in servers %} - + {% endfor %} {% for row in query_breakdown %} - - + + + {% set max_seconds = row.max_average_server_time().total_seconds() %} {% for server in servers %} - + {% if server in row.times_per_server %} + {% set server_time = sum(row.times_per_server[server]) / len(row.times_per_server[server]) %} + + {% else %} + + {% endif %} {% endfor %} {% endfor %} @@ -182,8 +201,8 @@

Breakdown by intentional delay

- - + + {% endfor %}
Query type
Overall{{ server }}{{ server }}
{{ row.query_type }}{{ format_time(row.total_time) }}{{ format_time(row.average_time) }}{{ format_time(row.total_time) }}{{ format_time(row.average_time) }}{% if server in row.times_per_server %}{{ format_time(sum(row.times_per_server[server]) / len(row.times_per_server[server])) }}{% endif %} + {{ format_time(server_time) }} +
{{ row.scenario_type }} {{ row.reason }}{{ format_time(row.total_time) }}{{ format_time(row.average_time) }}{{ format_time(row.total_time) }}{{ format_time(row.average_time) }}
diff --git a/monitoring/uss_qualifier/reports/timing/generate.py b/monitoring/uss_qualifier/reports/timing/generate.py index 683aa618f5..712419a29c 100644 --- a/monitoring/uss_qualifier/reports/timing/generate.py +++ b/monitoring/uss_qualifier/reports/timing/generate.py @@ -1,4 +1,5 @@ import os +from colorsys import hsv_to_rgb from dataclasses import dataclass from datetime import timedelta @@ -36,9 +37,17 @@ def generate_timing_report( try: if report.report.start_time is None: raise _GenerationError("start_time is missing") - if report.report.end_time is None: + if report.report.end_time: + duration = ( + report.report.end_time.datetime - report.report.start_time.datetime + ) + elif report.report.latest_timestamp: + duration = ( + report.report.latest_timestamp - report.report.start_time.datetime + ) + else: raise _GenerationError("end_time is missing") - duration = report.report.end_time.datetime - report.report.start_time.datetime + scenario_summaries, query_summaries, delays_summary = _summarize(report.report) scenario_breakdown = _make_scenario_breakdown( report, scenario_summaries, config @@ -62,13 +71,40 @@ def generate_timing_report( duration=duration, codebase_version=get_code_version(), scenario_breakdown=scenario_breakdown, + max_total_seconds_scenario=max( + r.total_time.total_seconds() for r in scenario_breakdown + ), + max_average_seconds_scenario=max( + r.average_time.total_seconds() for r in scenario_breakdown + ), servers=servers, query_breakdown=query_breakdown, + max_total_seconds_query=max( + r.total_time.total_seconds() for r in query_breakdown + ) + if query_breakdown + else 0, + max_average_seconds_query=max( + r.average_time.total_seconds() for r in query_breakdown + ) + if query_breakdown + else 0, delays_breakdown=delays_breakdown, + max_total_seconds_delay=max( + r.total_time.total_seconds() for r in delays_breakdown + ) + if delays_breakdown + else 0, + max_average_seconds_delay=max( + r.average_time.total_seconds() for r in delays_breakdown + ) + if delays_breakdown + else 0, round=round, len=len, sum=_sum, format_time=_format_time, + color_of=_color_of, ) ) @@ -91,6 +127,12 @@ def _format_time(dt: timedelta) -> str: return f"{seconds:.3f}s" +def _color_of(f: float) -> str: + hue = 0.58 * (1 - min(max(f, 0), 1)) + rgb = hsv_to_rgb(hue, 0.4, 1) + return f"{int(rgb[0] * 255.99):02x}{int(rgb[1] * 255.99):02x}{int(rgb[2] * 255.99):02x}" + + def _sum(items): """This sum function works with a list of timedeltas (or any type defining self-addition), unlike the built-in sum.""" result = None @@ -195,7 +237,11 @@ def _summarize( raise _GenerationError( f"test scenario {scenario.scenario_type} is missing start_time" ) - if "end_time" not in scenario or not scenario.end_time: + if "end_time" in scenario and scenario.end_time: + duration = scenario.end_time.datetime - scenario.start_time.datetime + elif scenario.latest_timestamp: + duration = scenario.latest_timestamp - scenario.start_time.datetime + else: raise _GenerationError( f"test scenario {scenario.scenario_type} started at {scenario.start_time} is missing end_time" ) @@ -254,8 +300,7 @@ def _summarize( { scenario.scenario_type: _ScenarioSummary( instances=1, - total_time=scenario.end_time.datetime - - scenario.start_time.datetime, + total_time=duration, query_time=query_time, delay_time=delay_time, ) @@ -384,6 +429,14 @@ def average_time(self) -> timedelta: n += len(dts) return total / n + def max_average_server_time(self) -> timedelta | None: + if not self.times_per_server: + return None + return max( + (_sum(values) or timedelta(seconds=0)) / len(values) + for values in self.times_per_server.values() + ) + def _make_query_breakdown( summaries: _QuerySummaryCollection, config: TimingReportConfiguration diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/solo_happy_path.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/solo_happy_path.md new file mode 100644 index 0000000000..52bf798414 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/solo_happy_path.md @@ -0,0 +1,7 @@ +# Solo happy path test scenario + +## Description + +This test scenario is deprecated and removed since monitoring v0.26.0. + +## Resources diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/solo_happy_path.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/solo_happy_path.py new file mode 100644 index 0000000000..2e9985c625 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/solo_happy_path.py @@ -0,0 +1,9 @@ +from deprecation import deprecated + +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class SoloHappyPath(TestScenario): + @deprecated(deprecated_in="0.26.0") + def run(self, context): + pass