[ui, ci] retain artifacts from test runs including test timing (#24555)

* retain artifacts from test runs including test timing

* Pinning commit hashes for action helpers

* trigger for ui-test run

* Trying to isolate down to a simple upload

* Once more with mkdir

* What if we just wrote our own test reporter tho

* Let the partitioned runs handle placement

* Filter out common token logs, add a summary at the end, and note failures in logtime

* Custom reporter cannot also have an output file, he finds out two days late

* Aggregate summary, duration, and removing failure case

* Conditional test report generation

* Timeouts are errors

* Trying with un-partitioned input json file

* Remove the commented-out lines for main-only runs

* combine-ui-test-results as its own script

ui/app/index.html

@@ -24,6 +24,7 @@
     {{content-for "body"}}
 
     <script src="{{rootURL}}assets/vendor.js"></script>
+
     <script src="{{rootURL}}assets/nomad-ui.js"></script>
 
     {{content-for "body-footer"}}

ui/test-reporter.js

@@ -0,0 +1,159 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+/* eslint-env node */
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const path = require('path');
+
+class JsonReporter {
+  constructor(out, socket, config) {
+    this.out = out || process.stdout;
+    this.results = [];
+
+    // Get output file from Testem config, which is set by the --json-report=path argument
+    this.outputFile = config?.fileOptions?.custom_report_file;
+    this.generateReport = !!this.outputFile;
+
+    if (this.generateReport) {
+      console.log(
+        `[Reporter] Initializing with output file: ${this.outputFile}`
+      );
+
+      try {
+        fs.mkdirSync(path.dirname(this.outputFile), { recursive: true });
+
+        // Initialize the results file
+        fs.writeFileSync(
+          this.outputFile,
+          JSON.stringify(
+            {
+              summary: { total: 0, passed: 0, failed: 0 },
+              timestamp: new Date().toISOString(),
+              tests: [],
+            },
+            null,
+            2
+          )
+        );
+        console.log('[Reporter] Initialized results file');
+      } catch (err) {
+        console.error('[Reporter] Error initializing results file:', err);
+      }
+    } else {
+      console.log('[Reporter] No report file configured, skipping JSON output');
+    }
+
+    process.on('SIGINT', () => {
+      console.log('[Reporter] Received SIGINT, finishing up...');
+      this.finish();
+      process.exit(0);
+    });
+
+    this.testCounter = 0;
+    this.startTime = Date.now();
+  }
+
+  filterLogs(logs) {
+    return logs.filter((log) => {
+      // Filter out token-related logs
+      if (
+        log.text &&
+        (log.text.includes('Accessor:') ||
+          log.text.includes('log in with a JWT') ||
+          log.text === 'TOKENS:' ||
+          log.text === '=====================================')
+      ) {
+        return false;
+      }
+
+      // Keep non-warning logs that aren't token-related
+      return log.type !== 'warn';
+    });
+  }
+
+  report(prefix, data) {
+    if (!data || !data.name) {
+      console.log(`[Reporter] Skipping invalid test result: ${data.name}`);
+      return;
+    }
+
+    this.testCounter++;
+    console.log(`[Reporter] Test #${this.testCounter}: ${data.name}`);
+
+    const partitionMatch = data.name.match(/^Exam Partition (\d+) - (.*)/);
+
+    const result = {
+      name: partitionMatch ? partitionMatch[2] : data.name.trim(),
+      partition: partitionMatch ? parseInt(partitionMatch[1], 10) : null,
+      browser: prefix,
+      passed: !data.failed,
+      duration: data.runDuration,
+      error: data.failed ? data.error : null,
+      logs: this.filterLogs(data.logs || []),
+    };
+
+    if (result.passed) {
+      console.log('- [PASS]');
+    } else {
+      console.log('- [FAIL]');
+      console.log('- Error:', result.error);
+      console.log('- Logs:', result.logs);
+    }
+
+    this.results.push(result);
+  }
+
+  writeCurrentResults() {
+    console.log('[Reporter] Writing current results...');
+    try {
+      const passed = this.results.filter((r) => r.passed).length;
+      const failed = this.results.filter((r) => !r.passed).length;
+      const total = this.results.length;
+      const duration = Date.now() - this.startTime;
+
+      const output = {
+        summary: { total, passed, failed },
+        timestamp: new Date().toISOString(),
+        duration,
+        tests: this.results,
+      };
+
+      if (this.generateReport) {
+        fs.writeFileSync(this.outputFile, JSON.stringify(output, null, 2));
+      }
+
+      // Print a summary
+      console.log('\n[Reporter] Test Summary:');
+      console.log(`- Total:  ${total}`);
+      console.log(`- Passed: ${passed}`);
+      console.log(`- Failed: ${failed}`);
+      console.log(`- Duration: ${duration}ms`);
+      if (failed > 0) {
+        console.log('\n[Reporter] Failed Tests:');
+        this.results
+          .filter((r) => !r.passed)
+          .forEach((r) => {
+            console.log(`❌ ${r.name}`);
+            if (r.error) {
+              console.error(r.error);
+            }
+          });
+      }
+
+      console.log('[Reporter] Successfully wrote results');
+    } catch (err) {
+      console.error('[Reporter] Error writing results:', err);
+    }
+  }
+  finish() {
+    console.log('[Reporter] Finishing up...');
+    this.writeCurrentResults();
+    console.log('[Reporter] Done.');
+  }
+}
+
+module.exports = JsonReporter;

ui/testem.js

@@ -3,7 +3,24 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
+// @ts-check
+
 'use strict';
+const JsonReporter = require('./test-reporter');
+
+/**
+ * Get the path for the test results file based on the command line arguments
+ * @returns {string} The path to the test results file
+ */
+const getReportPath = () => {
+  const jsonReportArg = process.argv.find((arg) =>
+    arg.startsWith('--json-report=')
+  );
+  if (jsonReportArg) {
+    return jsonReportArg.split('=')[1];
+  }
+  return null;
+};
 
 const config = {
   test_page: 'tests/index.html?hidepassed',
@@ -13,6 +30,12 @@ const config = {
   browser_start_timeout: 120,
   parallel: -1,
   framework: 'qunit',
+  reporter: JsonReporter,
+  custom_report_file: getReportPath(),
+  // NOTE: we output this property as custom_report_file instead of report_file.
+  // See https://github.com/testem/testem/issues/1073, report_file + custom reporter results in double output.
+  debug: true,
+
   browser_args: {
     // New format in testem/master, but not in a release yet
     // Chrome: {