[ui, ci] retain artifacts from test runs including test timing (#24555)

* retain artifacts from test runs including test timing

* Pinning commit hashes for action helpers

* trigger for ui-test run

* Trying to isolate down to a simple upload

* Once more with mkdir

* What if we just wrote our own test reporter tho

* Let the partitioned runs handle placement

* Filter out common token logs, add a summary at the end, and note failures in logtime

* Custom reporter cannot also have an output file, he finds out two days late

* Aggregate summary, duration, and removing failure case

* Conditional test report generation

* Timeouts are errors

* Trying with un-partitioned input json file

* Remove the commented-out lines for main-only runs

* combine-ui-test-results as its own script

.github/workflows/test-ui.yml

@@ -2,14 +2,14 @@ name: test-ui
 on:
   pull_request:
     paths:
-      - 'ui/**'
+      - "ui/**"
   push:
     branches:
       - main
       - release/**
       - test-ui
     paths:
-      - 'ui/**'
+      - "ui/**"
 
 jobs:
   pre-test:
@@ -36,7 +36,6 @@ jobs:
       - pre-test
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
     timeout-minutes: 30
-    continue-on-error: true
     defaults:
       run:
         working-directory: ui
@@ -44,6 +43,8 @@ jobs:
       matrix:
         partition: [1, 2, 3, 4]
         split: [4]
+        # Note: If we ever change the number of partitions, we'll need to update the
+        # finalize.combine step to match
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-js
@@ -63,8 +64,19 @@ jobs:
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
-        run: yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }}
-
+        run: |
+          yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
+        continue-on-error: true
+      - name: Express timeout failure
+        if: ${{ failure() }}
+        run: exit 1
+      - name: Upload partition test results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: test-results-${{ matrix.partition }}
+          path: ui/test-results/test-results.json
+          retention-days: 90
   finalize:
     needs:
       - pre-test
@@ -88,6 +100,24 @@ jobs:
           jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
+      - name: Download all test results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          pattern: test-results-*
+          path: test-results
+
+      - name: Combine test results for comparison
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: node ../scripts/combine-ui-test-results.js
+      - name: Upload combined results for comparison
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: test-results-${{ github.sha }}
+          path: ui/combined-test-results.json
+          retention-days: 90
+
       - name: finalize
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}

scripts/combine-ui-test-results.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+'use strict';
+const fs = require('fs');
+
+const NUM_PARTITIONS = 4;
+
+function combineResults() {
+  const results = [];
+  let duration = 0;
+  let aggregateSummary = { total: 0, passed: 0, failed: 0 };
+
+  for (let i = 1; i <= NUM_PARTITIONS; i++) {
+    try {
+      const data = JSON.parse(
+        fs.readFileSync(`../test-results/test-results-${i}/test-results.json`).toString()
+      );
+      results.push(...data.tests);
+      duration += data.duration;
+      aggregateSummary.total += data.summary.total;
+      aggregateSummary.passed += data.summary.passed;
+      aggregateSummary.failed += data.summary.failed;
+    } catch (err) {
+      console.error(`Error reading partition ${i}:`, err);
+    }
+  }
+
+  const output = {
+    timestamp: new Date().toISOString(),
+    sha: process.env.GITHUB_SHA,
+    summary: {
+      total: aggregateSummary.total,
+      passed: aggregateSummary.passed,
+      failed: aggregateSummary.failed
+    },
+    duration,
+    tests: results
+  };
+
+  fs.writeFileSync('../ui/combined-test-results.json', JSON.stringify(output, null, 2));
+}
+
+if (require.main === module) {
+  combineResults();
+}
+
+module.exports = combineResults;

ui/app/index.html

@@ -24,6 +24,7 @@
     {{content-for "body"}}
 
     <script src="{{rootURL}}assets/vendor.js"></script>
+
     <script src="{{rootURL}}assets/nomad-ui.js"></script>
 
     {{content-for "body-footer"}}

ui/test-reporter.js

@@ -0,0 +1,159 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+/* eslint-env node */
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const path = require('path');
+
+class JsonReporter {
+  constructor(out, socket, config) {
+    this.out = out || process.stdout;
+    this.results = [];
+
+    // Get output file from Testem config, which is set by the --json-report=path argument
+    this.outputFile = config?.fileOptions?.custom_report_file;
+    this.generateReport = !!this.outputFile;
+
+    if (this.generateReport) {
+      console.log(
+        `[Reporter] Initializing with output file: ${this.outputFile}`
+      );
+
+      try {
+        fs.mkdirSync(path.dirname(this.outputFile), { recursive: true });
+
+        // Initialize the results file
+        fs.writeFileSync(
+          this.outputFile,
+          JSON.stringify(
+            {
+              summary: { total: 0, passed: 0, failed: 0 },
+              timestamp: new Date().toISOString(),
+              tests: [],
+            },
+            null,
+            2
+          )
+        );
+        console.log('[Reporter] Initialized results file');
+      } catch (err) {
+        console.error('[Reporter] Error initializing results file:', err);
+      }
+    } else {
+      console.log('[Reporter] No report file configured, skipping JSON output');
+    }
+
+    process.on('SIGINT', () => {
+      console.log('[Reporter] Received SIGINT, finishing up...');
+      this.finish();
+      process.exit(0);
+    });
+
+    this.testCounter = 0;
+    this.startTime = Date.now();
+  }
+
+  filterLogs(logs) {
+    return logs.filter((log) => {
+      // Filter out token-related logs
+      if (
+        log.text &&
+        (log.text.includes('Accessor:') ||
+          log.text.includes('log in with a JWT') ||
+          log.text === 'TOKENS:' ||
+          log.text === '=====================================')
+      ) {
+        return false;
+      }
+
+      // Keep non-warning logs that aren't token-related
+      return log.type !== 'warn';
+    });
+  }
+
+  report(prefix, data) {
+    if (!data || !data.name) {
+      console.log(`[Reporter] Skipping invalid test result: ${data.name}`);
+      return;
+    }
+
+    this.testCounter++;
+    console.log(`[Reporter] Test #${this.testCounter}: ${data.name}`);
+
+    const partitionMatch = data.name.match(/^Exam Partition (\d+) - (.*)/);
+
+    const result = {
+      name: partitionMatch ? partitionMatch[2] : data.name.trim(),
+      partition: partitionMatch ? parseInt(partitionMatch[1], 10) : null,
+      browser: prefix,
+      passed: !data.failed,
+      duration: data.runDuration,
+      error: data.failed ? data.error : null,
+      logs: this.filterLogs(data.logs || []),
+    };
+
+    if (result.passed) {
+      console.log('- [PASS]');
+    } else {
+      console.log('- [FAIL]');
+      console.log('- Error:', result.error);
+      console.log('- Logs:', result.logs);
+    }
+
+    this.results.push(result);
+  }
+
+  writeCurrentResults() {
+    console.log('[Reporter] Writing current results...');
+    try {
+      const passed = this.results.filter((r) => r.passed).length;
+      const failed = this.results.filter((r) => !r.passed).length;
+      const total = this.results.length;
+      const duration = Date.now() - this.startTime;
+
+      const output = {
+        summary: { total, passed, failed },
+        timestamp: new Date().toISOString(),
+        duration,
+        tests: this.results,
+      };
+
+      if (this.generateReport) {
+        fs.writeFileSync(this.outputFile, JSON.stringify(output, null, 2));
+      }
+
+      // Print a summary
+      console.log('\n[Reporter] Test Summary:');
+      console.log(`- Total:  ${total}`);
+      console.log(`- Passed: ${passed}`);
+      console.log(`- Failed: ${failed}`);
+      console.log(`- Duration: ${duration}ms`);
+      if (failed > 0) {
+        console.log('\n[Reporter] Failed Tests:');
+        this.results
+          .filter((r) => !r.passed)
+          .forEach((r) => {
+            console.log(`❌ ${r.name}`);
+            if (r.error) {
+              console.error(r.error);
+            }
+          });
+      }
+
+      console.log('[Reporter] Successfully wrote results');
+    } catch (err) {
+      console.error('[Reporter] Error writing results:', err);
+    }
+  }
+  finish() {
+    console.log('[Reporter] Finishing up...');
+    this.writeCurrentResults();
+    console.log('[Reporter] Done.');
+  }
+}
+
+module.exports = JsonReporter;

ui/testem.js

@@ -3,7 +3,24 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
+// @ts-check
+
 'use strict';
+const JsonReporter = require('./test-reporter');
+
+/**
+ * Get the path for the test results file based on the command line arguments
+ * @returns {string} The path to the test results file
+ */
+const getReportPath = () => {
+  const jsonReportArg = process.argv.find((arg) =>
+    arg.startsWith('--json-report=')
+  );
+  if (jsonReportArg) {
+    return jsonReportArg.split('=')[1];
+  }
+  return null;
+};
 
 const config = {
   test_page: 'tests/index.html?hidepassed',
@@ -13,6 +30,12 @@ const config = {
   browser_start_timeout: 120,
   parallel: -1,
   framework: 'qunit',
+  reporter: JsonReporter,
+  custom_report_file: getReportPath(),
+  // NOTE: we output this property as custom_report_file instead of report_file.
+  // See https://github.com/testem/testem/issues/1073, report_file + custom reporter results in double output.
+  debug: true,
+
   browser_args: {
     // New format in testem/master, but not in a release yet
     // Chrome: {