From e3ac00f30eec3e114dddb3abd41d1f9ae493aba0 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 18 Dec 2024 08:07:44 +0000 Subject: [PATCH] github: notify Slack when CI fails on merge to main/release (#24690) This change creates a reusable workflow for notifying Slack on CI failures. The message will include useful links and information about the failure, so product engineers can investigate and fix any problems. The new workflow is used by selected workflows which trigger on merges to main or release/* branches. The notification is only sent on failure and when the event was a push (PR merge) meaning the number of notifications should be minimal. The aim is to help identify and draw attention to failure across our release branches, in particular when automated processes happen. --- .github/workflows/build.yml | 19 +++++ .github/workflows/security-scan.yml | 15 ++++ .github/workflows/test-core.yaml | 16 ++++ .github/workflows/test-e2e.yml | 13 ++++ .../workflows/test-failure-notification.yml | 75 +++++++++++++++++++ .github/workflows/test-ui.yml | 15 ++++ .github/workflows/test-windows.yml | 16 ++++ 7 files changed, 169 insertions(+) create mode 100644 .github/workflows/test-failure-notification.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc64826b8..86eb0f2e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -377,6 +377,25 @@ jobs: echo "::group::Smoke test binary" docker run --rm -v "$PWD:/src" registry.access.redhat.com/ubi7/ubi-minimal:7.9-1057 /src/nomad version + handle-failure: + needs: + - get-go-version + - get-product-version + - generate-metadata-file + - build-other + - build-linux + - build-darwin + - build-docker + - minimum-os + if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure') + uses: ./.github/workflows/test-failure-notification.yml + secrets: inherit + with: + actor: ${{ github.triggering_actor }} + git-branch: ${{ github.ref_name }} + workflow-run-id: ${{ github.run_id }} + workflow-name: ${{ github.workflow }} + permissions: contents: read id-token: write diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index a458be15f..7f0027356 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -69,3 +69,18 @@ jobs: uses: github/codeql-action/upload-sarif@8fd294e26a0e458834582b0fe4988d79966c7c0a # codeql-bundle-v2.18.4 with: sarif_file: results.sarif + + handle-failure: + permissions: + contents: read + id-token: write + needs: + - scan + if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure') + uses: ./.github/workflows/test-failure-notification.yml + secrets: inherit + with: + actor: ${{ github.triggering_actor }} + git-branch: ${{ github.ref_name }} + workflow-run-id: ${{ github.run_id }} + workflow-name: ${{ github.workflow }} diff --git a/.github/workflows/test-core.yaml b/.github/workflows/test-core.yaml index 9b006697d..eccdbc49b 100644 --- a/.github/workflows/test-core.yaml +++ b/.github/workflows/test-core.yaml @@ -121,6 +121,22 @@ jobs: sudo -E env "PATH=$PATH" hc-install install -version ${{env.CONSUL_VERSION}} -path '${{env.GOPATH}}/bin' consul sudo sed -i 's!Defaults!#Defaults!g' /etc/sudoers sudo -E env "PATH=$PATH" make test-nomad + + handle-failure: + needs: + - checks + - compile + - tests-api + - tests-groups + if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure') + uses: ./.github/workflows/test-failure-notification.yml + secrets: inherit + with: + actor: ${{ github.triggering_actor }} + git-branch: ${{ github.ref_name }} + workflow-run-id: ${{ github.run_id }} + workflow-name: ${{ github.workflow }} + permissions: contents: read id-token: write diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 10a5dbc19..4751227b2 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -82,6 +82,19 @@ jobs: sudo sed -i 's!Defaults!#Defaults!g' /etc/sudoers sudo -E env "PATH=$PATH" make integration-test-consul + handle-failure: + needs: + - test-e2e-vault + - test-e2e-consul + if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure') + uses: ./.github/workflows/test-failure-notification.yml + secrets: inherit + with: + actor: ${{ github.triggering_actor }} + git-branch: ${{ github.ref_name }} + workflow-run-id: ${{ github.run_id }} + workflow-name: ${{ github.workflow }} + permissions: contents: read id-token: write diff --git a/.github/workflows/test-failure-notification.yml b/.github/workflows/test-failure-notification.yml new file mode 100644 index 000000000..086480656 --- /dev/null +++ b/.github/workflows/test-failure-notification.yml @@ -0,0 +1,75 @@ +--- +name: CI Test Failure Notification + +on: + workflow_call: + inputs: + actor: + description: 'Triggering actor of the failed the workflow' + required: true + type: string + git-branch: + description: 'Git branch name' + required: true + type: string + workflow-name: + description: 'Name of the GitHub Action workflow' + required: true + type: string + workflow-run-id: + description: 'GitHub Action run ID that failed' + required: true + type: string + +jobs: + send-notification: + runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux"]') || 'ubuntu-latest' }} + steps: + - name: Retrieve Vault-hosted Secrets + if: endsWith(github.repository, '-enterprise') + id: vault + uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 + with: + url: ${{ vars.CI_VAULT_URL }} + method: ${{ vars.CI_VAULT_METHOD }} + path: ${{ vars.CI_VAULT_PATH }} + jwtGithubAudience: ${{ vars.CI_VAULT_AUD }} + secrets: |- + kv/data/teams/nomad/slack-webhooks feed-nomad | SLACK_FEED_NOMAD ; + - name: Send Slack notification + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + with: + webhook: "${{ env.SLACK_FEED_NOMAD || secrets.SLACK_FEED_NOMAD_CI_FAILURE }}" + webhook-type: incoming-webhook + payload: | + { + "text": ":x: CI Workflow '${{ inputs.workflow-name }}' has failed", + "attachments": [ + { + "color": "#C41E3A", + "blocks": [ + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Branch:* <${{ github.server_url }}/${{ github.repository }}/tree/${{ inputs.git-branch }}|${{ inputs.git-branch }}>" + }, + { + "type": "mrkdwn", + "text": "*From:* @${{ inputs.actor }}" + }, + { + "type": "mrkdwn", + "text": "*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ inputs.workflow-run-id }}|${{ inputs.workflow-run-id }}>" + } + ] + } + ] + } + ] + } + +permissions: + contents: read + id-token: write \ No newline at end of file diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml index 547846087..69b941d7b 100644 --- a/.github/workflows/test-ui.yml +++ b/.github/workflows/test-ui.yml @@ -123,6 +123,21 @@ jobs: PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }} PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }} run: yarn percy build:finalize + + handle-failure: + needs: + - pre-test + - tests + - finalize + if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure') + uses: ./.github/workflows/test-failure-notification.yml + secrets: inherit + with: + actor: ${{ github.triggering_actor }} + git-branch: ${{ github.ref_name }} + workflow-run-id: ${{ github.run_id }} + workflow-name: ${{ github.workflow }} + permissions: contents: read id-token: write diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 39dc4d930..e4567c7f3 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -98,5 +98,21 @@ jobs: with: name: results.xml path: results.xml + + handle-failure: + permissions: + contents: read + id-token: write + needs: + - test-windows + if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure') + uses: ./.github/workflows/test-failure-notification.yml + secrets: inherit + with: + actor: ${{ github.triggering_actor }} + git-branch: ${{ github.ref_name }} + workflow-run-id: ${{ github.run_id }} + workflow-name: ${{ github.workflow }} + permissions: contents: read