From 897f389f00950b086e8d24007e2956d9b5361179 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 16 Nov 2026 14:58:44 +0600 Subject: [PATCH 1/5] ci: send smoke Telegram alerts only when the status changes smoke.yml and the post-deploy smoke in deploy.yml now alert once when the smoke test starts failing and once when it recovers, and stay silent while the status is unchanged. The previous result comes from the GitHub API (previous completed run of the same workflow and target), using the job's GITHUB_TOKEN with actions: read; no new secrets or state store. - telegram-alert: new status-change.mjs decides failed/recovered/none; an unknown previous result alerts on failure and stays silent on a pass. - smoke.yml: run-name carries the environment so runs can be matched. - deploy.yml: smoke compared per branch using the smoke step's conclusion; deploy failures still alert every time. A rollback to a commit without smoke.js now skips the step instead of reporting a pass. - e2e-nightly.yml is unchanged and keeps alerting on every failure. --- .github/actions/telegram-alert/action.yml | 81 ++++++- .../actions/telegram-alert/status-change.mjs | 169 ++++++++++++++ .github/workflows/deploy.yml | 47 +++- .github/workflows/smoke.yml | 18 +- .../tests/unit/tooling/status-change.test.js | 210 ++++++++++++++++++ 5 files changed, 504 insertions(+), 21 deletions(-) create mode 100644 .github/actions/telegram-alert/status-change.mjs create mode 100644 happyview-modules/tests/unit/tooling/status-change.test.js diff --git a/.github/actions/telegram-alert/action.yml b/.github/actions/telegram-alert/action.yml index 28b174d..4f2972b 100644 --- a/.github/actions/telegram-alert/action.yml +++ b/.github/actions/telegram-alert/action.yml @@ -1,7 +1,10 @@ -name: Telegram failure alert +name: Telegram status alert description: > - Sends a Telegram message for a failed run. Skips with a notice when the token or chat ID is - empty, and only warns if sending fails, so it never changes the job result. + Sends a Telegram message for a failed run, and with previous-workflow set only when the result + changed since the previous run: one message when it starts failing, one when it recovers, none + while the status stays the same. Skips with a notice when the token or chat ID is empty, and only + warns if deciding or sending fails, so it never changes the job result. Needs Node.js on the + runner (preinstalled on ubuntu-latest). inputs: bot-token: description: Telegram bot token (pass secrets.TELEGRAM_BOT_TOKEN). @@ -14,21 +17,79 @@ inputs: summary: description: What failed and where, e.g. "smoke test failed (Production, main)". required: true + recovered-summary: + description: > + What recovered and where, e.g. "smoke test recovered (Production, main)". Used when the + result changes from failure to success; a generic text is used when empty. + required: false + default: "" failed-checks-file: description: > Optional path to a file with one line naming the failing checks (written by pnpm smoke / e2e via FAILED_CHECKS_FILE). Ignored when missing. required: false default: "" + result: + description: > + Result being reported: "success" or "failure" (pass job.status or steps..outcome). + Any other value, such as "cancelled" or "skipped", sends nothing. The default suits callers + that run this action only after a failure. + required: false + default: failure + previous-workflow: + description: > + File name of the workflow whose previous completed run to compare with, e.g. smoke.yml. Only + the previous run of the same target counts (see previous-match). When empty, nothing is + compared: a failure always alerts and a success never does. When the previous result cannot + be read or does not exist, the same applies. + required: false + default: "" + previous-match: + description: > + How to recognise a run of the same target: "title" (same run name, for workflows whose + run-name carries the environment) or "branch" (same branch, for workflows where the branch + implies the environment). + required: false + default: title + previous-step: + description: > + Optional name of a step. When set, a previous run's result is that step's conclusion instead + of the run's, and previous runs where the step did not finish are skipped. + required: false + default: "" + github-token: + description: > + Token for reading previous runs (pass github.token); the job needs "actions: read". + required: false + default: "" runs: using: composite steps: + - id: decide + shell: bash + env: + RESULT: ${{ inputs.result }} + PREVIOUS_WORKFLOW: ${{ inputs.previous-workflow }} + PREVIOUS_MATCH: ${{ inputs.previous-match }} + PREVIOUS_STEP: ${{ inputs.previous-step }} + GITHUB_TOKEN: ${{ inputs.github-token }} + run: | + # Writes send=failed|recovered|none to $GITHUB_OUTPUT. If the script itself breaks, fall + # back to the old behaviour (alert on failure) rather than lose a failure alert. + if ! node "$GITHUB_ACTION_PATH/status-change.mjs"; then + echo "::warning::Could not decide whether the status changed; alerting on failures only." + if [ "$RESULT" = failure ]; then echo "send=failed" >> "$GITHUB_OUTPUT"; fi + fi + # The token goes to curl through stdin (-K -), so it never appears in argv or the log. - - shell: bash + - if: steps.decide.outputs.send == 'failed' || steps.decide.outputs.send == 'recovered' + shell: bash env: TELEGRAM_BOT_TOKEN: ${{ inputs.bot-token }} TELEGRAM_CHAT_ID: ${{ inputs.chat-id }} + SEND: ${{ steps.decide.outputs.send }} SUMMARY: ${{ inputs.summary }} + RECOVERED_SUMMARY: ${{ inputs.recovered-summary }} FAILED_CHECKS_FILE: ${{ inputs.failed-checks-file }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | @@ -36,11 +97,15 @@ runs: echo "::notice::Telegram alert skipped: TELEGRAM_BOT_TOKEN or TELEGRAM_CHAT_ID is not set." exit 0 fi - # Short on purpose: what failed, which checks (already capped), and the run link. - text="❌ $SUMMARY" - if [ -n "$FAILED_CHECKS_FILE" ] && [ -s "$FAILED_CHECKS_FILE" ]; then - text="$text + # Short on purpose: what changed, which checks failed (already capped), and the run link. + if [ "$SEND" = recovered ]; then + text="✅ ${RECOVERED_SUMMARY:-recovered: $SUMMARY}" + else + text="❌ $SUMMARY" + if [ -n "$FAILED_CHECKS_FILE" ] && [ -s "$FAILED_CHECKS_FILE" ]; then + text="$text Failing: $(head -c 800 "$FAILED_CHECKS_FILE" | head -n 1)" + fi fi text="$text $RUN_URL" diff --git a/.github/actions/telegram-alert/status-change.mjs b/.github/actions/telegram-alert/status-change.mjs new file mode 100644 index 0000000..a824af9 --- /dev/null +++ b/.github/actions/telegram-alert/status-change.mjs @@ -0,0 +1,169 @@ +/** + * Decides whether a finished run should send a Telegram message, by comparing its result with the + * previous completed run of the same workflow and target (read from the GitHub API). + * + * failing now, not failing before (or unknown) -> "failed" + * passing now, failing before -> "recovered" + * anything else -> "none" + * + * "Unknown previous" (first run, API error, nothing comparable) alerts on a failure and stays + * silent on a pass, so a broken lookup can only produce an extra alert, never hide a failure. + * Lookups never throw out of `run()`, so the alert step cannot change the job result. + * + * Lives in the action directory, not in happyview-modules/tooling, so deploy.yml can fetch it from + * the workflow's own commit when it rolls back to an older commit that predates it. + */ +import { appendFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +/** Most recent completed runs read per lookup. */ +const PAGE_SIZE = 50; +/** Comparable runs examined before giving up; bounds the per-run job lookups in step mode. */ +const MAX_CANDIDATES = 10; +const API_TIMEOUT_MS = 15_000; +/** @type {Record} */ +const STATE = { pass: "passing", fail: "failing" }; + +/** @typedef {"pass" | "fail"} Result */ +/** @typedef {"failed" | "recovered" | "none"} Alert */ +/** @typedef {(path: string) => Promise} GithubApi */ + +/** + * Maps a GitHub conclusion, or a job/step outcome, to a result. Cancelled, skipped and anything + * else is not a result. + * @param {string | null | undefined} conclusion + * @returns {Result | undefined} + */ +export function toResult(conclusion) { + if (conclusion === "success") return "pass"; + if (conclusion === "failure" || conclusion === "timed_out") return "fail"; + return undefined; +} + +/** + * @param {Result | undefined} current + * @param {Result | undefined} previous + * @returns {Alert} + */ +export function decideAlert(current, previous) { + if (current === "fail") return previous === "fail" ? "none" : "failed"; + if (current === "pass") return previous === "fail" ? "recovered" : "none"; + return "none"; +} + +/** + * Result of the step called `stepName` in `run`, or undefined when that step did not run or did not + * finish (for example the deploy failed before the smoke test started). + * @param {GithubApi} api @param {string} repo @param {{ id: number }} run @param {string} stepName + * @returns {Promise} + */ +async function stepResult(api, repo, run, stepName) { + const { jobs } = await api(`/repos/${repo}/actions/runs/${run.id}/jobs?per_page=100`); + for (const job of jobs) { + const step = (job.steps ?? []).find((/** @type {{ name: string }} */ s) => s.name === stepName); + if (step) return toResult(step.conclusion); + } + return undefined; +} + +/** + * Latest completed run of `workflow`, before the current one, for the same target, that produced a + * pass or a fail. "Same target" is the current run's `display_title` (match "title", used where + * run-name carries the environment) or its branch (match "branch", used where the environment is + * implied by the branch). With `step`, a run's result is that step's conclusion rather than the + * run's, and runs where the step did not finish are skipped. + * @param {{ api: GithubApi, repo: string, runId: string, workflow: string, + * match: string, step?: string }} options + * @returns {Promise<{ result: Result, url: string } | undefined>} + */ +export async function findPreviousResult({ api, repo, runId, workflow, match, step }) { + if (match !== "title" && match !== "branch") + throw new Error(`previous-match must be "title" or "branch", got "${match}".`); + const current = await api(`/repos/${repo}/actions/runs/${runId}`); + const sameTarget = + match === "branch" + ? (/** @type {any} */ run) => run.head_branch === current.head_branch + : (/** @type {any} */ run) => run.display_title === current.display_title; + + const query = new URLSearchParams({ status: "completed", per_page: String(PAGE_SIZE) }); + if (match === "branch") query.set("branch", current.head_branch); + const { workflow_runs: runs } = await api( + `/repos/${repo}/actions/workflows/${workflow}/runs?${query}`, + ); + + const candidates = runs + .filter((/** @type {any} */ run) => String(run.id) !== String(runId) && sameTarget(run)) + .slice(0, MAX_CANDIDATES); + for (const run of candidates) { + const result = step ? await stepResult(api, repo, run, step) : toResult(run.conclusion); + if (result) return { result, url: run.html_url }; + } + return undefined; +} + +/** + * Reads its inputs from `env`, as the composite action passes them. + * @param {Record} env @param {GithubApi} api + * @returns {Promise<{ send: Alert, notice: string, warning?: string }>} + */ +export async function run(env, api) { + const current = toResult(env.RESULT); + const workflow = env.PREVIOUS_WORKFLOW; + /** @type {Awaited>} */ + let previous; + let warning; + if (current && workflow) { + try { + previous = await findPreviousResult({ + api, + repo: env.GITHUB_REPOSITORY ?? "", + runId: env.GITHUB_RUN_ID ?? "", + workflow, + match: env.PREVIOUS_MATCH || "title", + step: env.PREVIOUS_STEP || undefined, + }); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + warning = `Could not read previous ${workflow} runs (${reason}); treating the previous status as unknown.`; + } + } + const send = decideAlert(current, previous?.result); + const now = current ? STATE[current] : `"${env.RESULT}"`; + const before = previous + ? `${STATE[previous.result]} in the previous run (${previous.url})` + : "of unknown status before"; + return { send, warning, notice: `Telegram alert: ${send}. Now ${now}, ${before}.` }; +} + +/** + * @param {{ token?: string, baseUrl?: string, fetchImpl?: typeof fetch }} options + * @returns {GithubApi} + */ +export function createApi({ token, baseUrl = "https://api.github.com", fetchImpl = fetch }) { + return async (path) => { + // The token only ever travels in this header: not in argv, the URL or any log line. + const response = await fetchImpl(`${baseUrl}${path}`, { + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + signal: AbortSignal.timeout(API_TIMEOUT_MS), + }); + if (!response.ok) throw new Error(`GET ${path} returned HTTP ${response.status}`); + return response.json(); + }; +} + +async function main() { + const env = process.env; + const api = createApi({ token: env.GITHUB_TOKEN, baseUrl: env.GITHUB_API_URL || undefined }); + const { send, notice, warning } = await run(env, api); + if (warning) process.stdout.write(`::warning::${warning}\n`); + process.stdout.write(`::notice::${notice}\n`); + const line = `send=${send}\n`; + if (env.GITHUB_OUTPUT) appendFileSync(env.GITHUB_OUTPUT, line); + else process.stdout.write(line); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await main(); diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7dc5329..c41c3aa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -101,6 +101,10 @@ jobs: needs: resolve runs-on: ubuntu-latest environment: ${{ needs.resolve.outputs.environment }} + # actions: read lets the alert step read this workflow's previous runs. + permissions: + contents: read + actions: read # Deploys to one environment (install + smoke) run one at a time and are never cancelled midway. concurrency: group: deploy-${{ needs.resolve.outputs.environment }} @@ -145,24 +149,26 @@ jobs: # Smoke test runs in this job so the deploy lock above covers it: another deploy to the same # environment cannot start until this check finishes. Anonymous GETs only, no secrets. - # A rollback to a commit that predates smoke.js skips it. + # A rollback to a commit that predates smoke.js skips it (the step shows as skipped, so it + # never counts as a smoke result). The step name is also read by the alert step below. + - name: Note that the deployed commit has no smoke test + if: hashFiles('happyview-modules/tooling/smoke.js') == '' + run: | + echo "::notice::Skipping smoke test: the deployed commit predates it." + - name: Smoke test the deployed API id: smoke + if: hashFiles('happyview-modules/tooling/smoke.js') != '' timeout-minutes: 5 env: API_BASE_URL: ${{ vars.HAPPYVIEW_BASE_URL }} FAILED_CHECKS_FILE: ${{ runner.temp }}/failed-checks.txt - run: | - if [ ! -f happyview-modules/tooling/smoke.js ]; then - echo "::notice::Skipping smoke test: the deployed commit predates it." - exit 0 - fi - pnpm smoke + run: pnpm smoke # A rollback checks out an older commit that may lack the alert action, so fetch it from the # workflow's own commit. - name: Check out alert action - if: failure() + if: success() || failure() uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} @@ -170,11 +176,30 @@ jobs: sparse-checkout: .github/actions persist-credentials: false - - name: Send Telegram alert on failure - if: failure() + # A failed deploy (anything other than the smoke test) alerts every time: each deploy is a + # one-off event, so there is no repeating alert to suppress. + - name: Send Telegram alert on deploy failure + if: failure() && steps.smoke.outcome != 'failure' + uses: ./.ci-alert/.github/actions/telegram-alert + with: + bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} + chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} + summary: deploy failed (${{ needs.resolve.outputs.environment }}, ${{ needs.resolve.outputs.branch }}) + + # The post-deploy smoke test alerts only when its result differs from the last deploy to this + # environment (same branch) whose smoke test finished: one alert when it starts failing, one + # when it recovers. Deploys that failed before the smoke test do not count. + - name: Send Telegram alert when the smoke status changes + if: (success() || failure()) && (steps.smoke.outcome == 'success' || steps.smoke.outcome == 'failure') uses: ./.ci-alert/.github/actions/telegram-alert with: bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} + github-token: ${{ github.token }} + result: ${{ steps.smoke.outcome }} + previous-workflow: deploy.yml + previous-match: branch + previous-step: Smoke test the deployed API failed-checks-file: ${{ runner.temp }}/failed-checks.txt - summary: ${{ steps.smoke.outcome == 'failure' && 'post-deploy smoke test failed' || 'deploy failed' }} (${{ needs.resolve.outputs.environment }}, ${{ needs.resolve.outputs.branch }}) + summary: post-deploy smoke test failed (${{ needs.resolve.outputs.environment }}, ${{ needs.resolve.outputs.branch }}) + recovered-summary: post-deploy smoke test recovered (${{ needs.resolve.outputs.environment }}, ${{ needs.resolve.outputs.branch }}) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 57a087d..b3ccbe3 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -2,6 +2,10 @@ name: Smoke # Catches drift between deploys: runs the same read-only smoke test as deploy.yml against the # live API. Scheduled runs target Production; use "Run workflow" for Test or Staging. +# Telegram: one alert when the smoke test starts failing, one when it recovers, none in between. +# The alert compares with the previous run of this environment by its run name, so the run name +# must stay unique per environment. +run-name: Smoke ${{ inputs.environment || 'Production' }} on: schedule: - cron: "*/30 * * * *" @@ -21,6 +25,10 @@ jobs: name: Smoke test (${{ inputs.environment || 'Production' }}) runs-on: ubuntu-latest timeout-minutes: 5 + # actions: read lets the alert step read this workflow's previous runs. + permissions: + contents: read + actions: read # Own group per environment, deliberately not the deploy group: a smoke run never queues # behind or cancels a deploy. A run that is still going when the next one starts is left alone. concurrency: @@ -66,11 +74,17 @@ jobs: FAILED_CHECKS_FILE: ${{ runner.temp }}/failed-checks.txt run: pnpm smoke - - name: Send Telegram alert on failure - if: failure() + # Runs after a pass or a fail (not a cancel) and sends only when the result changed. + - name: Send Telegram alert when the status changes + if: success() || failure() uses: ./.github/actions/telegram-alert with: bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} + github-token: ${{ github.token }} + result: ${{ job.status }} + previous-workflow: smoke.yml + previous-match: title failed-checks-file: ${{ runner.temp }}/failed-checks.txt summary: smoke test failed (${{ inputs.environment || 'Production' }}, ${{ github.ref_name }}) + recovered-summary: smoke test recovered (${{ inputs.environment || 'Production' }}, ${{ github.ref_name }}) diff --git a/happyview-modules/tests/unit/tooling/status-change.test.js b/happyview-modules/tests/unit/tooling/status-change.test.js new file mode 100644 index 0000000..d5edb0a --- /dev/null +++ b/happyview-modules/tests/unit/tooling/status-change.test.js @@ -0,0 +1,210 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + createApi, + decideAlert, + findPreviousResult, + run, + toResult, +} from "../../../../.github/actions/telegram-alert/status-change.mjs"; + +const REPO = "hypercerts-org/api"; + +const completed = (id, conclusion, extra = {}) => ({ + id, + status: "completed", + conclusion, + display_title: "Smoke Production", + head_branch: "main", + html_url: `https://github.com/${REPO}/actions/runs/${id}`, + ...extra, +}); + +/** + * Fake GitHub API. `current` is the running workflow run, `runs` the completed list (newest + * first) and `jobs` maps a run id to its jobs. Records every requested path. + */ +function fakeApi({ + current = { id: 100, display_title: "Smoke Production", head_branch: "main" }, + runs = [], + jobs = {}, +}) { + const paths = []; + const api = async (path) => { + paths.push(path); + if (path === `/repos/${REPO}/actions/runs/${current.id}`) return current; + if (path.startsWith(`/repos/${REPO}/actions/workflows/`)) return { workflow_runs: runs }; + const jobsFor = path.match(/\/actions\/runs\/(\d+)\/jobs/); + if (jobsFor) return { jobs: jobs[jobsFor[1]] ?? [] }; + throw new Error(`unexpected request ${path}`); + }; + return { api, paths }; +} + +const lookup = (api, overrides = {}) => + findPreviousResult({ + api, + repo: REPO, + runId: "100", + workflow: "smoke.yml", + match: "title", + ...overrides, + }); + +test("alerts only when the result changes", () => { + const table = [ + ["fail", "pass", "failed"], + ["fail", undefined, "failed"], + ["fail", "fail", "none"], + ["pass", "fail", "recovered"], + ["pass", "pass", "none"], + ["pass", undefined, "none"], + [undefined, "fail", "none"], + ]; + for (const [current, previous, expected] of table) + assert.equal(decideAlert(current, previous), expected, `${current} after ${previous}`); +}); + +test("only success and failure count as results", () => { + assert.equal(toResult("success"), "pass"); + assert.equal(toResult("failure"), "fail"); + assert.equal(toResult("timed_out"), "fail"); + for (const other of ["cancelled", "skipped", "neutral", "", null, undefined]) + assert.equal(toResult(other), undefined, String(other)); +}); + +test("previous result skips cancelled and skipped runs", async () => { + const { api, paths } = fakeApi({ + runs: [ + completed(99, "cancelled"), + completed(98, "skipped"), + completed(96, "failure"), + completed(95, "success"), + ], + }); + const previous = await lookup(api); + assert.deepEqual(previous, { result: "fail", url: `https://github.com/${REPO}/actions/runs/96` }); + assert.ok( + paths.some((p) => p.includes("/workflows/smoke.yml/runs?status=completed")), + "reads completed runs of the named workflow file", + ); +}); + +test("previous result ignores the current run and other environments", async () => { + const { api } = fakeApi({ + runs: [ + completed(100, "failure"), + completed(99, "failure", { display_title: "Smoke Staging" }), + completed(98, "success"), + ], + }); + assert.equal((await lookup(api))?.result, "pass"); +}); + +test("branch matching compares only runs from the current run's branch", async () => { + const { api, paths } = fakeApi({ + current: { id: 100, display_title: "Deploy production (push)", head_branch: "production" }, + runs: [ + completed(99, "failure", { head_branch: "main" }), + completed(98, "success", { head_branch: "production" }), + ], + }); + const previous = await lookup(api, { workflow: "deploy.yml", match: "branch" }); + assert.equal(previous?.result, "pass"); + assert.ok(paths.some((p) => p.includes("branch=production"))); +}); + +test("step matching uses the step's result and skips runs where it did not finish", async () => { + const smoke = (conclusion) => [ + { name: "Install", steps: [{ name: "Smoke test", conclusion }] }, + { name: "Resolve", steps: [{ name: "Set up job", conclusion: "success" }] }, + ]; + const { api } = fakeApi({ + current: { id: 100, display_title: "Deploy main (push)", head_branch: "main" }, + runs: [ + // Deploy failed before the smoke test started: the run failed, but there is no smoke result. + completed(99, "failure", { head_branch: "main" }), + // Smoke passed; a later step failed the run. + completed(98, "failure", { head_branch: "main" }), + completed(97, "success", { head_branch: "main" }), + ], + jobs: { 99: smoke("skipped"), 98: smoke("success"), 97: smoke("failure") }, + }); + const previous = await lookup(api, { + match: "branch", + workflow: "deploy.yml", + step: "Smoke test", + }); + assert.equal(previous?.result, "pass"); + assert.ok(previous?.url.endsWith("/98")); +}); + +test("previous result is unknown when no comparable run exists", async () => { + const { api } = fakeApi({ runs: [completed(99, "cancelled")] }); + assert.equal(await lookup(api), undefined); +}); + +const env = (overrides) => ({ + RESULT: "failure", + PREVIOUS_WORKFLOW: "smoke.yml", + GITHUB_REPOSITORY: REPO, + GITHUB_RUN_ID: "100", + ...overrides, +}); + +test("run: first failure alerts, repeat failure is silent, recovery alerts once", async () => { + const failing = fakeApi({ runs: [completed(99, "success")] }); + assert.equal((await run(env(), failing.api)).send, "failed"); + + const stillFailing = fakeApi({ runs: [completed(99, "failure"), completed(98, "success")] }); + assert.equal((await run(env(), stillFailing.api)).send, "none"); + + const recovered = fakeApi({ runs: [completed(99, "failure")] }); + assert.equal((await run(env({ RESULT: "success" }), recovered.api)).send, "recovered"); + + const stillPassing = fakeApi({ runs: [completed(99, "success")] }); + assert.equal((await run(env({ RESULT: "success" }), stillPassing.api)).send, "none"); +}); + +test("run: a cancelled job sends nothing and reads nothing", async () => { + const { api, paths } = fakeApi({ runs: [completed(99, "failure")] }); + assert.equal((await run(env({ RESULT: "cancelled" }), api)).send, "none"); + assert.deepEqual(paths, []); +}); + +test("run: without a workflow to compare with, a failure alerts and a pass is silent", async () => { + const { api, paths } = fakeApi({}); + assert.equal((await run(env({ PREVIOUS_WORKFLOW: "" }), api)).send, "failed"); + assert.equal((await run(env({ PREVIOUS_WORKFLOW: "", RESULT: "success" }), api)).send, "none"); + assert.deepEqual(paths, []); +}); + +test("run: a failed lookup warns, alerts on failure and stays silent on a pass", async () => { + const broken = async (path) => { + throw new Error(`GET ${path} returned HTTP 403`); + }; + const failed = await run(env(), broken); + assert.equal(failed.send, "failed"); + assert.match(failed.warning, /treating the previous status as unknown/); + assert.equal((await run(env({ RESULT: "success" }), broken)).send, "none"); +}); + +test("run: a bad previous-match fails safe with a message that names the fix", async () => { + const { api } = fakeApi({}); + const outcome = await run(env({ PREVIOUS_MATCH: "env" }), api); + assert.equal(outcome.send, "failed"); + assert.match(outcome.warning, /previous-match must be "title" or "branch"/); +}); + +test("the GitHub token is sent as a bearer header and never put in the URL", async () => { + const seen = []; + const fetchImpl = async (url, init) => { + seen.push({ url, headers: init.headers }); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }; + const api = createApi({ token: "ghs_secret", baseUrl: "https://api.example", fetchImpl }); + await api("/repos/a/b/actions/runs/1"); + assert.equal(seen[0].url, "https://api.example/repos/a/b/actions/runs/1"); + assert.equal(seen[0].headers.Authorization, "Bearer ghs_secret"); + assert.ok(!seen[0].url.includes("ghs_secret")); +}); From 47c02a8e751811c9ef9b2d5d1d97dd2e3438608c Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 16 Nov 2026 14:58:45 +0600 Subject: [PATCH 2/5] docs: describe change-only smoke Telegram alerts in CONTRIBUTING --- CONTRIBUTING.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ee265e..97fc0af 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,7 +89,7 @@ API_BASE_URL=https://test.api.hypercerts.dev pnpm e2e # pagination walks, ba - To force a deploy, run the Deploy workflow by hand from the environment's branch. Pushes that only change docs, tests or tooling skip the deploy. - A new query needs a case in `happyview-modules/tooling/smoke-cases.js`, or `pnpm test:unit` fails. - The API URLs for smoke and e2e are written in `smoke.yml` and `e2e-nightly.yml`; change them there if a URL changes. -- When a deploy, smoke or e2e run fails, a short Telegram message names the failing checks and links to the run. It needs the repository secrets `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID`; without them the alert is skipped. +- A failed deploy or e2e run sends a short Telegram message with a link to the run. The smoke test (after each deploy, and every 30 minutes on Production) sends a message only when its status changes: one when it starts failing, naming the failing checks, and one when it recovers. While it keeps failing or keeps passing, it stays silent. The alerts need the repository secrets `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID`; without them they are skipped. How the post-deploy checks fit together (`deploy.yml`, `smoke.yml`, `e2e-nightly.yml`): @@ -104,7 +104,7 @@ flowchart LR cron["Cron, every 30 min"] --> smoke nightly["Nightly"] --> e2e["E2E checks"] deploy -- failure --> tg["Telegram alert"] - smoke -- failure --> tg + smoke -- "status change" --> tg e2e -- failure --> tg ``` @@ -117,7 +117,13 @@ flowchart LR - Manifest coverage: every installed query must have a smoke case. - **E2E checks (`pnpm e2e`):** `e2e-nightly.yml` at 03:17 UTC, Test, then Staging, then Production. Pagination walks (at most 60 pages), 100-subject batches, and loosely compared cross-endpoint counts. - **Retries:** both runners' HTTP client retries a timeout or network error 3 times (1s, 2s, 4s); 4xx and 5xx responses fail at once. -- **Telegram alert:** sent when the deploy, the smoke test or the e2e checks fail, if the `TELEGRAM_*` secrets are set. +- **Telegram alert:** sent if the `TELEGRAM_*` secrets are set. + - A failed deploy (before the smoke test) and a failed e2e run alert every time. + - The smoke test alerts only when its result differs from the previous completed run: one failure message when it starts failing, one "recovered" message when it passes again, nothing in between. Cancelled and skipped runs are ignored. + - `smoke.yml` compares with the previous run of the same environment (matched by its run name, `Smoke `). The post-deploy smoke test compares with the previous deploy to the same branch whose smoke test finished. The two do not see each other's results. + - With no previous result to compare (first run, no comparable recent run, GitHub API read fails), a failure alerts and a pass stays silent. + - The previous runs are read with the job's `GITHUB_TOKEN` (`actions: read`); nothing else is stored. The logic is in `.github/actions/telegram-alert/status-change.mjs`. + - Sending an alert never changes the job result. - Both runners write a results table to the GitHub step summary. ## Operations with external effects From c9eda8f85cbf7f08e38c07ec8532a5a5f3e8bd8b Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 16 Nov 2026 15:10:34 +0600 Subject: [PATCH 3/5] ci: dedupe smoke.yml alerts on the smoke step result The change check compared whole-run conclusions, so a lost Telegram send, an install flake, or a run that never finished the smoke test counted as "already alerted" and silenced the next real failure. Give the smoke step an id and compare its outcome and the previous run's step of the same name, as deploy.yml does. Failures before the smoke test finished alert every time in a separate step. --- .github/workflows/smoke.yml | 27 +++++++++++++++++++++------ CONTRIBUTING.md | 6 +++--- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index b3ccbe3..f420ac9 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -3,8 +3,9 @@ name: Smoke # Catches drift between deploys: runs the same read-only smoke test as deploy.yml against the # live API. Scheduled runs target Production; use "Run workflow" for Test or Staging. # Telegram: one alert when the smoke test starts failing, one when it recovers, none in between. -# The alert compares with the previous run of this environment by its run name, so the run name -# must stay unique per environment. +# The alert compares the smoke step's result with the previous run of this environment, found by its +# run name, so the run name must stay unique per environment. Any other failure of this workflow +# (install, URL lookup) alerts every time. run-name: Smoke ${{ inputs.environment || 'Production' }} on: schedule: @@ -68,23 +69,37 @@ jobs: esac echo "url=$url" >> "$GITHUB_OUTPUT" + # The step name is also read by the alert step below to find the previous result. - name: Smoke test the live API + id: smoke env: API_BASE_URL: ${{ steps.target.outputs.url }} FAILED_CHECKS_FILE: ${{ runner.temp }}/failed-checks.txt run: pnpm smoke - # Runs after a pass or a fail (not a cancel) and sends only when the result changed. - - name: Send Telegram alert when the status changes - if: success() || failure() + # A failure before the smoke test finished (install, URL lookup) is a one-off event, not a + # repeating smoke result, so it alerts every time. + - name: Send Telegram alert on workflow failure + if: failure() && steps.smoke.outcome != 'failure' + uses: ./.github/actions/telegram-alert + with: + bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} + chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} + summary: smoke workflow failed before the smoke test finished (${{ inputs.environment || 'Production' }}, ${{ github.ref_name }}) + + # Alerts only when the smoke step's result differs from the last run of this environment + # whose smoke step finished. Runs that failed before it, or were cancelled, do not count. + - name: Send Telegram alert when the smoke status changes + if: (success() || failure()) && (steps.smoke.outcome == 'success' || steps.smoke.outcome == 'failure') uses: ./.github/actions/telegram-alert with: bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} chat-id: ${{ secrets.TELEGRAM_CHAT_ID }} github-token: ${{ github.token }} - result: ${{ job.status }} + result: ${{ steps.smoke.outcome }} previous-workflow: smoke.yml previous-match: title + previous-step: Smoke test the live API failed-checks-file: ${{ runner.temp }}/failed-checks.txt summary: smoke test failed (${{ inputs.environment || 'Production' }}, ${{ github.ref_name }}) recovered-summary: smoke test recovered (${{ inputs.environment || 'Production' }}, ${{ github.ref_name }}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 97fc0af..940264e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -118,9 +118,9 @@ flowchart LR - **E2E checks (`pnpm e2e`):** `e2e-nightly.yml` at 03:17 UTC, Test, then Staging, then Production. Pagination walks (at most 60 pages), 100-subject batches, and loosely compared cross-endpoint counts. - **Retries:** both runners' HTTP client retries a timeout or network error 3 times (1s, 2s, 4s); 4xx and 5xx responses fail at once. - **Telegram alert:** sent if the `TELEGRAM_*` secrets are set. - - A failed deploy (before the smoke test) and a failed e2e run alert every time. - - The smoke test alerts only when its result differs from the previous completed run: one failure message when it starts failing, one "recovered" message when it passes again, nothing in between. Cancelled and skipped runs are ignored. - - `smoke.yml` compares with the previous run of the same environment (matched by its run name, `Smoke `). The post-deploy smoke test compares with the previous deploy to the same branch whose smoke test finished. The two do not see each other's results. + - A failed deploy (before the smoke test), a `smoke.yml` run that fails before its smoke test finishes, and a failed e2e run alert every time. + - The smoke test alerts only when its result differs from the previous run whose smoke test finished: one failure message when it starts failing, one "recovered" message when it passes again, nothing in between. Cancelled and skipped runs are ignored. + - `smoke.yml` compares with the previous run of the same environment (matched by its run name, `Smoke `), by the result of its smoke step. The post-deploy smoke test compares with the previous deploy to the same branch whose smoke test finished. The two do not see each other's results. - With no previous result to compare (first run, no comparable recent run, GitHub API read fails), a failure alerts and a pass stays silent. - The previous runs are read with the job's `GITHUB_TOKEN` (`actions: read`); nothing else is stored. The logic is in `.github/actions/telegram-alert/status-change.mjs`. - Sending an alert never changes the job result. From f3fc744b392a1ae2a879e5db5ba0ba98c48b5945 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 16 Nov 2026 15:10:34 +0600 Subject: [PATCH 4/5] ci: keep the alert-action checkout from failing a good deploy The checkout now runs after successful deploys too, so a GitHub fetch error would turn a good deploy red and then run an alert step with a missing action. Make it continue-on-error and run the alert steps only when it succeeded. --- .github/workflows/deploy.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c41c3aa..3cd1e98 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -166,9 +166,12 @@ jobs: run: pnpm smoke # A rollback checks out an older commit that may lack the alert action, so fetch it from the - # workflow's own commit. + # workflow's own commit. This now runs after a good deploy too, so a failed fetch must not turn + # that deploy red (continue-on-error); the alert steps below run only if it succeeded. - name: Check out alert action + id: alert-action if: success() || failure() + continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} @@ -179,7 +182,7 @@ jobs: # A failed deploy (anything other than the smoke test) alerts every time: each deploy is a # one-off event, so there is no repeating alert to suppress. - name: Send Telegram alert on deploy failure - if: failure() && steps.smoke.outcome != 'failure' + if: failure() && steps.smoke.outcome != 'failure' && steps.alert-action.outcome == 'success' uses: ./.ci-alert/.github/actions/telegram-alert with: bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} @@ -190,7 +193,7 @@ jobs: # environment (same branch) whose smoke test finished: one alert when it starts failing, one # when it recovers. Deploys that failed before the smoke test do not count. - name: Send Telegram alert when the smoke status changes - if: (success() || failure()) && (steps.smoke.outcome == 'success' || steps.smoke.outcome == 'failure') + if: (success() || failure()) && steps.alert-action.outcome == 'success' && (steps.smoke.outcome == 'success' || steps.smoke.outcome == 'failure') uses: ./.ci-alert/.github/actions/telegram-alert with: bot-token: ${{ secrets.TELEGRAM_BOT_TOKEN }} From 9df026a8eda5b832b3f198be7d23cc7a5e042cce Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 16 Nov 2026 15:10:35 +0600 Subject: [PATCH 5/5] telegram-alert: retry the send and treat a missing decision as failed curl --retry 2 shrinks the chance that a transient Telegram error loses an alert. If status-change.mjs exits 0 without writing send= (for example when the entry-point guard is false), alert on failure like any other script error. A spawn test covers the script run as a program, and the test that asserted the impossible "current run is in the completed list" case is trimmed. --- .github/actions/telegram-alert/action.yml | 9 ++--- .../tests/unit/tooling/status-change.test.js | 35 +++++++++++++++---- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.github/actions/telegram-alert/action.yml b/.github/actions/telegram-alert/action.yml index 4f2972b..6c9388f 100644 --- a/.github/actions/telegram-alert/action.yml +++ b/.github/actions/telegram-alert/action.yml @@ -74,9 +74,10 @@ runs: PREVIOUS_STEP: ${{ inputs.previous-step }} GITHUB_TOKEN: ${{ inputs.github-token }} run: | - # Writes send=failed|recovered|none to $GITHUB_OUTPUT. If the script itself breaks, fall - # back to the old behaviour (alert on failure) rather than lose a failure alert. - if ! node "$GITHUB_ACTION_PATH/status-change.mjs"; then + # Writes send=failed|recovered|none to $GITHUB_OUTPUT. If the script breaks, or exits without + # writing a decision (for example it was not run as the entry point), fall back to the old + # behaviour (alert on failure) rather than lose a failure alert. + if ! node "$GITHUB_ACTION_PATH/status-change.mjs" || ! grep -q '^send=' "$GITHUB_OUTPUT"; then echo "::warning::Could not decide whether the status changed; alerting on failures only." if [ "$RESULT" = failure ]; then echo "send=failed" >> "$GITHUB_OUTPUT"; fi fi @@ -110,7 +111,7 @@ runs: text="$text $RUN_URL" printf 'url = "https://api.telegram.org/bot%s/sendMessage"\n' "$TELEGRAM_BOT_TOKEN" | - curl --silent --show-error --fail --max-time 15 --output /dev/null --config - \ + curl --silent --show-error --fail --retry 2 --max-time 15 --output /dev/null --config - \ --data-urlencode "chat_id=$TELEGRAM_CHAT_ID" \ --data-urlencode "text=$text" \ || echo "::warning::Telegram alert could not be sent." diff --git a/happyview-modules/tests/unit/tooling/status-change.test.js b/happyview-modules/tests/unit/tooling/status-change.test.js index d5edb0a..407a555 100644 --- a/happyview-modules/tests/unit/tooling/status-change.test.js +++ b/happyview-modules/tests/unit/tooling/status-change.test.js @@ -1,5 +1,10 @@ import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import test from "node:test"; +import { fileURLToPath } from "node:url"; import { createApi, decideAlert, @@ -9,6 +14,9 @@ import { } from "../../../../.github/actions/telegram-alert/status-change.mjs"; const REPO = "hypercerts-org/api"; +const SCRIPT = fileURLToPath( + new URL("../../../../.github/actions/telegram-alert/status-change.mjs", import.meta.url), +); const completed = (id, conclusion, extra = {}) => ({ id, @@ -90,13 +98,9 @@ test("previous result skips cancelled and skipped runs", async () => { ); }); -test("previous result ignores the current run and other environments", async () => { +test("previous result ignores other environments", async () => { const { api } = fakeApi({ - runs: [ - completed(100, "failure"), - completed(99, "failure", { display_title: "Smoke Staging" }), - completed(98, "success"), - ], + runs: [completed(99, "failure", { display_title: "Smoke Staging" }), completed(98, "success")], }); assert.equal((await lookup(api))?.result, "pass"); }); @@ -208,3 +212,22 @@ test("the GitHub token is sent as a bearer header and never put in the URL", asy assert.equal(seen[0].headers.Authorization, "Bearer ghs_secret"); assert.ok(!seen[0].url.includes("ghs_secret")); }); + +// action.yml runs the script as a program and reads `send=` from GITHUB_OUTPUT. The entry-point +// guard and the output write are not reachable through `run()`. +test("run as a program, it writes its decision to GITHUB_OUTPUT", () => { + const dir = mkdtempSync(join(tmpdir(), "status-change-")); + const output = join(dir, "github-output"); + try { + // Explicit env: nothing from a real CI run leaks in, and with no PREVIOUS_WORKFLOW it makes no + // network call. + const { status, stderr } = spawnSync(process.execPath, [SCRIPT], { + env: { PATH: process.env.PATH, RESULT: "failure", GITHUB_OUTPUT: output }, + encoding: "utf8", + }); + assert.equal(status, 0, stderr); + assert.equal(readFileSync(output, "utf8"), "send=failed\n"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +});