Skip to content

Change perfdata output for alert subcheck to be alert name and status #153

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
martialblog wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Change perfdata output for alert subcheck to be alert name and status #153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3133d9f
Fix alert state when no alerts are included
martialblog Nov 12, 2026
8e73f61
Change perfdata output for alert subcheck to be alert name and status
martialblog Nov 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 3 additions & 8 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,13 @@ Examples:
$ check_prometheus alert --name "PrometheusAlertmanagerJobMissing"
[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00
\_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00
| firing=1 pending=0 inactive=0
| PrometheusAlertmanagerJobMissing=2

$ check_prometheus a alert --name "PrometheusAlertmanagerJobMissing" --name "PrometheusTargetMissing"
[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00
\_[OK] [PrometheusTargetMissing] is inactive
\_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00
| total=2 firing=1 pending=0 inactive=1
| PrometheusAlertmanagerJobMissing=2 PrometheusTargetMissing=0

Flags:
-S, --label-key-state string Use the given AlertRule label to override the exit state for firing alerts.
Expand Down Expand Up @@ -201,8 +201,6 @@ $ check_prometheus alert
\_[OK] [HostHighCpuLoad] - Job: [alertmanager]
\_[CRITICAL] [HighResultLatency] - Job: [prometheus] on Instance: [localhost:9090] is firing - value: 11.00
\_[CRITICAL] [HighResultLatency] - Job: [node-exporter] on Instance: [node-exporter:9100] is firing - value: 10.00
| total=6 firing=3 pending=0 inactive=3

```

#### Checking multiple alerts
Expand All @@ -213,12 +211,11 @@ $ check_prometheus alert --name "HostHighCpuLoad" --name "HighResultLatency"
\_[OK] [HostHighCpuLoad] is inactive
\_[CRITICAL] [HighResultLatency] - Job: [prometheus] on Instance: [localhost:9090] is firing - value: 11.00
\_[CRITICAL] [HighResultLatency] - Job: [node-exporter] on Instance: [node-exporter:9100] is firing - value: 10.00
| total=3 firing=2 pending=0 inactive=1
```

```bash
$ check_prometheus alert --name "HostHighCpuLoad" --name "PrometheusTargetMissing"
OK - Alerts inactive | total=2 firing=0 pending=0 inactive=2
OK - Alerts inactive
```

#### Checking alerts via their labels
Expand Down Expand Up @@ -248,14 +245,12 @@ In Prometheus a "watchdog" or "dead man's switch" is an alert that is always fir
$ check_prometheus alert --name Watchdog -W --no-alerts-state 2
[OK] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive
\_ [OK] [Watchdog] is firing - value: 1.00 - {"alertname":"Watchdog","severity":"none"}
|total=1 firing=1 pending=0 inactive=0
```

```bash
$ check_prometheus alert --name Watchdog -W --no-alerts-state 2
[CRITICAL] No alerts retrieved
\_ [CRITICAL] No alerts retrieved
|total=0 firing=0 pending=0 inactive=0
```

## License
Expand Down
35 changes: 14 additions & 21 deletions cmd/alert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,13 @@ inactive = 0`,
$ check_prometheus alert --name "PrometheusAlertmanagerJobMissing"
[CRITICAL] - [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00
\_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00
| firing=1 pending=0 inactive=0
| PrometheusAlertmanagerJobMissing=2

$ check_prometheus alert --name "PrometheusAlertmanagerJobMissing" --name "PrometheusTargetMissing"
[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00
\_[OK] [PrometheusTargetMissing] is inactive
\_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00
| total=2 firing=1 pending=0 inactive=1`,
| PrometheusAlertmanagerJobMissing=2 PrometheusTargetMissing=0`,
Run: func(_ *cobra.Command, _ []string) {
// Convert --no-alerts-state to integer and validate input
noAlertsState, err := convertStateToInt(cliAlertConfig.NoAlertsState)
Expand Down Expand Up @@ -86,21 +86,13 @@ inactive = 0`,

// If there are no rules we can exit early
if len(rules) == 0 {
// Just an empty PerfdataList to have consistent perfdata output
pdlist := check.PerfdataList{
{Label: "total", Value: 0},
{Label: "firing", Value: 0},
{Label: "pending", Value: 0},
{Label: "inactive", Value: 0},
}

// Since the user is expecting the state of a certain alert and
// it that is not present it might be noteworthy.
if cliAlertConfig.AlertName != nil {
check.ExitWithPerfdata(check.Unknown, pdlist, "No such alert defined")
check.Exit(check.Unknown, "No such alert defined")
}

check.ExitWithPerfdata(noAlertsState, pdlist, "No alerts defined")
check.Exit(noAlertsState, "No alerts defined")
}

// Set initial capacity to reduce memory allocations
Expand All @@ -110,6 +102,8 @@ inactive = 0`,
}

var overall goresult.Overall
// We need to count how many times we added an alert for the no-alerts-state
var counterPartialResult int

for _, rl := range rules {
// If it's not the Alert we're looking for, Skip!
Expand Down Expand Up @@ -170,8 +164,11 @@ inactive = 0`,
}

sc.SetState(rlStatus)
sc.AddPerfdata(&check.Perfdata{Label: rl.AlertingRule.Name, Value: int(rlStatus)})
sc.SetOutput(rl.GetOutput())
overall.AddSubcheck(sc)

counterPartialResult++
}

// Handle active alerts
Expand Down Expand Up @@ -224,22 +221,18 @@ inactive = 0`,
// Set the alert in the internal Type to generate the output
rl.Alert = alert
sc.SetOutput(rl.GetOutput())
sc.AddPerfdata(&check.Perfdata{Label: rl.AlertingRule.Name, Value: int(rlStatus)})

@martialblog martialblog Nov 12, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This label for the perfdata is not unique.

overall.AddSubcheck(sc)

counterPartialResult++
}
}
}

counterAlert := counterFiring + counterPending + counterInactive

perfList := check.PerfdataList{
{Label: "total", Value: counterAlert},
{Label: "firing", Value: counterFiring},
{Label: "pending", Value: counterPending},
{Label: "inactive", Value: counterInactive},
}

// When there are no alerts we add an empty PartialResult just to have consistent output
if counterAlert == 0 {
if counterPartialResult == 0 {
sc := goresult.NewPartialResult()
sc.SetDefaultState(noAlertsState)
sc.SetOutput("No alerts retrieved")
Expand All @@ -252,7 +245,7 @@ inactive = 0`,
counterPending,
counterInactive))

check.ExitWithPerfdata(overall.GetStatus(), perfList, overall.GetOutput())
check.Exit(overall.GetStatus(), overall.GetOutput())
},
}

Expand Down
13 changes: 13 additions & 0 deletions cmd/alert_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,19 @@ exit status 2
\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
|total=3 firing=1 pending=1 inactive=1
exit status 2
`,
},
{
name: "alert-include-label-no-alerts-state",
server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write(loadTestdata(alertTestDataSet1))
})),
args: []string{"run", "../main.go", "alert", "--include-label", "foo=bar", "--no-alerts-state", "1", "-P", "true"},
expected: `[WARNING] - No alerts retrieved
\_ [WARNING] No alerts retrieved
|total=2 firing=1 pending=1 inactive=0
exit status 1
`,
},
{
Expand Down
Loading