matrix-org / matrix-org/matrix-hookshot

Add first-class handling for AlertManager webhook types with links from annotations

Open
#635 2 comments 8 reactions 0 assignees View on GitHub
T-Enhancement
Dominant language
TypeScript
Stars
450
Forks
95
Avg merge
1d 2h
Merged PRs (30d)
23

Description

We're using the following JS transformation for creating Generic Webhooks for Alertmanager alerting to Matrix rooms (and I'm not entirely sure where we got it from at the moment )

``` js
function statusBadge(status, severity) {
let statusColor;
if (status === "resolved") {
return `[RESOLVED]`;
}

switch(severity) {
case 'resolved':
case 'critical':
return `[FIRING - CRITICAL]`;
case 'warning':
return `[FIRING - WARNING]`;
default:
return `[${status.toUpperCase()}]`;
}
}

function silenceLink(alert, externalURL) {
filters = []
for (const [label, val] of Object.entries(alert.labels)) {
filters.push(encodeURIComponent(`${label}="${val}"`));
}
return `silence`;
}

if (!data.alerts) {
result = {
version: 'v2',
empty: true,
};
return;
}

const plainErrors = [];
const htmlErrors = [];
const { externalURL, alerts } = data;

for (const alert of data.alerts) {
plainErrors.push(`**[${alert.status.toUpperCase()} - ${alert.labels.severity}]** - [${alert.labels.alertname}](https://wiki.foo.dev/alerts/${alert.labels.alertname}): ${alert.annotations.description} [source](${alert.generatorURL})`);
htmlErrors.push(`${statusBadge(alert.status, alert.labels.severity)} ${alert.labels.alertname}: ${alert.annotations.description} source ${silenceLink(alert, externalURL)}`)
result = {
version: 'v2',
plain: plainErrors.join(`\n\n`),
html: htmlErrors.join(`
`),
msgtype: 'm.text'
};
}
```

This works well enough, but it gets complicated now that we're exploring appending further context URLs (grafana links, ArgoCD links etc) inside the alert's `annotations` field like so:

``` yaml
groups:
- name: argocd-apps
rules:
- alert: ArgoCDAppStuckUnhealthy
annotations:
runbook_url: https://wiki.foo.dev/argocd/ArgoCDAppStuckUnhealthy
summary: ArgoCD Application is in unhealthy state
description: |-
ArgoCD Application '{{ $labels.name }}' in namespace {{ $labels.dest_namespace }} is in {{ $labels.health_status }} state for longer than 5 minutes.
application_argocd_url: "https://argocd.foo.dev/applications/{{ $labels.name }}"
expr: |-
sum(argocd_app_info{ health_status!="Healthy", dest_namespace=~"{{ $targetNamespace }}" }) by (name, dest_namespace, health_status) > 0
for: 5m
labels:
severity: warning
```

At first I tried stuffing the URL into the description but the Javascript hook we've ended up with didn't seem to parse it correctly into a link (whether I used markdown or html, could be pebcak issue), but it would be cool to have the ability to specify an AlertManager webhook without a transform and have a map of `annotation` fields and link labels that gets built into a list of links in the issue, like this:

image

Which we've clumsily hacked together with this transform:

``` js
function statusBadge(status, severity) {
let statusColor;
if (status === "resolved") {
return `[RESOLVED]`;
}

switch (severity) {
case 'resolved':
case 'critical':
return `[FIRING - CRITICAL]`;
case 'warning':
return `[FIRING - WARNING]`;
default:
return `[${status.toUpperCase()}]`;
}
}

function wrapUrlHTML(link) {
return `${link.text}`
}

function wrapLinkListHTML(links) {
output = "

    \n"
    for (i = 0; i < links.length; i++) {
    output += `
  • ${wrapUrlHTML(links[i])}
  • \n`
    }
    output += "
\n"

return output
}

function wrapLinkListMD(links) {
output = "";
for (i = 0; i < links.length; i++) {
output += `- ${wrapUrlHTML(links[i])}\n`
}

return output
}

function silenceLink(alert, externalURL) {
filters = []
for (const [label, val] of Object.entries(alert.labels)) {
filters.push(encodeURIComponent(`${label}="${val}"`));
}

return {
href: `${externalURL}#silences/new?filter={${filters.join(", ")}}`,
text: "Silence alert"
}
}

function alertLinks(alert, externalURL) {
links = []

appName = `${alert.labels.pod}/${alert.labels.container}`

links.push(sourceLink(alert));
links.push(silenceLink(alert, externalURL));

if (alert.annotations.hasOwnProperty('application_argocd_url')) {
links.push({
href: alert.annotations.application_argocd_url,
text: `View App in ArgoCD`
});
}

if (alert.annotations.hasOwnProperty('logs_url')) {
links.push({
href: alert.annotations.logs_url,
text: `View logs for ${appName}`
});
}

if (alert.annotations.hasOwnProperty('argocd_logs_url')) {
links.push({
href: alert.annotations.argocd_logs_url,
text: `View logs for ArgoCD`
});
}
return links;
}

function sourceLink(alert) {
return {
href: alert.generatorURL,
text: "View alert in VMAlert"
}
}

function alertWikiLink(alert) {
return {
href: `https://wiki.element.dev/alerts/${alert.labels.alertname}`,
text: alert.labels.alertname
}
}

function buildAlertHtml(alert, externalURL) {
return `${statusBadge(alert.status, alert.labels.severity)} ${wrapUrlHTML(alertWikiLink(alert))}: ${alert.annotations.description}\n${wrapLinkListHTML(alertLinks(alert, externalURL))}`
}

function buildAlertPlain(alert, externalURL) {
return `**[${alert.status.toUpperCase()} - ${alert.labels.severity}]** - [${alert.labels.alertname}](https://wiki.element.dev/alerts/${alert.labels.alertname}): ${alert.annotations.description} \n${wrapLinkListMD(alertLinks(alert, externalURL))}`
}

if (!data.alerts) {
result = {
version: 'v2',
empty: true,
};
return;
}

const plainErrors = [];
const htmlErrors = [];
const { externalURL, alerts } = data;

for (const alert of data.alerts) {
plainErrors.push(buildAlertPlain(alert, externalURL));
htmlErrors.push(buildAlertHtml(alert, externalURL));
result = {
version: 'v2',
plain: plainErrors.join(`\n\n`),
html: htmlErrors.join(`
`),
msgtype: 'm.text'
};
}
```

This seems overkill, and won't scale for us, especially since this is a manual configuration step and can't be pre-provisioned / managed via IaC as far as I can tell.

Would it be possible to create a class of webhook aside from `Generic Webhook` that could handle Alertmanager hooks with it's own configuration to escape some of this transform stuff?

Contributor guide

Open the contributing guide

Research direction

Start by reviewing the existing Generic Webhook entry point and configuration, then compare them with the Alertmanager webhook payload and annotation examples in this issue. Done means Alertmanager hooks can map annotation fields to labeled links without requiring a manual transformation, while preserving the shown alert and silence links.

Written by the indexing model from the issue text.

Assessment

Tech stack
javascript, typescript
Domain
api, backend
Issue type
Feature
Difficulty
5/5
Estimated time
Over a week
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
30/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.