matrix-org / matrix-org/matrix-hookshot
Add first-class handling for AlertManager webhook types with links from annotations
- Dominant language
- TypeScript
- Stars
- 450
- Forks
- 95
- Avg merge
- 1d 2h
- Merged PRs (30d)
- 23
Description
We're using the following JS transformation for creating Generic Webhooks for Alertmanager alerting to Matrix rooms (and I'm not entirely sure where we got it from at the moment )
``` js
function statusBadge(status, severity) {
let statusColor;
if (status === "resolved") {
return `[RESOLVED]`;
}
switch(severity) {
case 'resolved':
case 'critical':
return `[FIRING - CRITICAL]`;
case 'warning':
return `[FIRING - WARNING]`;
default:
return `[${status.toUpperCase()}]`;
}
}
function silenceLink(alert, externalURL) {
filters = []
for (const [label, val] of Object.entries(alert.labels)) {
filters.push(encodeURIComponent(`${label}="${val}"`));
}
return `silence`;
}
if (!data.alerts) {
result = {
version: 'v2',
empty: true,
};
return;
}
const plainErrors = [];
const htmlErrors = [];
const { externalURL, alerts } = data;
for (const alert of data.alerts) {
plainErrors.push(`**[${alert.status.toUpperCase()} - ${alert.labels.severity}]** - [${alert.labels.alertname}](https://wiki.foo.dev/alerts/${alert.labels.alertname}): ${alert.annotations.description} [source](${alert.generatorURL})`);
htmlErrors.push(`${statusBadge(alert.status, alert.labels.severity)} ${alert.labels.alertname}: ${alert.annotations.description} source ${silenceLink(alert, externalURL)}`)
result = {
version: 'v2',
plain: plainErrors.join(`\n\n`),
html: htmlErrors.join(`
`),
msgtype: 'm.text'
};
}
```
This works well enough, but it gets complicated now that we're exploring appending further context URLs (grafana links, ArgoCD links etc) inside the alert's `annotations` field like so:
``` yaml
groups:
- name: argocd-apps
rules:
- alert: ArgoCDAppStuckUnhealthy
annotations:
runbook_url: https://wiki.foo.dev/argocd/ArgoCDAppStuckUnhealthy
summary: ArgoCD Application is in unhealthy state
description: |-
ArgoCD Application '{{ $labels.name }}' in namespace {{ $labels.dest_namespace }} is in {{ $labels.health_status }} state for longer than 5 minutes.
application_argocd_url: "https://argocd.foo.dev/applications/{{ $labels.name }}"
expr: |-
sum(argocd_app_info{ health_status!="Healthy", dest_namespace=~"{{ $targetNamespace }}" }) by (name, dest_namespace, health_status) > 0
for: 5m
labels:
severity: warning
```
At first I tried stuffing the URL into the description but the Javascript hook we've ended up with didn't seem to parse it correctly into a link (whether I used markdown or html, could be pebcak issue), but it would be cool to have the ability to specify an AlertManager webhook without a transform and have a map of `annotation` fields and link labels that gets built into a list of links in the issue, like this:

Which we've clumsily hacked together with this transform:
``` js
function statusBadge(status, severity) {
let statusColor;
if (status === "resolved") {
return `[RESOLVED]`;
}
switch (severity) {
case 'resolved':
case 'critical':
return `[FIRING - CRITICAL]`;
case 'warning':
return `[FIRING - WARNING]`;
default:
return `[${status.toUpperCase()}]`;
}
}
function wrapUrlHTML(link) {
return `${link.text}`
}
function wrapLinkListHTML(links) {
output = "
- \n"
- ${wrapUrlHTML(links[i])} \n`
for (i = 0; i < links.length; i++) {
output += `
}
output += "
return output
}
function wrapLinkListMD(links) {
output = "";
for (i = 0; i < links.length; i++) {
output += `- ${wrapUrlHTML(links[i])}\n`
}
return output
}
function silenceLink(alert, externalURL) {
filters = []
for (const [label, val] of Object.entries(alert.labels)) {
filters.push(encodeURIComponent(`${label}="${val}"`));
}
return {
href: `${externalURL}#silences/new?filter={${filters.join(", ")}}`,
text: "Silence alert"
}
}
function alertLinks(alert, externalURL) {
links = []
appName = `${alert.labels.pod}/${alert.labels.container}`
links.push(sourceLink(alert));
links.push(silenceLink(alert, externalURL));
if (alert.annotations.hasOwnProperty('application_argocd_url')) {
links.push({
href: alert.annotations.application_argocd_url,
text: `View App in ArgoCD`
});
}
if (alert.annotations.hasOwnProperty('logs_url')) {
links.push({
href: alert.annotations.logs_url,
text: `View logs for ${appName}`
});
}
if (alert.annotations.hasOwnProperty('argocd_logs_url')) {
links.push({
href: alert.annotations.argocd_logs_url,
text: `View logs for ArgoCD`
});
}
return links;
}
function sourceLink(alert) {
return {
href: alert.generatorURL,
text: "View alert in VMAlert"
}
}
function alertWikiLink(alert) {
return {
href: `https://wiki.element.dev/alerts/${alert.labels.alertname}`,
text: alert.labels.alertname
}
}
function buildAlertHtml(alert, externalURL) {
return `${statusBadge(alert.status, alert.labels.severity)} ${wrapUrlHTML(alertWikiLink(alert))}: ${alert.annotations.description}\n${wrapLinkListHTML(alertLinks(alert, externalURL))}`
}
function buildAlertPlain(alert, externalURL) {
return `**[${alert.status.toUpperCase()} - ${alert.labels.severity}]** - [${alert.labels.alertname}](https://wiki.element.dev/alerts/${alert.labels.alertname}): ${alert.annotations.description} \n${wrapLinkListMD(alertLinks(alert, externalURL))}`
}
if (!data.alerts) {
result = {
version: 'v2',
empty: true,
};
return;
}
const plainErrors = [];
const htmlErrors = [];
const { externalURL, alerts } = data;
for (const alert of data.alerts) {
plainErrors.push(buildAlertPlain(alert, externalURL));
htmlErrors.push(buildAlertHtml(alert, externalURL));
result = {
version: 'v2',
plain: plainErrors.join(`\n\n`),
html: htmlErrors.join(`
`),
msgtype: 'm.text'
};
}
```
This seems overkill, and won't scale for us, especially since this is a manual configuration step and can't be pre-provisioned / managed via IaC as far as I can tell.
Would it be possible to create a class of webhook aside from `Generic Webhook` that could handle Alertmanager hooks with it's own configuration to escape some of this transform stuff?
Contributor guide
Research direction
Start by reviewing the existing Generic Webhook entry point and configuration, then compare them with the Alertmanager webhook payload and annotation examples in this issue. Done means Alertmanager hooks can map annotation fields to labeled links without requiring a manual transformation, while preserving the shown alert and silence links.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- javascript, typescript
- Domain
- api, backend
- Issue type
- Feature
- Difficulty
- 5/5
- Estimated time
- Over a week
- Activity status
- Stale
- Clarity
- Mostly clear
- Newbie friendliness
- 30/100