Adaptive thresholds and max notifications per hour

Feel free to close this issue if you like, just figured I'd share this use-case.

I'm using a custom email notify with Monirail, and, I don't wanna get bombarded with emails if lots of errors are triggered.

# Adaptive Notifier

## Purpose
- Sends threshold-based alerts with rate limiting.
- Tracks recent monitor history in SQLite and adapts thresholds.
- Cleans up old history to prevent DB bloat.

## Features
- Rate-limited emails (`MAX_EMAILS_PER_HOUR`).
- History window (`HISTORY_WINDOW_MS`) and adaptive threshold multiplier.
- Automatic cleanup of old monitor data.

```typescript
import { monitor, source, notify, watch } from "monirail";
import { adaptiveNotifier } from "./utils/adaptive";

const adaptiveEmail = notify({
type: "custom",
send: async (payload) => {
if (await adaptiveNotifier(payload)) {
// trigger email send
}
},
});

const monitors = [
monitor({
name: "app errors",
type: "threshold",
notifyOnNoData: false,
source: source({ type: "environment_logs" }),
filter: `@service:"app" AND @level:error`,
timeWindow: 5,
value: 20,
notifyOn: "above",
notify: [adaptiveEmail],
}),
];

try {
// make sure they don't crash
await Promise.all(monitors);

// Check monitors every minute
watch(1, monitors);
} catch (e) {
if (e instanceof Error) {
console.error(e.message);
} else {
console.error(e);
}
}
```

`adaptive.ts`
```typescript
import { Database } from "bun:sqlite";
import { NotificationPayload } from "monirail/index";

// -------------------------
// Constants
// -------------------------
const MAX_EMAILS_PER_HOUR = 3;
const HISTORY_WINDOW_MS = 60 * 60 * 1000; // 1 hour
const THRESHOLD_INCREASE = 1.5;

// -------------------------
// Types
// -------------------------
type MonitorState = {
monitor_name: string;
last_updated: number;
emails_sent: number;
history: string; // JSON string of { timestamp, value }[]
threshold_multiplier: number;
};

type HistoryEntry = {
timestamp: number;
value: number;
};

// -------------------------
// Database
// -------------------------
const db = new Database(process.env.SQLITE_DB_FILE!);

db.run(`
CREATE TABLE IF NOT EXISTS monitor_state (
monitor_name TEXT PRIMARY KEY,
last_updated INTEGER NOT NULL,
emails_sent INTEGER NOT NULL DEFAULT 0,
history TEXT NOT NULL DEFAULT '[]',
threshold_multiplier REAL NOT NULL DEFAULT 1.0
);
`);

// -------------------------
// Utilities
// -------------------------

/** Prunes old entries in a single history array */
function pruneHistory(
history: HistoryEntry[],
maxAgeMs: number = HISTORY_WINDOW_MS
): HistoryEntry[] {
const now = Date.now();
return history.filter((h) => now - h.timestamp <= maxAgeMs);
}

/** Cleans up all monitor histories in the database */
export function cleanupMonitorHistories() {
const now = Date.now();

const monitors = db
.prepare("SELECT monitor_name, history FROM monitor_state")
.all() as { monitor_name: string; history: string }[];

for (const monitor of monitors) {
const history: HistoryEntry[] = JSON.parse(monitor.history);
const pruned = pruneHistory(history);

if (pruned.length !== history.length) {
db.run(
`
UPDATE monitor_state
SET history = ?
WHERE monitor_name = ?
`,
[JSON.stringify(pruned), monitor.monitor_name]
);
}
}
}

/** Fetch monitor state by name, creating it if missing */
function getOrCreateMonitorState(name: string): MonitorState {
const now = Date.now();

let row = db
.prepare("SELECT * FROM monitor_state WHERE monitor_name = ?")
.get(name) as MonitorState | undefined;

if (!row) {
const initialHistory = JSON.stringify([]);
const initialMultiplier = 1.0;

db.run(
`
INSERT INTO monitor_state (monitor_name, last_updated, emails_sent, history, threshold_multiplier)
VALUES (?, ?, ?, ?, ?)
`,
[name, now, 0, initialHistory, initialMultiplier]
);

row = {
monitor_name: name,
last_updated: now,
emails_sent: 0,
history: initialHistory,
threshold_multiplier: initialMultiplier,
};
}

return row;
}

/** Upserts monitor state */
function upsertMonitorState(
row: MonitorState,
history: HistoryEntry[],
emailsSent: number,
thresholdMultiplier: number
) {
const now = Date.now();
db.run(
`
INSERT INTO monitor_state (monitor_name, last_updated, emails_sent, history, threshold_multiplier)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(monitor_name) DO UPDATE SET
last_updated = ?,
emails_sent = ?,
history = ?,
threshold_multiplier = ?
`,
[
row.monitor_name,
now,
emailsSent,
JSON.stringify(history),
thresholdMultiplier,
now,
emailsSent,
JSON.stringify(history),
thresholdMultiplier,
]
);
}

// -------------------------
// Adaptive Notifier
// -------------------------
export async function adaptiveNotifier(
payload: NotificationPayload
): Promise {
if (payload.type !== "threshold" || !payload.monitor?.name) return false;

// Clean up old history
cleanupMonitorHistories();

const name = payload.monitor.name;
const now = Date.now();

// Fetch or create monitor state
const row = getOrCreateMonitorState(name);

// Parse and prune history
let history: HistoryEntry[] = pruneHistory(JSON.parse(row.history));

// Append current value
history.push({ timestamp: now, value: payload.value });

// Calculate threshold
const threshold = payload.threshold * row.threshold_multiplier;
const alertsInWindow = history.filter((h) => h.value > threshold).length;

// Reset hourly email count if more than an hour has passed
let emailsSent =
now - row.last_updated > HISTORY_WINDOW_MS ? 0 : row.emails_sent;

// Determine if we should send an email
let sendEmailFlag = false;
if (payload.value > threshold && emailsSent < MAX_EMAILS_PER_HOUR) {
sendEmailFlag = true;
emailsSent++;
}

// Increase threshold multiplier if needed
let thresholdMultiplier = row.threshold_multiplier;
if (alertsInWindow > MAX_EMAILS_PER_HOUR) {
thresholdMultiplier *= THRESHOLD_INCREASE;
}

// Persist updated state
upsertMonitorState(row, history, emailsSent, thresholdMultiplier);

return sendEmailFlag;
}

```

Maybe something we can build directly into Monirail?

Start by reviewing the monitor, notify, and watch entry points shown in the example, then compare the proposed adaptive.ts flow with Monirail's existing notification behavior. The issue does not name repository files or tests; done would need an agreed Monirail design for rate-limited notifications, adaptive thresholds, history cleanup, and persistence.

Written by the indexing model from the issue text.