Add diagnostics to Playwright visual regression tests for font loading issues

Problem

Visual regression tests using Playwright are failing due to fonts not being rendered correctly in screenshots. This affects both local and CI (GitHub Actions) runs. Current tests do not provide enough information to diagnose whether fonts are missing due to network issues, timing, or environment differences.

Proposed Solution

Instrument the Playwright visual regression tests to collect detailed diagnostics about font loading for each snapshot. Move relevant helper functions to a dedicated test utility module for maintainability.

Key enhancements:

• Network instrumentation: Log all font file requests and their success/failure status.
• Font readiness: Wait for all fonts to finish loading before taking screenshots, and log the status of each font family.
• Diagnostics artifacts: For each test, save JSON files with font network activity, loaded font faces, and computed font-family details for key selectors.
• Prefont vs postfont screenshots: Capture screenshots before and after fonts load to detect FOUT/FOIT visually.
• Strict mode (optional): Fail the test if any expected font family is missing, controlled by an environment variable.

Sample Implementation

test/utils/fontDiagnostics.ts

// Move these helpers to a new test/utils/fontDiagnostics.ts
export async function waitForFonts(page, opts: { timeoutMs?: number } = {}) {
  const timeout = opts.timeoutMs ?? 8000;
  const start = Date.now();
  let lastPending = -1;
  while (Date.now() - start < timeout) {
    const pending = await page.evaluate(() => {
      const fontFaces = Array.from(document.fonts);
      const loading = fontFaces.filter(f => f.status === 'loading').length;
      return loading;
    });
    if (pending === 0 && lastPending === 0) break;
    lastPending = pending;
    await page.waitForTimeout(150);
  }
  try {
    await page.evaluate(async () => { await document.fonts.ready; });
  } catch {}
  const fontStatus = await page.evaluate(() => {
    const fontFaces = Array.from(document.fonts).map(f => ({
      family: f.family,
      status: f.status,
      weight: (f as any).weight,
      style: (f as any).style
    }));
    return { fontFaces };
  });
  return fontStatus;
}

export function setupFontNetworkLogging(page, collector) {
  page.on('request', req => {
    const url = req.url();
    if (/\.(woff2?|ttf|otf)(\?|$)/i.test(url) || /fonts\.googleapis|fonts\.gstatic/.test(url)) {
      collector.requests.push({ url, method: req.method(), state: 'requested', start: Date.now() });
    }
  });
  page.on('requestfinished', req => {
    const url = req.url();
    if (/\.(woff2?|ttf|otf)(\?|$)/i.test(url) || /fonts\.googleapis|fonts\.gstatic/.test(url)) {
      const entry = collector.requests.find(r => r.url === url && r.state === 'requested');
      if (entry) {
        entry.state = 'finished';
        entry.end = Date.now();
      }
    }
  });
  page.on('requestfailed', req => {
    const url = req.url();
    if (/\.(woff2?|ttf|otf)(\?|$)/i.test(url) || /fonts\.googleapis|fonts\.gstatic/.test(url)) {
      const entry = collector.requests.find(r => r.url === url && r.state === 'requested');
        if (entry) {
          entry.state = 'failed';
          entry.error = req.failure();
          entry.end = Date.now();
        } else {
          collector.requests.push({ url, method: req.method(), state: 'failed', error: req.failure(), end: Date.now() });
        }
    }
  });
}

tests/snapshots.spec.ts

import { test, expect } from '@playwright/test';
import * as fs from 'fs';
import * as path from 'path';
import { waitForFonts, setupFontNetworkLogging } from '../test/utils/fontDiagnostics';

const templatesDir = path.join(__dirname, '..', 'templates');
const artifactsDir = path.join(__dirname, '..', 'artifacts');

function getAllHtmlFiles(dirPath: string, arrayOfFiles: string[] = [], relativeDir: string = ''): string[] {
  const files = fs.readdirSync(dirPath);
  for (const file of files) {
    if (file.startsWith('_') || file.startsWith('.')) continue;
    const currentRelativePath = path.join(relativeDir, file);
    const fullPath = path.join(dirPath, file);
    if (fs.statSync(fullPath).isDirectory()) {
      arrayOfFiles = getAllHtmlFiles(fullPath, arrayOfFiles, currentRelativePath);
    } else if (file.endsWith('.html')) {
      arrayOfFiles.push(currentRelativePath);
    }
  }
  return arrayOfFiles;
}

const templateFiles = getAllHtmlFiles(templatesDir);
const expectedFamilies = (process.env.EXPECT_FONTS || '').split(',').map(f => f.trim()).filter(Boolean);

test.beforeAll(async () => {
  if (!fs.existsSync(artifactsDir)) {
    fs.mkdirSync(artifactsDir, { recursive: true });
  }
});

for (const templateFile of templateFiles) {
  test(`visual regression for ${templateFile}`, async ({ page }, testInfo) => {
    test.slow();
    const sanitizedTemplateFile = templateFile.replace(/[<>:"/\\|?*]/g, '_').replace(/ /g, '_');
    const baseName = sanitizedTemplateFile;
    const fontNet = { requests: [] };
    setupFontNetworkLogging(page, fontNet);
    let navigationError = null;
    try {
      await page.goto(templateFile, { waitUntil: 'domcontentloaded', timeout: 10000 });
    } catch (e) {
      navigationError = e;
      console.log(`Navigation issue on ${templateFile}: ${e}`);
    }
    await page.screenshot({ path: path.join(artifactsDir, `${baseName}_prefonts.png`), fullPage: true });
    const fontStatus = await waitForFonts(page);
    await page.screenshot({ path: path.join(artifactsDir, `${baseName}_postfonts.png`), fullPage: true });
    const computedSamples = await page.evaluate(() => {
      const selectors = ['h1', 'h2', 'p', 'nav', '.tt-main-navigation', '.header_mobile'];
      const data = [];
      for (const sel of selectors) {
        document.querySelectorAll(sel).forEach(el => {
          const cs = getComputedStyle(el);
          data.push({
            selector: sel,
            text: (el as HTMLElement).innerText.slice(0, 80),
            fontFamily: cs.fontFamily,
            fontWeight: cs.fontWeight,
            fontStyle: cs.fontStyle,
            fontSize: cs.fontSize
          });
        });
      }
      return data;
    });
    await fs.promises.writeFile(
      path.join(artifactsDir, `${baseName}_font-network.json`),
      JSON.stringify(fontNet, null, 2)
    );
    await fs.promises.writeFile(
      path.join(artifactsDir, `${baseName}_fonts-status.json`),
      JSON.stringify(fontStatus, null, 2)
    );
    await fs.promises.writeFile(
      path.join(artifactsDir, `${baseName}_computed-fonts.json`),
      JSON.stringify(computedSamples, null, 2)
    );
    await fs.promises.writeFile(
      path.join(artifactsDir, `${baseName}_content.html`),
      await page.content()
    );
    if (expectedFamilies.length) {
      const loadedFamilies = new Set(
        fontStatus.fontFaces.filter(f => f.status === 'loaded').map(f => f.family.replace(/['"]/g, '').toLowerCase())
      );
      const usedFamilies = new Set(
        computedSamples.map(s => s.fontFamily.split(',')[0].replace(/['"]/g, '').toLowerCase())
      );
      const missing = expectedFamilies.map(f => f.toLowerCase()).filter(f => !loadedFamilies.has(f) && !usedFamilies.has(f));
      if (missing.length) {
        testInfo.annotations.push({ type: 'missing-fonts', description: `Missing expected fonts: ${missing.join(', ')}` });
        throw new Error(`Expected fonts not loaded: ${missing.join(', ')}`);
      }
    }
    await expect(page).toHaveScreenshot(`${templateFile}.png`, { animations: 'disabled', fullPage: true, maxDiffPixels: 100, timeout: 10000 });
    if (navigationError) {
      console.log(`(Non-fatal) navigation error recorded for ${templateFile}:`, navigationError);
    }
  });
}

Acceptance Criteria

• Font diagnostics helpers are moved to a dedicated test/utils module.
• Visual regression tests use these helpers to log font requests and statuses.
• Artifacts for font diagnostics are saved alongside screenshots.
• Issue is labeled as both Bug and documentation (since docs on how to interpret results may be needed).

Start with tests/snapshots.spec.ts and the proposed test/utils/fontDiagnostics.ts module. Run the Playwright visual regression tests locally and in GitHub Actions, then verify that font requests, font status, computed-font data, and prefont/postfont screenshots are saved alongside the existing snapshots. Done means the helpers are separated, diagnostics are produced for each test, and the optional expected-font check works.