Files
aia-website/www/tools/audit-routes.mjs
T

418 lines
23 KiB
JavaScript

import fs from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { load } from 'cheerio';
import fg from 'fast-glob';
import matter from 'gray-matter';
import { createPublicationManifest } from '../src/utils/publication-policy.ts';
import { validateRedirects } from './generate-redirects.mjs';
export const ORIGIN = 'https://www.azinstitute4autism.com';
const ROOT = fileURLToPath(new URL('..', import.meta.url));
const REPORTS = path.join(ROOT, 'reports');
const BASELINE = 'route-baseline-2026-10-06.json';
const AGENT = 'AIA-Route-Audit';
const now = () => new Date().toISOString();
const compare = (a, b) => a.localeCompare(b, 'en');
const sorted = (items) => [...new Set(items)].sort(compare);
// Keep semantic query parameters (including unknown ones), their order, and
// fragments. The page key omits fragments only: anchors share an HTTP document.
export function normalizeReference(value, base = ORIGIN) {
let url;
try { url = new URL(value, base); } catch { return null; }
if (url.origin !== ORIGIN || url.username || url.password) return null;
if (!['http:', 'https:'].includes(url.protocol)) return null;
for (const key of [...url.searchParams.keys()]) {
if (/^(utm_.+|hsLang|hsCtaAttrib|gclid|dclid|fbclid|msclkid|_ga|_gl)$/i.test(key)) {
url.searchParams.delete(key);
}
}
const pathname = url.pathname.replace(/\/+$/, '') || '/';
const route = pathname + url.search;
return { route, reference: route + url.hash };
}
export function isPageRoute(route) {
const pathname = new URL(route, ORIGIN).pathname;
return !/^\/(?:hubfs|hub_generated|hs|_hcms|assets|_astro)(?:\/|$)/.test(pathname)
&& !/\.(?!html?$)[a-z\d]{1,8}$/i.test(pathname);
}
export function addDiscovery(records, value, source, observedAt, base = ORIGIN) {
const normalized = normalizeReference(value, base);
if (!normalized || !isPageRoute(normalized.route)) return null;
const { route, reference } = normalized;
const record = records.get(route) ?? {
route, discoveredAt: observedAt, sources: [], references: [], http: null
};
record.sources = sorted([...record.sources, source]);
if (reference !== route) record.references = sorted([...record.references, reference]);
records.set(route, record);
return record;
}
export function classify(local, production) {
const status = production?.http?.status;
const hasLocal = Boolean(local?.generated);
const redirect = local?.redirect;
if (status >= 300 && status < 400) {
if (redirect) {
const target = production?.http?.location ? normalizeReference(production.http.location)?.route : null;
return !target || target === redirect.to ? 'local-and-production-redirect' : 'local-production-redirect-conflict';
}
return hasLocal ? 'local-generated-production-redirect' : 'production-redirect-only';
}
if (status === 404 || status === 410) {
if (redirect) return 'local-redirect-production-absent';
return hasLocal ? (local.placeholder ? 'local-placeholder-production-absent' : 'local-only-production-absent')
: 'discovered-production-absent';
}
if (status >= 200 && status < 300) {
if (redirect) return 'local-redirect-production-live';
return hasLocal ? 'both-route-present-content-unverified' : 'live-only-verified';
}
const discovered = production?.sources?.some((s) => /^(sitemap|link|form|alternate):/.test(s));
if (discovered) {
if (redirect) return 'local-redirect-production-unverified';
return hasLocal ? 'both-discovered-http-unverified' : 'live-discovered-local-missing';
}
if (local) return redirect ? 'local-redirect-production-unverified'
: local.generated ? 'local-only-production-unverified' : 'local-source-not-generated';
return 'verification-inconclusive';
}
export function reconcile(local, production) {
const localMap = new Map(local.map((r) => [r.route, r]));
const liveMap = new Map(production.map((r) => [r.route, r]));
return sorted([...localMap.keys(), ...liveMap.keys()]).map((route) => {
const local = localMap.get(route) ?? null;
const production = liveMap.get(route) ?? null;
return {
route, local, production,
inProductionSitemap: Boolean(production?.sources.some((s) => s.startsWith('sitemap:'))),
disposition: classify(local, production)
};
});
}
export function summarize(routes) {
const hasSitemap = (r) => r.inProductionSitemap;
const generated = (r) => r.local?.generated;
const live = (r) => r.production?.http?.status >= 200 && r.production.http.status < 300;
return {
routes: routes.length,
localGenerated: routes.filter(generated).length,
localRedirects: routes.filter((r) => r.local?.redirect).length,
sitemap: routes.filter(hasSitemap).length,
sitemapLocalOverlap: routes.filter((r) => hasSitemap(r) && generated(r)).length,
sitemapLocalMissing: routes.filter((r) => hasSitemap(r) && !generated(r)).length,
localNotInSitemap: routes.filter((r) => generated(r) && !hasSitemap(r)).length,
verifiedLiveNotInSitemap: routes.filter((r) => live(r) && !hasSitemap(r)).length,
verifiedAbsent: routes.filter((r) => [404, 410].includes(r.production?.http?.status)).length,
dispositions: Object.fromEntries(sorted(routes.map((r) => r.disposition)).map((key) =>
[key, routes.filter((r) => r.disposition === key).length]))
};
}
// Only 404/410 establish absence. Redirects, blocks, errors and sitemap omission
// cannot. Compare inherited observations without replacing the original evidence.
export function baselineChanges(baseline, routes) {
const oldSitemap = new Set(baseline.production.filter((r) => r.sources.some((s) => s.startsWith('sitemap:'))).map((r) => r.route));
const currentSitemap = new Set(routes.filter((r) => r.inProductionSitemap).map((r) => r.route));
const observations = [];
for (const old of baseline.production) {
const current = routes.find((r) => r.route === old.route)?.production;
if (!old.http || !current?.http || !Number.isInteger(current.http.status)) continue;
for (const key of ['status', 'location', 'canonical', 'title', 'locale', 'robots']) {
// A redirect/error response without parsed HTML has unknown metadata;
// do not report its uncollected fields as removals from the live page.
if (!(key in old.http) || !(key in current.http)) continue;
if ((old.http[key] ?? null) !== (current.http[key] ?? null)) {
observations.push({ route: old.route, field: key, inherited: old.http[key], current: current.http[key] ?? null });
}
}
}
return {
sitemapAdded: sorted([...currentSitemap].filter((r) => !oldSitemap.has(r))),
sitemapRemoved: sorted([...oldSitemap].filter((r) => !currentSitemap.has(r))),
observations
};
}
export function parseRobots(text) {
const groups = [];
const sitemaps = [];
let group = null;
for (const line of text.split(/\r?\n/)) {
const match = line.replace(/#.*/, '').match(/^\s*([\w-]+)\s*:\s*(.*?)\s*$/);
if (!match) continue;
const [, rawKey, value] = match;
const key = rawKey.toLowerCase();
if (key === 'sitemap') { sitemaps.push(value); continue; }
if (key === 'user-agent') {
if (!group || group.started) { group = { agents: [], rules: [], delay: 0, started: false }; groups.push(group); }
group.agents.push(value.toLowerCase());
} else if (group) {
group.started = true;
if (['allow', 'disallow'].includes(key) && value) group.rules.push({ allow: key === 'allow', pattern: value });
if (key === 'crawl-delay' && Number.isFinite(Number(value))) group.delay = Math.max(0, Number(value) * 1000);
}
}
const specific = groups.filter((g) => g.agents.includes(AGENT.toLowerCase()));
const applicable = specific.length ? specific : groups.filter((g) => g.agents.includes('*'));
return { rules: applicable.flatMap((g) => g.rules), delay: Math.max(750, ...applicable.map((g) => g.delay)), sitemaps };
}
export function robotsAllows(url, policy) {
const { pathname, search } = new URL(url, ORIGIN);
const matches = policy.rules.filter(({ pattern }) => {
const end = pattern.endsWith('$');
const body = end ? pattern.slice(0, -1) : pattern;
const regex = body.split('*').map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('.*');
return new RegExp('^' + regex + (end ? '$' : '')).test(pathname + search);
}).sort((a, b) => b.pattern.replace(/[*$]/g, '').length - a.pattern.replace(/[*$]/g, '').length || Number(b.allow) - Number(a.allow));
return !matches.length || matches[0].allow;
}
function metadata(html) {
const $ = load(html);
return {
title: $('head title').text().trim(),
canonical: $('link[rel="canonical"]').attr('href') ?? null,
locale: $('html').attr('lang') ?? null,
robots: $('meta[name="robots"]').attr('content') ?? null
};
}
async function localInventory() {
const records = new Map();
const entries = [];
for (const file of (await fg('src/content/{pages,blog}/**/*.{md,mdx}', { cwd: ROOT })).sort(compare)) {
const source = await fs.readFile(path.join(ROOT, file), 'utf8');
const { data } = matter(source);
const [, , collection, ...parts] = file.split('/');
entries.push({ id: parts.join('/').replace(/\.mdx?$/, ''), collection, data, file, source });
}
for (const { route, eligible, sitemapEligible, canonical, externalCanonical, entry } of createPublicationManifest(entries, ORIGIN)) {
const { data, file, source } = entry;
records.set(route, {
route, source: file, draft: data.draft === true,
placeholder: /TODO: Replace this placeholder/i.test(source),
sourceCanonical: data.canonical ?? null, locale: data.lang, generated: false,
eligible, noindex: data.noindex === true, policyCanonical: canonical, externalCanonical, sitemapEligible
});
}
const htmlFiles = (await fg('dist/**/*.html', { cwd: ROOT })).sort(compare);
if (!htmlFiles.length) throw new Error('Build output unavailable. Run npm run build first.');
for (const file of htmlFiles) {
const route = file.replace(/^dist/, '').replace(/\/index\.html$/, '/').replace(/\.html$/, '').replace(/\/$/, '') || '/';
records.set(route, { ...records.get(route), route, generated: true, ...metadata(await fs.readFile(path.join(ROOT, file), 'utf8')) });
}
const redirects = validateRedirects(
JSON.parse(await fs.readFile(path.join(ROOT, 'src/data/redirects.json'), 'utf8')),
[...records.values()].filter(({ generated }) => generated).map(({ route }) => route)
);
for (const { from, to, status, reason } of redirects) {
records.set(from, {
...records.get(from), route: from, generated: false, eligible: false,
sitemapEligible: false, redirect: { to, status, reason }
});
}
return [...records.values()].sort((a, b) => compare(a.route, b.route));
}
async function sourceFingerprint() {
const hash = createHash('sha256');
for (const file of (await fg([
'src/**/*', 'public/**/*', 'astro.config.mjs', 'package.json', 'package-lock.json',
'tools/audit-routes.mjs', 'tools/generate-redirects.mjs'
], { cwd: ROOT, onlyFiles: true })).sort(compare)) {
hash.update(file + '\0'); hash.update(await fs.readFile(path.join(ROOT, file))); hash.update('\0');
}
return hash.digest('hex');
}
async function discover(local, baseline) {
const records = new Map();
const requests = [];
const warnings = [];
const cache = new Map();
let policy = { rules: [], delay: 750, sitemaps: [] };
let lastRequest = 0;
let halted = false;
async function get(url, purpose) {
url = new URL(url, ORIGIN).href;
if (new URL(url).origin !== ORIGIN) { warnings.push(`Skipped off-origin request: ${url}`); return null; }
if (cache.has(url)) return cache.get(url);
if (halted || !robotsAllows(url, policy)) {
const result = { http: { checkedAt: now(), status: null, error: halted ? 'request-budget-or-server-stop' : 'robots-disallowed' } };
cache.set(url, result); return result;
}
if (requests.length >= 80) { halted = true; warnings.push('80-request budget reached; remaining routes unverified.'); return get(url, purpose); }
await new Promise((resolve) => setTimeout(resolve, Math.max(0, policy.delay - (Date.now() - lastRequest))));
lastRequest = Date.now();
const checkedAt = now();
let result;
try {
const response = await fetch(url, { method: 'GET', redirect: 'manual', signal: AbortSignal.timeout(15000), headers: { 'User-Agent': `${AGENT}/1.0 (read-only migration inventory)` } });
const contentType = response.headers.get('content-type') ?? '';
result = { http: { checkedAt, status: response.status }, text: await response.text(), contentType };
const location = response.headers.get('location');
if (location) result.http.location = new URL(location, url).href;
if (response.status === 429 || response.status >= 500) {
halted = true; warnings.push(`Stopped requests after HTTP ${response.status}: ${url}`);
}
} catch (error) {
result = { http: { checkedAt, status: null, error: error.message } };
halted = true; warnings.push(`Stopped requests after network failure: ${url}`);
}
requests.push({ url, purpose, ...result.http });
cache.set(url, result);
return result;
}
const robots = await get('/robots.txt', 'robots');
if (![200, 404, 410].includes(robots.http.status)) throw new Error(`Cannot establish robots policy: ${JSON.stringify(robots.http)}`);
policy = parseRobots(robots.http.status === 200 ? robots.text : '');
if (policy.delay > 60000) throw new Error('robots crawl-delay exceeds this interactive audit budget; no page requests made.');
const pendingSitemaps = sorted([`${ORIGIN}/sitemap.xml`, ...policy.sitemaps]);
const seenSitemaps = new Set();
while (pendingSitemaps.length && seenSitemaps.size < 10) {
const url = pendingSitemaps.shift();
if (seenSitemaps.has(url)) continue;
seenSitemaps.add(url);
const response = await get(url, 'sitemap');
if (response?.http.status !== 200) { warnings.push(`Sitemap unavailable: ${url}`); continue; }
const $ = load(response.text, { xml: true });
$('sitemapindex > sitemap > loc').each((_i, e) => pendingSitemaps.push(new URL($(e).text(), url).href));
$('urlset > url').each((_i, e) => {
const record = addDiscovery(records, $(e).children('loc').text(), `sitemap:${url}`, response.http.checkedAt);
if (record) record.sitemapLastmod = $(e).children('lastmod').text() || null;
});
}
if (pendingSitemaps.length) warnings.push('Sitemap limit reached; discovery is incomplete.');
if (![...records.values()].some((r) => r.sources.some((s) => s.startsWith('sitemap:')))) warnings.push('No sitemap routes discovered; sitemap absence is not reliable.');
async function page(route) {
const record = records.get(route);
const response = await get(route, 'page');
if (!response) return;
record.http = response.http;
if (response.http.status !== 200 || !response.contentType?.includes('text/html')) return;
record.http = { ...response.http, ...metadata(response.text) };
const $ = load(response.text);
// Discover references only; do not recursively visit article links or submit
// forms. Form actions identify public search routes, not backend endpoints.
$('a[href], link[hreflang], form[action]').each((_i, e) => {
const node = $(e);
if (e.tagName === 'form' && (node.attr('method') ?? 'get').toLowerCase() !== 'get') return;
const kind = e.tagName === 'form' ? 'form' : e.tagName === 'link' ? 'alternate' : 'link';
const href = node.attr('href') ?? node.attr('action');
if (href) addDiscovery(records, href, `${kind}:${route}`, response.http.checkedAt, new URL(route, ORIGIN));
});
}
const seeds = ['/', '/es', '/library', '/es/library', '/ar/library'];
for (const route of seeds) { addDiscovery(records, route, 'seed:index', now()); await page(route); }
const localRoutes = new Set(local.filter((r) => r.generated).map((r) => r.route));
const targets = sorted([
...local.filter((r) => r.generated && !records.get(r.route)?.sources.some((s) => s.startsWith('sitemap:'))).map((r) => r.route),
...[...records.values()].filter((r) => r.sources.some((s) => s.startsWith('sitemap:')) && !localRoutes.has(r.route)).map((r) => r.route),
...baseline.production.filter((r) => r.http).map((r) => r.route)
]);
for (const route of targets) {
addDiscovery(records, route, localRoutes.has(route) ? 'target:local-or-baseline' : 'target:baseline-or-missing', now());
if (!seeds.includes(route)) await page(route);
}
// Verify discovered utility routes, including pagination; never broad-crawl
// articles or follow query variants automatically.
const utilities = sorted([...records.keys()].filter((r) => !r.includes('?') &&
(/\/(?:page\/\d+|author\/[^/]+)$/.test(r) || r === '/search' || r.startsWith('/lp/'))));
for (const route of utilities) if (!records.get(route).http) await page(route);
return {
checkedAt: now(), policy: { method: 'GET', minimumIntervalMs: policy.delay, maxRequests: 80, redirects: 'recorded, not followed', origin: ORIGIN },
robots: { ...robots.http, text: robots.text ?? '' }, requests, warnings,
production: [...records.values()].sort((a, b) => compare(a.route, b.route))
};
}
// One row per route/request keeps the JSON compact and directly reviewable.
export function compactJson(value) {
return '{\n' + Object.entries(value).map(([key, entry]) => ` ${JSON.stringify(key)}: ` +
(Array.isArray(entry) ? '[\n' + entry.map((row) => ' ' + JSON.stringify(row)).join(',\n') + '\n ]' : JSON.stringify(entry))).join(',\n') + '\n}\n';
}
async function latestEvidence() {
const files = await fg('route-reconciliation-*.json', { cwd: REPORTS });
if (!files.length) throw new Error('No saved reconciliation. Run npm run audit:routes online first.');
const dated = await Promise.all(files.map(async (file) => ({ file, generatedAt: JSON.parse(await fs.readFile(path.join(REPORTS, file), 'utf8')).generatedAt })));
dated.sort((a, b) => compare(a.generatedAt, b.generatedAt));
return path.join(REPORTS, dated.at(-1).file);
}
async function main() {
const args = process.argv.slice(2);
const check = args.includes('--check');
const offline = args.includes('--offline');
let evidenceFile;
for (let i = 0; i < args.length; i++) {
if (args[i] === '--evidence' && args[i + 1]) evidenceFile = path.resolve(args[++i]);
else if (!['--check', '--offline'].includes(args[i])) throw new Error(`Unknown or incomplete option: ${args[i]}`);
}
if (evidenceFile && !check && !offline) throw new Error('--evidence requires --check or --offline.');
const baselineText = await fs.readFile(path.join(REPORTS, BASELINE), 'utf8');
const baseline = JSON.parse(baselineText);
const baselineSha256 = createHash('sha256').update(baselineText).digest('hex');
const local = await localInventory();
const fingerprint = await sourceFingerprint();
let discovery;
if (check || offline) {
evidenceFile ??= await latestEvidence();
const saved = JSON.parse(await fs.readFile(evidenceFile, 'utf8'));
if (![1, 2].includes(saved.schemaVersion) || saved.baselineSha256 !== baselineSha256) throw new Error('Saved evidence schema/baseline does not match.');
discovery = { ...saved.discovery, production: saved.routes.flatMap((r) => r.production ? [r.production] : []) };
const routes = reconcile(local, discovery.production);
if (check) {
if (saved.schemaVersion !== 2) throw new Error('Saved evidence predates redirect-aware reconciliation. Run --offline to produce a schema 2 report.');
if (fingerprint !== saved.sourceFingerprint) throw new Error('Application source/assets changed since evidence capture. Build, then run --offline to produce a new reconciliation.');
const localRedirects = local.filter(({ redirect }) => redirect).map(({ route, redirect }) => ({ route, ...redirect }));
for (const [key, expected] of Object.entries({ routes, localRedirects, summary: summarize(routes), baselineChanges: baselineChanges(baseline, routes) })) {
if (JSON.stringify(saved[key]) !== JSON.stringify(expected)) throw new Error(`Offline ${key} mismatch. Rebuild/reconcile; saved evidence was not changed.`);
}
if (discovery.warnings.length) throw new Error(`Saved production discovery is incomplete: ${discovery.warnings.join('; ')}`);
console.log(`Offline check passed: ${path.basename(evidenceFile)} (no network or writes).`);
console.log(JSON.stringify(saved.summary, null, 2));
return;
}
} else discovery = await discover(local, baseline);
const routes = reconcile(local, discovery.production);
const generatedAt = now();
const { production: _production, ...discoveryDetails } = discovery;
const report = {
schemaVersion: 2, generatedAt, mode: offline ? 'offline-reconciliation' : 'fresh-discovery',
gitHead: execFileSync('git', ['rev-parse', 'HEAD'], { cwd: ROOT, encoding: 'utf8' }).trim(),
sourceFingerprint: fingerprint, baselineFile: BASELINE, baselineSha256,
limitations: ['Route presence is not content or visual parity.', 'Unrequested sitemap URLs are not HTTP-verified.', 'Existing dist required; rebuild before capture if application files changed.', 'Fragments are preserved references, not separately fetched documents.', 'Query variants are retained; automatic requests do not submit search terms.', 'Fresh requests record redirects without following them; inherited observations generally recorded final responses. A status difference alone is not evidence of a newly introduced redirect.', 'Local redirects are repository definitions; this report does not establish activation on a deployed host.'],
localUtilities: (await fg(['dist/robots.txt', 'dist/sitemap.xml'], { cwd: ROOT })).map((f) => '/' + f.slice(5)).sort(compare),
discovery: discoveryDetails,
localRedirects: local.filter(({ redirect }) => redirect).map(({ route, redirect }) => ({ route, ...redirect })),
summary: summarize(routes), baselineChanges: baselineChanges(baseline, routes), routes
};
const name = `route-reconciliation-${generatedAt.slice(0, 10)}${offline ? '-offline' : ''}.json`;
let output = path.join(REPORTS, name);
if (await fs.access(output).then(() => true).catch(() => false)) {
output = path.join(REPORTS, name.replace('.json', `-${generatedAt.slice(11).replace(/[:.]/g, '-')}.json`));
}
await fs.writeFile(output, compactJson(report), { flag: 'wx' });
console.log(`Saved ${path.relative(ROOT, output)}; ${discovery.requests.length} recorded production requests${offline ? ' (reused; no new requests)' : ''}.`);
console.log(JSON.stringify(report.summary, null, 2));
if (discovery.warnings.length) throw new Error(`Evidence saved with incomplete discovery: ${discovery.warnings.join('; ')}`);
}
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) {
main().catch((error) => { console.error(`Route audit failed: ${error.message}`); process.exitCode = 1; });
}