91 lines
5.8 KiB
JavaScript
91 lines
5.8 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
import fs from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import * as cheerio from 'cheerio';
|
|
export const sourceOrigin = 'https://www.azinstitute4autism.com';
|
|
export const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
|
|
|
|
export function extractPayload(source) {
|
|
const $ = cheerio.load(source);
|
|
return JSON.parse($('script[type="application/json"]').filter((_, el) => {
|
|
try { return Array.isArray(JSON.parse($(el).text()).streams); } catch { return false; }
|
|
}).first().text());
|
|
}
|
|
export function resourceReferences(html, view) {
|
|
const $ = cheerio.load(html, {}, false), references = [];
|
|
const record = (value, label, attribute) => {
|
|
let url;
|
|
try { url = new URL(value, sourceOrigin); } catch { return; }
|
|
if (url.origin !== sourceOrigin || !url.pathname.startsWith('/hubfs/')) return;
|
|
url.hash = '';
|
|
references.push({ original: value, sourceUrl: url.href, originalPath: url.pathname, label, attribute, view });
|
|
};
|
|
$('[href], [src], [srcset]').each((_, el) => {
|
|
for (const attr of ['href', 'src']) if ($(el).attr(attr)) record($(el).attr(attr), $(el).text().trim() || $(el).attr('alt') || '', attr);
|
|
for (const value of ($(el).attr('srcset') || '').split(',').filter(Boolean)) record(value.trim().split(/\s+/)[0], $(el).attr('alt') || '', 'srcset');
|
|
});
|
|
// Unknown CSS resource dependencies must not be silently considered migrated.
|
|
for (const match of html.matchAll(/url\(\s*['"]?([^)'"\s]+)['"]?\s*\)/g)) record(match[1], '', 'css-url');
|
|
return references;
|
|
}
|
|
export function rewriteResources(html, records) {
|
|
const mappings = new Map();
|
|
for (const record of records) {
|
|
if (!record.rewriteApproved) continue;
|
|
if (!record.destinationImplemented || !record.destinationVerified || !record.deliveryPath?.startsWith('/') || record.deliveryPath.startsWith('//') || /[?<>"'\s]/.test(record.deliveryPath)) throw new Error('Unverified resource mapping');
|
|
if (!['PUBLIC-APPROVED', 'STAFF-INTERNAL', 'MANAGEMENT-INTERNAL'].includes(record.classification)) throw new Error('Unclassified resource mapping');
|
|
if (record.classification !== 'PUBLIC-APPROVED' && !record.destinationAccessControlled) throw new Error('Internal destination lacks access control');
|
|
for (const ref of record.references) {
|
|
if (ref.attribute === 'css-url' || ref.attribute === 'srcset') throw new Error('CSS/srcset rewriting requires separate review');
|
|
mappings.set(ref.original, record.deliveryPath);
|
|
}
|
|
}
|
|
let count = 0;
|
|
// Attribute-only replacement preserves all unrelated original bytes and labels.
|
|
const rewritten = html.replace(/\b(href|src)\s*=\s*(["'])(.*?)\2/gi, (whole, attr, quote, value) => {
|
|
const decoded = cheerio.load(`<a href=${quote}${value}${quote}></a>`)('a').attr('href');
|
|
const target = mappings.get(decoded);
|
|
if (!target) return whole;
|
|
count++; return `${attr}=${quote}${target}${quote}`;
|
|
});
|
|
return { html: rewritten, count };
|
|
}
|
|
export async function collectResource(record, directory, approvedOrigins = [sourceOrigin], fetcher = fetch) {
|
|
const limit = 50 * 1024 * 1024;
|
|
const name = sha256(record.sourceUrl);
|
|
const destination = path.join(directory, name);
|
|
if (record.sha256 && record.localFile === name) {
|
|
try {
|
|
const bytes = await fs.readFile(destination);
|
|
if (sha256(bytes) === record.sha256 && bytes.length === record.size) return { ...record, downloadStatus: 'verified-existing' };
|
|
} catch { /* Retry missing/invalid local copy without overwriting it. */ }
|
|
}
|
|
let url = new URL(record.sourceUrl), response;
|
|
for (let redirects = 0; redirects <= 4; redirects++) {
|
|
if (url.protocol !== 'https:' || url.username || url.password || !approvedOrigins.includes(url.origin) || !url.pathname.startsWith('/hubfs/')) throw new Error('Unapproved resource destination');
|
|
response = await fetcher(url, { redirect: 'manual', signal: AbortSignal.timeout(20000), headers: { Accept: '*/*' } });
|
|
if ([301,302,303,307,308].includes(response.status)) {
|
|
await response.body?.cancel();
|
|
const location = response.headers.get('location');
|
|
if (!location || redirects === 4) throw new Error('Resource redirect limit');
|
|
url = new URL(location, url); continue;
|
|
}
|
|
break;
|
|
}
|
|
if (!response?.ok || Number(response.headers.get('content-length') || 0) > limit) { await response?.body?.cancel(); throw new Error('Resource HTTP/size failure'); }
|
|
const mime = (response.headers.get('content-type') || '').split(';')[0].toLowerCase();
|
|
const chunks = []; let size = 0;
|
|
const reader = response.body.getReader();
|
|
// Fetch timeout remains active through body streaming.
|
|
try { while (true) { const next = await reader.read(); if (next.done) break; size += next.value.length; if (size > limit) throw new Error('Resource size limit'); chunks.push(next.value); } }
|
|
catch (error) { await reader.cancel(); throw error; }
|
|
const bytes = Buffer.concat(chunks);
|
|
if (!size || /(?:html|javascript)/i.test(mime) || /^\s*(?:<!doctype html|<html|<script)/i.test(bytes.subarray(0, 512).toString())) throw new Error('Resource is empty or executable/error HTML');
|
|
if ((mime === 'application/pdf' || /\.pdf$/i.test(url.pathname)) && !bytes.subarray(0, 5).equals(Buffer.from('%PDF-'))) throw new Error('Invalid PDF signature');
|
|
if (mime === 'image/png' && bytes.subarray(0, 8).toString('hex') !== '89504e470d0a1a0a') throw new Error('Invalid PNG signature');
|
|
if (mime === 'image/jpeg' && bytes.subarray(0, 3).toString('hex') !== 'ffd8ff') throw new Error('Invalid JPEG signature');
|
|
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
|
await fs.writeFile(destination, bytes, { flag: 'wx', mode: 0o600 });
|
|
return { ...record, localFile: name, sha256: sha256(bytes), size, mime, downloadStatus: 'downloaded', verificationStatus: 'size-hash-and-applicable-signature-checked' };
|
|
}
|