refactor(migration): remove obsolete mirror extraction tooling
- delete extraction scripts that depended on the removed local mirror - remove stale extract npm scripts and migration-brief references - drop the old mirror ignore rule from .gitignore - update reports and redirect reasons to use live-source wording
This commit is contained in:
@@ -58,6 +58,3 @@ temp/
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
Desktop.ini
|
||||
|
||||
# Read-only wget mirror used as migration source material
|
||||
/www.azinstitute4autism.com/
|
||||
|
||||
+2
-26
@@ -383,7 +383,6 @@ www/
|
||||
rtl.css
|
||||
|
||||
tools/
|
||||
extract-site.mjs
|
||||
crawl-live-site.mjs
|
||||
audit-links.mjs
|
||||
generate-redirects.mjs
|
||||
@@ -856,39 +855,17 @@ src/pages/es/[...slug].astro
|
||||
|
||||
Implement routing in a clean, maintainable way.
|
||||
|
||||
## Extraction scripts
|
||||
## Migration utility scripts
|
||||
|
||||
Create:
|
||||
|
||||
```txt
|
||||
tools/extract-site.mjs
|
||||
tools/crawl-live-site.mjs
|
||||
tools/audit-links.mjs
|
||||
tools/generate-redirects.mjs
|
||||
tools/generate-sitemap.mjs
|
||||
```
|
||||
|
||||
### `extract-site.mjs`
|
||||
|
||||
This script should:
|
||||
|
||||
- scan the available source and generated output
|
||||
- identify HTML files
|
||||
- infer URL paths
|
||||
- identify page type
|
||||
- identify blog posts
|
||||
- identify listing pages
|
||||
- extract title/meta/H1/body/main content
|
||||
- extract canonical and Open Graph metadata
|
||||
- extract schema/JSON-LD where present
|
||||
- convert blog posts to Markdown where feasible
|
||||
- create page Markdown where useful
|
||||
- copy/normalize assets
|
||||
- rewrite local asset references
|
||||
- create initial reports
|
||||
|
||||
The script does not need to be perfect, but it should be useful and documented.
|
||||
|
||||
### `crawl-live-site.mjs`
|
||||
|
||||
This script should:
|
||||
@@ -898,7 +875,7 @@ This script should:
|
||||
- compare discovered URLs against the current site source and generated output
|
||||
- document missing URLs/assets
|
||||
- avoid aggressive crawling
|
||||
- respect the scope of `www.azinstitute4autism.com`
|
||||
- respect the live-site scope
|
||||
|
||||
### `audit-links.mjs`
|
||||
|
||||
@@ -1109,7 +1086,6 @@ Add scripts similar to:
|
||||
"dev": "astro dev",
|
||||
"build": "astro check && astro build",
|
||||
"preview": "astro preview",
|
||||
"extract": "node tools/extract-site.mjs",
|
||||
"crawl": "node tools/crawl-live-site.mjs",
|
||||
"audit:links": "node tools/audit-links.mjs",
|
||||
"generate:sitemap": "node tools/generate-sitemap.mjs",
|
||||
|
||||
@@ -10,8 +10,6 @@
|
||||
"build": "astro build",
|
||||
"build:sandbox": "NODE_OPTIONS=--require=./tools/localhost-dns.cjs astro build",
|
||||
"preview": "astro preview",
|
||||
"extract": "node tools/extract-fallback.mjs",
|
||||
"extract:full": "node tools/extract-site.mjs",
|
||||
"extract:blog-footers": "node tools/extract-blog-footers.mjs",
|
||||
"crawl:live": "node tools/crawl-live-site.mjs",
|
||||
"audit:links": "node tools/audit-links.mjs",
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
# Cleanup Log
|
||||
|
||||
- Replaced HubSpot-generated wrappers, inline styles, scripts, analytics, and modules with reusable Astro components.
|
||||
- Preserved canonical clean URLs while excluding local mirror query-string and AMP duplicates.
|
||||
- Preserved canonical clean URLs while excluding HubSpot query-string and AMP duplicates.
|
||||
- Consolidated responsive `hs-fs` image variants to canonical `hubfs` assets where available.
|
||||
- Excluded mirrored HubSpot CSS and JavaScript from the new public asset package.
|
||||
- Self-hosted canonical images, SVGs, fonts, and PDFs from the mirror.
|
||||
- Excluded HubSpot CSS and JavaScript from the new public asset package.
|
||||
- Self-hosted canonical images, SVGs, fonts, and PDFs where practical.
|
||||
- Recreated navigation and footer from structured JSON.
|
||||
- Recreated visible contact and consultation forms as static accessible HTML.
|
||||
- Used the source palette: AIA blue `#254080`, dark blue, straw yellow, orange, and teal.
|
||||
- Used the live site's locally mirrored Playfair Display headings, Rubik body copy, and Caveat accent font.
|
||||
- Used self-hosted Playfair Display headings, Rubik body copy, and Caveat accent font.
|
||||
- Added multilingual collection and route structure with Arabic RTL support.
|
||||
- Kept all extracted source content editable as Markdown.
|
||||
- Replaced the initial generic rounded-card visual system with live-derived
|
||||
@@ -21,18 +21,17 @@
|
||||
- Reorganized the homepage source into named `home:` frontmatter blocks so the
|
||||
content is easier to inspect and edit without changing the rendered layout.
|
||||
- Replaced the extracted linear team article with a dedicated live-derived team grid.
|
||||
- Repaired mirror-rewritten PDF, lightbox, CTA, and relative content links.
|
||||
- Repaired rewritten PDF, lightbox, CTA, and relative content links.
|
||||
- Limited language-switcher choices to translations that have generated routes.
|
||||
- Verified and implemented the current production likes/views API contract.
|
||||
- Recovered 18 live page-banner assets that the wget mirror missed because
|
||||
HubSpot injected them through malformed inline `background-image` styles.
|
||||
- Recovered 18 live page-banner assets that HubSpot injected through malformed
|
||||
inline `background-image` styles.
|
||||
- Restored the live library banner and consultation form section background.
|
||||
- Added a mapped-page-imagery audit to prevent missing visual assets from
|
||||
silently passing the source link audit.
|
||||
- Removed duplicated article titles, author/date blocks, and featured images
|
||||
from all 65 blog Markdown bodies; these elements are rendered by the shared
|
||||
blog-post layout. Updated both extractors and added a blog-content audit to
|
||||
prevent recurrence.
|
||||
blog-post layout. Added a blog-content audit to prevent recurrence.
|
||||
- Converted 19 FAQ-bearing blog posts to MDX and restored a reusable
|
||||
`FAQAccordion.astro` component that owns the live-style accordion and
|
||||
matching `FAQPage` JSON-LD.
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
## Status
|
||||
|
||||
A maintainable Astro migration and substantial fidelity-first pass are
|
||||
implemented. The live public site, not the wget mirror, was used as the
|
||||
authority for the shared shell, English and Spanish homepages, service-page
|
||||
family, team page, library indexes, and blog-post family.
|
||||
implemented. The live public site was used as the authority for the shared
|
||||
shell, English and Spanish homepages, service-page family, team page, library
|
||||
indexes, and blog-post family.
|
||||
|
||||
## Migrated Content and Assets
|
||||
|
||||
@@ -15,9 +15,9 @@ family, team page, library indexes, and blog-post family.
|
||||
- 167 self-hosted images, 20 font files, and two PDF downloads.
|
||||
- Current June and May 2026 English library articles are included.
|
||||
|
||||
The raw mirror remains untouched. HubSpot-generated wrappers, analytics,
|
||||
scripts, CSS, query-language duplicates, AMP variants, pagination, and author
|
||||
archive variants are not carried into the Astro implementation.
|
||||
HubSpot-generated wrappers, analytics, scripts, CSS, query-language duplicates,
|
||||
AMP variants, pagination, and author archive variants are not carried into the
|
||||
Astro implementation.
|
||||
|
||||
## Live-Source Fidelity Pass
|
||||
|
||||
@@ -55,7 +55,7 @@ Implemented fidelity work includes:
|
||||
links, fawn backgrounds, spacing, rounded presentation, and quote-mark SVG.
|
||||
- Replaced generic oversized cards, rounded controls, and marketing heroes
|
||||
with live-derived typography, palette, widths, spacing, and compact controls.
|
||||
- Corrected material live/mirror discrepancies found during the pass,
|
||||
- Corrected material live-source discrepancies found during the pass,
|
||||
including current homepage ESA copy, testimonial content, and ABA copy.
|
||||
|
||||
## URLs and Redirects
|
||||
@@ -157,7 +157,7 @@ final production-build proof.
|
||||
- Perform full-page desktop and mobile visual comparisons for page-specific
|
||||
layouts beyond the completed homepage, team, service, library, and article
|
||||
families.
|
||||
- Editorially review extracted long-form content for remaining mirror artifacts,
|
||||
- Editorially review extracted long-form content for remaining extraction artifacts,
|
||||
stale phone/email references, and heading hierarchy.
|
||||
- Review all source alt text, keyboard behavior, screen-reader output, and
|
||||
contrast with accessibility tooling.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from,to,status,reason
|
||||
"/aba","/aba-therapy",301,"Brief alias to preserved mirror URL"
|
||||
"/autismevaluations","/autism-evaluations",301,"Brief alias to preserved mirror URL"
|
||||
"/learnersocialclub","/learner-social-club",301,"Brief alias to preserved mirror URL"
|
||||
"/aba","/aba-therapy",301,"Brief legacy alias to preserved URL"
|
||||
"/autismevaluations","/autism-evaluations",301,"Brief legacy alias to preserved URL"
|
||||
"/learnersocialclub","/learner-social-club",301,"Brief legacy alias to preserved URL"
|
||||
|
||||
|
@@ -18,8 +18,8 @@
|
||||
|
||||
## Cleanup
|
||||
|
||||
- HubSpot `?hsLang=...` mirror duplicates are excluded from generated routes.
|
||||
- AMP mirror variants are excluded because Astro pages are responsive and static.
|
||||
- HubSpot `?hsLang=...` duplicates are excluded from generated routes.
|
||||
- AMP variants are excluded because Astro pages are responsive and static.
|
||||
- Blog pagination and author archive variants are not preserved as separate generated pages.
|
||||
- Brief aliases `/aba`, `/autismevaluations`, and `/learnersocialclub` redirect to preserved source URLs.
|
||||
|
||||
|
||||
@@ -1,220 +0,0 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const mirror = path.resolve(root, '../www.azinstitute4autism.com');
|
||||
const content = path.join(root, 'src/content');
|
||||
const assets = path.join(root, 'public/assets');
|
||||
const reports = path.join(root, 'reports');
|
||||
const site = 'https://www.azinstitute4autism.com';
|
||||
const mkdir = (value) => fs.mkdir(value, { recursive: true });
|
||||
const quote = (value = '') => JSON.stringify(String(value).replace(/\s+/g, ' ').trim());
|
||||
const csv = (value = '') => `"${String(value).replaceAll('"', '""')}"`;
|
||||
const blogPreamble = /^# .+\n\n!\[[^\]]*\]\(\/assets\/images\/rula-diab-avatar\.jpg\)\n\n[^\n]+\n\n[^\n]+\n\n!\[[^\]]*\]\([^)]+\)\n\n/;
|
||||
|
||||
async function walk(dir, prefix = '') {
|
||||
const output = [];
|
||||
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
|
||||
const relative = path.join(prefix, entry.name);
|
||||
if (entry.isDirectory()) output.push(...await walk(path.join(dir, entry.name), relative));
|
||||
else output.push(relative);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function logical(file) {
|
||||
return file.replace(/(?:\.html)?\?(?:hsLang=[^.]+|hs_amp=true)\.html$/, '.html');
|
||||
}
|
||||
|
||||
function selectCanonical(files) {
|
||||
const map = new Map();
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.html') || file.includes('hs_amp=true') || /\/(?:page|author)\//.test(file)) continue;
|
||||
const target = logical(file);
|
||||
const key = `${target.startsWith('ar/') ? 'ar' : target.startsWith('es/') ? 'es' : 'en'}:${target}`;
|
||||
if (!map.has(key) || (!file.includes('?') && map.get(key).includes('?'))) map.set(key, file);
|
||||
}
|
||||
return [...map.values()];
|
||||
}
|
||||
|
||||
function decode(value = '') {
|
||||
return value
|
||||
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
|
||||
.replace(/&#x([\da-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)))
|
||||
.replaceAll('&', '&').replaceAll('"', '"').replaceAll(''', "'")
|
||||
.replaceAll('<', '<').replaceAll('>', '>').replaceAll(' ', ' ');
|
||||
}
|
||||
|
||||
function text(value = '') {
|
||||
return decode(value.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function normalizeHref(value = '') {
|
||||
return decode(value)
|
||||
.replace(/^https?:\/\/www\.azinstitute4autism\.com/, '')
|
||||
.replace(/(?:\.html)?(?:%3F|\?)(?:hsLang=[^.#]+|hs_amp=true)(?:\.html)?$/, '')
|
||||
.replace(/\.html$/, '')
|
||||
.replace(/^index$/, '/');
|
||||
}
|
||||
|
||||
function inline(value = '') {
|
||||
return text(value.replace(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, href, label) => {
|
||||
const cleanLabel = text(label);
|
||||
return cleanLabel ? `[${cleanLabel}](${normalizeHref(href)})` : '';
|
||||
}));
|
||||
}
|
||||
|
||||
function meta(html, name, property = false) {
|
||||
const key = property ? 'property' : 'name';
|
||||
const pattern = new RegExp(`<meta[^>]+${key}=["']${name}["'][^>]+content=["']([^"']*)`, 'i');
|
||||
const reverse = new RegExp(`<meta[^>]+content=["']([^"']*)["'][^>]+${key}=["']${name}["']`, 'i');
|
||||
return decode(html.match(pattern)?.[1] || html.match(reverse)?.[1] || '');
|
||||
}
|
||||
|
||||
function asset(value = '') {
|
||||
const match = value.match(/(?:hs-fs\/)?hubfs\/([^?#]+)/);
|
||||
return match ? `/assets/images/${path.basename(decodeURIComponent(match[1])).split('?')[0]}` : undefined;
|
||||
}
|
||||
|
||||
function normalizeTable(value = '') {
|
||||
return value
|
||||
.replace(/(<a\b[^>]*\bhref=["'])([^"']+)(["'])/gi,
|
||||
(_, prefix, href, suffix) => `${prefix}${normalizeHref(href)}${suffix}`)
|
||||
.replace(/(<img\b[^>]*\bsrc=["'])([^"']+)(["'])/gi,
|
||||
(_, prefix, src, suffix) => `${prefix}${asset(src) || src}${suffix}`);
|
||||
}
|
||||
|
||||
function markdownFrom(html) {
|
||||
let body = html.match(/<main[\s\S]*?<\/main>/i)?.[0] ||
|
||||
html.match(/blog-post__body[\s\S]*?(?=<footer|blog-post__tags|<\/article>)/i)?.[0] ||
|
||||
'';
|
||||
const tables = [];
|
||||
body = body
|
||||
.replace(/<(script|style|noscript|header|footer|nav|form)\b[\s\S]*?<\/\1>/gi, '')
|
||||
.replace(/<table[\s\S]*?<\/table>/gi, (match) => {
|
||||
const token = `__TABLE_${tables.length}__`;
|
||||
tables.push(normalizeTable(match));
|
||||
return `\n\n${token}\n\n`;
|
||||
})
|
||||
.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, value) => `\n\n> ${inline(value)}\n\n`)
|
||||
.replace(/<img[^>]+src=["']([^"']+)["'][^>]*alt=["']([^"']*)["'][^>]*>/gi, (_, src, alt) => {
|
||||
const local = asset(src);
|
||||
return local ? `\n\n\n\n` : '';
|
||||
})
|
||||
.replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, level, value) => `\n\n${'#'.repeat(Number(level))} ${inline(value)}\n\n`)
|
||||
.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_, value) => `\n- ${inline(value)}`)
|
||||
.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, value) => `\n\n${inline(value)}\n\n`)
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ');
|
||||
body = decode(body).replace(/[ \t]+/g, ' ').replace(/^\s+$/gm, '').replace(/\n{3,}/g, '\n\n').trim();
|
||||
return tables.reduce((output, table, index) => output.replaceAll(`__TABLE_${index}__`, table), body);
|
||||
}
|
||||
|
||||
function record(file, html) {
|
||||
const target = logical(file);
|
||||
const lang = target.startsWith('ar/') ? 'ar' : target.startsWith('es/') ? 'es' : 'en';
|
||||
const url = target === 'index.html' ? '/' : `/${target.replace(/\/index\.html$/, '').replace(/\.html$/, '')}`;
|
||||
const title = text(html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || url);
|
||||
const h1 = text(html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || title);
|
||||
const published = meta(html, 'article:published_time', true) ||
|
||||
html.match(/<time[^>]+datetime=["']([^"']+)/i)?.[1] || '';
|
||||
return {
|
||||
file, lang, url, title, h1,
|
||||
description: meta(html, 'description'),
|
||||
image: asset(meta(html, 'og:image', true)),
|
||||
date: /^\d{4}-\d{2}-\d{2}/.test(published) ? published.slice(0, 10) : '2024-01-01',
|
||||
markdown: markdownFrom(html)
|
||||
};
|
||||
}
|
||||
|
||||
function frontmatter(item, blog) {
|
||||
const slug = ['/', '/ar', '/es'].includes(item.url) ? 'index' : item.url.split('/').filter(Boolean).at(-1);
|
||||
const lines = [
|
||||
'---',
|
||||
`title: ${quote(item.title)}`,
|
||||
`description: ${quote(item.description)}`,
|
||||
`slug: ${quote(slug)}`,
|
||||
`canonical: ${quote(`${site}${item.url}`)}`,
|
||||
`lang: ${quote(item.lang)}`,
|
||||
`translationKey: ${quote(slug)}`,
|
||||
item.image ? `featuredImage: ${quote(item.image)}` : '',
|
||||
blog ? `date: "${item.date}"` : '',
|
||||
blog ? 'author: "rula-diab"' : '',
|
||||
blog ? 'category: "Library"' : '',
|
||||
blog ? 'tags: []' : '',
|
||||
'draft: false',
|
||||
'---',
|
||||
'',
|
||||
(blog ? item.markdown.replace(blogPreamble, '') : item.markdown) || `# ${item.h1}`,
|
||||
''
|
||||
].filter((line) => line !== '');
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
async function copySourceAssets(files) {
|
||||
const rows = ['source,target,size,kind'];
|
||||
const seen = new Set();
|
||||
await Promise.all(['images', 'fonts', 'downloads'].map((kind) => fs.rm(path.join(assets, kind), { recursive: true, force: true })));
|
||||
for (const file of files.filter((name) => name.startsWith('hubfs/') || name.startsWith('hs-fs/hubfs/') || name.startsWith('_hcms/googlefonts/'))) {
|
||||
const font = file.startsWith('_hcms/googlefonts/');
|
||||
const raw = font ? file.replace('_hcms/googlefonts/', '') : path.basename(file).split('?')[0];
|
||||
if (!font && !/\.(?:avif|gif|jpe?g|png|svg|webp|pdf)$/i.test(raw)) continue;
|
||||
const key = `${font}:${raw}`;
|
||||
if (!raw || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
const kind = font ? 'fonts' : raw.endsWith('.pdf') ? 'downloads' : 'images';
|
||||
const target = path.join(assets, kind, raw);
|
||||
await mkdir(path.dirname(target));
|
||||
await fs.copyFile(path.join(mirror, file), target);
|
||||
const size = (await fs.stat(target)).size;
|
||||
rows.push([file, path.relative(root, target), size, kind].map(csv).join(','));
|
||||
}
|
||||
await fs.writeFile(path.join(reports, 'asset-inventory.csv'), `${rows.join('\n')}\n`);
|
||||
}
|
||||
|
||||
async function removeMissingFeaturedImages() {
|
||||
for (const type of ['pages', 'blog']) {
|
||||
for (const file of await walk(path.join(content, type))) {
|
||||
const target = path.join(content, type, file);
|
||||
let source = await fs.readFile(target, 'utf8');
|
||||
const image = source.match(/^featuredImage:\s*"\/assets\/images\/([^"]+)"/m)?.[1];
|
||||
if (!image) continue;
|
||||
if (!await fs.access(path.join(assets, 'images', image)).then(() => true).catch(() => false)) {
|
||||
source = source.replace(/^featuredImage:.*\n/m, '');
|
||||
await fs.writeFile(target, source);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const files = await walk(mirror);
|
||||
const selected = selectCanonical(files);
|
||||
const records = [];
|
||||
for (const file of selected) records.push(record(file, await fs.readFile(path.join(mirror, file), 'utf8')));
|
||||
await Promise.all(['pages', 'blog', 'authors'].map((type) => fs.rm(path.join(content, type), { recursive: true, force: true })));
|
||||
for (const lang of ['en', 'ar', 'es']) {
|
||||
for (const type of ['pages', 'blog', 'authors']) await mkdir(path.join(content, type, lang));
|
||||
}
|
||||
for (const item of records) {
|
||||
const blog = item.url.includes('/library/') && !item.url.endsWith('/library');
|
||||
const slug = ['/', '/ar', '/es'].includes(item.url) ? 'index' : item.url.split('/').filter(Boolean).at(-1);
|
||||
await fs.writeFile(path.join(content, blog ? 'blog' : 'pages', item.lang, `${slug}.md`), frontmatter(item, blog));
|
||||
}
|
||||
for (const lang of ['en', 'ar', 'es']) {
|
||||
await fs.writeFile(path.join(content, 'authors', lang, 'rula-diab.md'), `---\nname: "Rula Diab"\nslug: "rula-diab"\ndescription: "Founder and clinical leader at Arizona Institute for Autism."\navatar: "/assets/images/rula-diab-avatar.jpg"\nlang: "${lang}"\ntranslationKey: "rula-diab"\n---\n`);
|
||||
}
|
||||
if (!records.some((item) => item.lang === 'ar' && item.url === '/ar')) {
|
||||
await fs.writeFile(path.join(content, 'pages/ar/index.md'), `---\ntitle: "معهد أريزونا للتوحد"\ndescription: "صفحة عربية تمهيدية لمعهد أريزونا للتوحد."\nslug: "index"\ncanonical: "https://www.azinstitute4autism.com/ar"\nlang: "ar"\ntranslationKey: "home"\ndraft: false\n---\n\n<!-- TODO: Replace this placeholder with reviewed Arabic content. -->\n\nتتوفر المقالات العربية الحالية في المكتبة. يرجى مراجعة المحتوى العربي قبل النشر.\n`);
|
||||
}
|
||||
const rows = ['source_file,url,language,type,title,description,h1'];
|
||||
for (const item of records) rows.push([item.file, item.url, item.lang, item.url.includes('/library/') ? 'blog' : 'page', item.title, item.description, item.h1].map(csv).join(','));
|
||||
await mkdir(reports);
|
||||
await fs.writeFile(path.join(reports, 'url-inventory.csv'), `${rows.join('\n')}\n`);
|
||||
await copySourceAssets(files);
|
||||
await removeMissingFeaturedImages();
|
||||
console.log(`Fallback extraction completed: ${records.length} canonical records.`);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -1,198 +0,0 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import fg from 'fast-glob';
|
||||
import { load } from 'cheerio';
|
||||
import TurndownService from 'turndown';
|
||||
import matter from 'gray-matter';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const mirror = path.resolve(root, '../www.azinstitute4autism.com');
|
||||
const site = 'https://www.azinstitute4autism.com';
|
||||
const contentRoot = path.join(root, 'src/content');
|
||||
const publicRoot = path.join(root, 'public/assets');
|
||||
const reportsRoot = path.join(root, 'reports');
|
||||
const turndown = new TurndownService({ headingStyle: 'atx', bulletListMarker: '-' });
|
||||
turndown.remove(['script', 'style', 'noscript', 'iframe']);
|
||||
|
||||
const mkdir = (dir) => fs.mkdir(dir, { recursive: true });
|
||||
const clean = (value = '') => value.replace(/\s+/g, ' ').trim();
|
||||
const csv = (value = '') => `"${String(value).replaceAll('"', '""')}"`;
|
||||
const yaml = (value = '') => JSON.stringify(String(value));
|
||||
const blogPreamble = /^# .+\n\n!\[[^\]]*\]\(\/assets\/images\/rula-diab-avatar\.jpg\)\n\n[^\n]+\n\n[^\n]+\n\n!\[[^\]]*\]\([^)]+\)\n\n/;
|
||||
|
||||
function contentMarkdown(markdown, isBlog) {
|
||||
return isBlog ? markdown.replace(blogPreamble, '') : markdown;
|
||||
}
|
||||
|
||||
function logicalFile(file) {
|
||||
return file.replace(/(?:\.html)?\?(?:hsLang=[^.]+|hs_amp=true)\.html$/, '.html');
|
||||
}
|
||||
|
||||
function canonicalFiles(files) {
|
||||
const selected = new Map();
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.html') || /\/(?:page|author)\//.test(file) || file.includes('hs_amp=true')) continue;
|
||||
const logical = logicalFile(file);
|
||||
const lang = logical.startsWith('ar/') ? 'ar' : logical.startsWith('es/') ? 'es' : 'en';
|
||||
const key = `${lang}:${logical}`;
|
||||
const current = selected.get(key);
|
||||
if (!current || (!file.includes('?') && current.includes('?'))) selected.set(key, file);
|
||||
}
|
||||
return [...selected.values()];
|
||||
}
|
||||
|
||||
function sourceUrl(file) {
|
||||
const logical = logicalFile(file);
|
||||
if (logical === 'index.html') return '/';
|
||||
return `/${logical.replace(/\/index\.html$/, '').replace(/\.html$/, '')}`;
|
||||
}
|
||||
|
||||
function isBlogFile(file) {
|
||||
return file.startsWith('library/') || file.includes('/library/');
|
||||
}
|
||||
|
||||
function languageFor(file, $) {
|
||||
if (file.startsWith('ar/')) return 'ar';
|
||||
if (file.startsWith('es/')) return 'es';
|
||||
return $('html').attr('lang')?.split('-')[0] || 'en';
|
||||
}
|
||||
|
||||
function localizeAsset(value = '') {
|
||||
const decoded = value.replace(/^https?:\/\/[^/]+/, '').replace(/^\.\.\//, '/');
|
||||
const match = decoded.match(/(?:\/)?(?:hs-fs\/)?hubfs\/([^?#]+)/);
|
||||
if (!match) return undefined;
|
||||
return `/assets/images/${path.basename(decodeURIComponent(match[1])).split('?')[0]}`;
|
||||
}
|
||||
|
||||
function extractRecord(file, html) {
|
||||
const $ = load(html);
|
||||
const lang = languageFor(file, $);
|
||||
const url = sourceUrl(file);
|
||||
const title = clean($('title').first().text()) || clean($('h1').first().text()) || url;
|
||||
const description = $('meta[name="description"]').attr('content') || '';
|
||||
const h1 = clean($('h1').first().text());
|
||||
const image = localizeAsset(
|
||||
$('meta[property="og:image"]').attr('content') ||
|
||||
$('.blog-post__body img, main img, .body-container-wrapper img').first().attr('src') ||
|
||||
''
|
||||
);
|
||||
const alt = clean($('.blog-post__body img, main img, .body-container-wrapper img').first().attr('alt') || '');
|
||||
const dateText = $('meta[property="article:published_time"]').attr('content') ||
|
||||
$('time').first().attr('datetime') || '2024-01-01';
|
||||
const date = /^\d{4}-\d{2}-\d{2}/.test(dateText) ? dateText.slice(0, 10) : '2024-01-01';
|
||||
const selector = isBlogFile(file) ? '.blog-post__body' : 'main, .body-container-wrapper';
|
||||
const body = $(selector).first().clone();
|
||||
body.find('header, footer, nav, form, script, style, noscript, .hs_cos_wrapper_type_form').remove();
|
||||
body.find('*').removeAttr('style').removeAttr('id').removeAttr('data-hs-cos-general-type').removeAttr('data-hs-cos-type');
|
||||
body.find('a').each((_, element) => {
|
||||
const href = $(element).attr('href');
|
||||
if (!href) return;
|
||||
$(element).attr('href', href
|
||||
.replace(/\.html(?:%3F|\?)[^"]*$/, '')
|
||||
.replace(/\.html$/, '')
|
||||
.replace(/^index$/, '/'));
|
||||
});
|
||||
body.find('img').each((_, element) => {
|
||||
const src = localizeAsset($(element).attr('src'));
|
||||
if (src) $(element).attr('src', src);
|
||||
});
|
||||
const tables = [];
|
||||
body.find('table').each((_, element) => {
|
||||
const token = `__TABLE_${tables.length}__`;
|
||||
tables.push($(element).prop('outerHTML'));
|
||||
$(element).replaceWith(`\n\n${token}\n\n`);
|
||||
});
|
||||
const markdown = contentMarkdown(
|
||||
tables.reduce((output, table, index) => output.replaceAll(`__TABLE_${index}__`, table),
|
||||
turndown.turndown(body.html() || '').replace(/\n{3,}/g, '\n\n').trim()),
|
||||
isBlogFile(file)
|
||||
);
|
||||
return { file, lang, url, title, description, h1, image, alt, date, markdown };
|
||||
}
|
||||
|
||||
function frontmatter(record, type) {
|
||||
const slug = ['/', '/ar', '/es'].includes(record.url) ? 'index' : record.url.split('/').filter(Boolean).at(-1);
|
||||
const data = {
|
||||
title: record.title,
|
||||
description: record.description,
|
||||
slug,
|
||||
canonical: `${site}${record.url}`,
|
||||
lang: record.lang,
|
||||
translationKey: slug,
|
||||
draft: false
|
||||
};
|
||||
if (record.image) data.featuredImage = record.image;
|
||||
if (record.alt) data.alt = record.alt;
|
||||
if (type === 'blog') Object.assign(data, {
|
||||
date: record.date,
|
||||
author: 'rula-diab',
|
||||
category: 'Library',
|
||||
tags: []
|
||||
});
|
||||
return matter.stringify(record.markdown || `# ${record.h1 || record.title}\n`, data);
|
||||
}
|
||||
|
||||
async function copyAssets() {
|
||||
const assets = await fg(['hubfs/*', 'hs-fs/hubfs/*', '_hcms/googlefonts/**/*'], { cwd: mirror, onlyFiles: true });
|
||||
const inventory = ['source,target,size,kind'];
|
||||
const seen = new Set();
|
||||
await Promise.all(['images', 'fonts', 'downloads'].map((kind) => fs.rm(path.join(publicRoot, kind), { recursive: true, force: true })));
|
||||
for (const source of assets) {
|
||||
const original = path.join(mirror, source);
|
||||
const stat = await fs.stat(original);
|
||||
const isFont = source.startsWith('_hcms/googlefonts/');
|
||||
const rawName = isFont ? source.replace('_hcms/googlefonts/', '') : path.basename(source).split('?')[0];
|
||||
if (!isFont && !/\.(?:avif|gif|jpe?g|png|svg|webp|pdf)$/i.test(rawName)) continue;
|
||||
if (!rawName || seen.has(`${isFont}:${rawName}`)) continue;
|
||||
seen.add(`${isFont}:${rawName}`);
|
||||
const target = path.join(publicRoot, isFont ? 'fonts' : rawName.endsWith('.pdf') ? 'downloads' : 'images', rawName);
|
||||
await mkdir(path.dirname(target));
|
||||
await fs.copyFile(original, target);
|
||||
inventory.push([source, path.relative(root, target), stat.size, isFont ? 'font' : path.extname(rawName).slice(1)].map(csv).join(','));
|
||||
}
|
||||
await fs.writeFile(path.join(reportsRoot, 'asset-inventory.csv'), `${inventory.join('\n')}\n`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await Promise.all([
|
||||
mkdir(contentRoot), mkdir(publicRoot), mkdir(reportsRoot),
|
||||
...['en', 'ar', 'es'].flatMap((lang) => [
|
||||
mkdir(path.join(contentRoot, 'pages', lang)),
|
||||
mkdir(path.join(contentRoot, 'blog', lang)),
|
||||
mkdir(path.join(contentRoot, 'authors', lang))
|
||||
])
|
||||
]);
|
||||
const files = canonicalFiles(await fg(['**/*.html'], { cwd: mirror, onlyFiles: true }));
|
||||
const records = [];
|
||||
for (const file of files) records.push(extractRecord(file, await fs.readFile(path.join(mirror, file), 'utf8')));
|
||||
const corePages = new Set(records.filter((r) => !isBlogFile(r.file)).map((r) => r.file));
|
||||
for (const record of records) {
|
||||
const isBlog = isBlogFile(record.file);
|
||||
if (!isBlog && !corePages.has(record.file)) continue;
|
||||
const slug = ['/', '/ar', '/es'].includes(record.url) ? 'index' : record.url.split('/').filter(Boolean).at(-1);
|
||||
const target = path.join(contentRoot, isBlog ? 'blog' : 'pages', record.lang, `${slug}.md`);
|
||||
await fs.writeFile(target, frontmatter(record, isBlog ? 'blog' : 'page'));
|
||||
}
|
||||
for (const lang of ['en', 'ar', 'es']) {
|
||||
await fs.writeFile(path.join(contentRoot, 'authors', lang, 'rula-diab.md'), matter.stringify('', {
|
||||
name: 'Rula Diab',
|
||||
slug: 'rula-diab',
|
||||
description: 'Founder and clinical leader at Arizona Institute for Autism.',
|
||||
avatar: '/assets/images/rula-diab-avatar.jpg',
|
||||
lang,
|
||||
translationKey: 'rula-diab'
|
||||
}));
|
||||
}
|
||||
const urlRows = ['source_file,url,language,type,title,description,h1'];
|
||||
for (const record of records) urlRows.push([
|
||||
record.file, record.url, record.lang,
|
||||
isBlogFile(record.file) ? 'blog' : 'page',
|
||||
record.title, record.description, record.h1
|
||||
].map(csv).join(','));
|
||||
await fs.writeFile(path.join(reportsRoot, 'url-inventory.csv'), `${urlRows.join('\n')}\n`);
|
||||
await copyAssets();
|
||||
console.log(`Extracted ${records.length} canonical pages and posts.`);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -1,9 +1,9 @@
|
||||
import fs from 'node:fs/promises';
|
||||
|
||||
const redirects = [
|
||||
{ from: '/aba', to: '/aba-therapy', status: 301, reason: 'Brief alias to preserved mirror URL' },
|
||||
{ from: '/autismevaluations', to: '/autism-evaluations', status: 301, reason: 'Brief alias to preserved mirror URL' },
|
||||
{ from: '/learnersocialclub', to: '/learner-social-club', status: 301, reason: 'Brief alias to preserved mirror URL' }
|
||||
{ from: '/aba', to: '/aba-therapy', status: 301, reason: 'Brief legacy alias to preserved URL' },
|
||||
{ from: '/autismevaluations', to: '/autism-evaluations', status: 301, reason: 'Brief legacy alias to preserved URL' },
|
||||
{ from: '/learnersocialclub', to: '/learner-social-club', status: 301, reason: 'Brief legacy alias to preserved URL' }
|
||||
];
|
||||
await fs.writeFile(new URL('../src/data/redirects.json', import.meta.url), `${JSON.stringify(redirects, null, 2)}\n`);
|
||||
await fs.writeFile(new URL('../reports/redirect-map.csv', import.meta.url),
|
||||
|
||||
Reference in New Issue
Block a user