diff --git a/.gitignore b/.gitignore index 7fa617a..e84420c 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,3 @@ temp/ .DS_Store Thumbs.db Desktop.ini - -# Read-only wget mirror used as migration source material -/www.azinstitute4autism.com/ diff --git a/MIGRATION_BRIEF.md b/MIGRATION_BRIEF.md index 0a755be..f847c11 100644 --- a/MIGRATION_BRIEF.md +++ b/MIGRATION_BRIEF.md @@ -383,7 +383,6 @@ www/ rtl.css tools/ - extract-site.mjs crawl-live-site.mjs audit-links.mjs generate-redirects.mjs @@ -856,39 +855,17 @@ src/pages/es/[...slug].astro Implement routing in a clean, maintainable way. -## Extraction scripts +## Migration utility scripts Create: ```txt -tools/extract-site.mjs tools/crawl-live-site.mjs tools/audit-links.mjs tools/generate-redirects.mjs tools/generate-sitemap.mjs ``` -### `extract-site.mjs` - -This script should: - -- scan the available source and generated output -- identify HTML files -- infer URL paths -- identify page type -- identify blog posts -- identify listing pages -- extract title/meta/H1/body/main content -- extract canonical and Open Graph metadata -- extract schema/JSON-LD where present -- convert blog posts to Markdown where feasible -- create page Markdown where useful -- copy/normalize assets -- rewrite local asset references -- create initial reports - -The script does not need to be perfect, but it should be useful and documented. - ### `crawl-live-site.mjs` This script should: @@ -898,7 +875,7 @@ This script should: - compare discovered URLs against the current site source and generated output - document missing URLs/assets - avoid aggressive crawling -- respect the scope of `www.azinstitute4autism.com` +- respect the live-site scope ### `audit-links.mjs` @@ -1109,7 +1086,6 @@ Add scripts similar to: "dev": "astro dev", "build": "astro check && astro build", "preview": "astro preview", - "extract": "node tools/extract-site.mjs", "crawl": "node tools/crawl-live-site.mjs", "audit:links": "node tools/audit-links.mjs", "generate:sitemap": "node tools/generate-sitemap.mjs", diff --git a/www/package.json b/www/package.json index adf80c8..7bad51b 100644 --- a/www/package.json +++ b/www/package.json @@ -10,8 +10,6 @@ "build": "astro build", "build:sandbox": "NODE_OPTIONS=--require=./tools/localhost-dns.cjs astro build", "preview": "astro preview", - "extract": "node tools/extract-fallback.mjs", - "extract:full": "node tools/extract-site.mjs", "extract:blog-footers": "node tools/extract-blog-footers.mjs", "crawl:live": "node tools/crawl-live-site.mjs", "audit:links": "node tools/audit-links.mjs", diff --git a/www/reports/cleanup-log.md b/www/reports/cleanup-log.md index e568948..6373a5b 100644 --- a/www/reports/cleanup-log.md +++ b/www/reports/cleanup-log.md @@ -1,14 +1,14 @@ # Cleanup Log - Replaced HubSpot-generated wrappers, inline styles, scripts, analytics, and modules with reusable Astro components. -- Preserved canonical clean URLs while excluding local mirror query-string and AMP duplicates. +- Preserved canonical clean URLs while excluding HubSpot query-string and AMP duplicates. - Consolidated responsive `hs-fs` image variants to canonical `hubfs` assets where available. -- Excluded mirrored HubSpot CSS and JavaScript from the new public asset package. -- Self-hosted canonical images, SVGs, fonts, and PDFs from the mirror. +- Excluded HubSpot CSS and JavaScript from the new public asset package. +- Self-hosted canonical images, SVGs, fonts, and PDFs where practical. - Recreated navigation and footer from structured JSON. - Recreated visible contact and consultation forms as static accessible HTML. - Used the source palette: AIA blue `#254080`, dark blue, straw yellow, orange, and teal. -- Used the live site's locally mirrored Playfair Display headings, Rubik body copy, and Caveat accent font. +- Used self-hosted Playfair Display headings, Rubik body copy, and Caveat accent font. - Added multilingual collection and route structure with Arabic RTL support. - Kept all extracted source content editable as Markdown. - Replaced the initial generic rounded-card visual system with live-derived @@ -21,18 +21,17 @@ - Reorganized the homepage source into named `home:` frontmatter blocks so the content is easier to inspect and edit without changing the rendered layout. - Replaced the extracted linear team article with a dedicated live-derived team grid. -- Repaired mirror-rewritten PDF, lightbox, CTA, and relative content links. +- Repaired rewritten PDF, lightbox, CTA, and relative content links. - Limited language-switcher choices to translations that have generated routes. - Verified and implemented the current production likes/views API contract. -- Recovered 18 live page-banner assets that the wget mirror missed because - HubSpot injected them through malformed inline `background-image` styles. +- Recovered 18 live page-banner assets that HubSpot injected through malformed + inline `background-image` styles. - Restored the live library banner and consultation form section background. - Added a mapped-page-imagery audit to prevent missing visual assets from silently passing the source link audit. - Removed duplicated article titles, author/date blocks, and featured images from all 65 blog Markdown bodies; these elements are rendered by the shared - blog-post layout. Updated both extractors and added a blog-content audit to - prevent recurrence. + blog-post layout. Added a blog-content audit to prevent recurrence. - Converted 19 FAQ-bearing blog posts to MDX and restored a reusable `FAQAccordion.astro` component that owns the live-style accordion and matching `FAQPage` JSON-LD. diff --git a/www/reports/migration-summary.md b/www/reports/migration-summary.md index a3acb1d..fb2f833 100644 --- a/www/reports/migration-summary.md +++ b/www/reports/migration-summary.md @@ -3,9 +3,9 @@ ## Status A maintainable Astro migration and substantial fidelity-first pass are -implemented. The live public site, not the wget mirror, was used as the -authority for the shared shell, English and Spanish homepages, service-page -family, team page, library indexes, and blog-post family. +implemented. The live public site was used as the authority for the shared +shell, English and Spanish homepages, service-page family, team page, library +indexes, and blog-post family. ## Migrated Content and Assets @@ -15,9 +15,9 @@ family, team page, library indexes, and blog-post family. - 167 self-hosted images, 20 font files, and two PDF downloads. - Current June and May 2026 English library articles are included. -The raw mirror remains untouched. HubSpot-generated wrappers, analytics, -scripts, CSS, query-language duplicates, AMP variants, pagination, and author -archive variants are not carried into the Astro implementation. +HubSpot-generated wrappers, analytics, scripts, CSS, query-language duplicates, +AMP variants, pagination, and author archive variants are not carried into the +Astro implementation. ## Live-Source Fidelity Pass @@ -55,7 +55,7 @@ Implemented fidelity work includes: links, fawn backgrounds, spacing, rounded presentation, and quote-mark SVG. - Replaced generic oversized cards, rounded controls, and marketing heroes with live-derived typography, palette, widths, spacing, and compact controls. -- Corrected material live/mirror discrepancies found during the pass, +- Corrected material live-source discrepancies found during the pass, including current homepage ESA copy, testimonial content, and ABA copy. ## URLs and Redirects @@ -157,7 +157,7 @@ final production-build proof. - Perform full-page desktop and mobile visual comparisons for page-specific layouts beyond the completed homepage, team, service, library, and article families. -- Editorially review extracted long-form content for remaining mirror artifacts, +- Editorially review extracted long-form content for remaining extraction artifacts, stale phone/email references, and heading hierarchy. - Review all source alt text, keyboard behavior, screen-reader output, and contrast with accessibility tooling. diff --git a/www/reports/redirect-map.csv b/www/reports/redirect-map.csv index 499025f..2010b19 100644 --- a/www/reports/redirect-map.csv +++ b/www/reports/redirect-map.csv @@ -1,4 +1,4 @@ from,to,status,reason -"/aba","/aba-therapy",301,"Brief alias to preserved mirror URL" -"/autismevaluations","/autism-evaluations",301,"Brief alias to preserved mirror URL" -"/learnersocialclub","/learner-social-club",301,"Brief alias to preserved mirror URL" +"/aba","/aba-therapy",301,"Brief legacy alias to preserved URL" +"/autismevaluations","/autism-evaluations",301,"Brief legacy alias to preserved URL" +"/learnersocialclub","/learner-social-club",301,"Brief legacy alias to preserved URL" diff --git a/www/reports/seo-audit.md b/www/reports/seo-audit.md index 0d00747..a55692e 100644 --- a/www/reports/seo-audit.md +++ b/www/reports/seo-audit.md @@ -18,8 +18,8 @@ ## Cleanup -- HubSpot `?hsLang=...` mirror duplicates are excluded from generated routes. -- AMP mirror variants are excluded because Astro pages are responsive and static. +- HubSpot `?hsLang=...` duplicates are excluded from generated routes. +- AMP variants are excluded because Astro pages are responsive and static. - Blog pagination and author archive variants are not preserved as separate generated pages. - Brief aliases `/aba`, `/autismevaluations`, and `/learnersocialclub` redirect to preserved source URLs. diff --git a/www/tools/extract-fallback.mjs b/www/tools/extract-fallback.mjs deleted file mode 100644 index f07453b..0000000 --- a/www/tools/extract-fallback.mjs +++ /dev/null @@ -1,220 +0,0 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); -const mirror = path.resolve(root, '../www.azinstitute4autism.com'); -const content = path.join(root, 'src/content'); -const assets = path.join(root, 'public/assets'); -const reports = path.join(root, 'reports'); -const site = 'https://www.azinstitute4autism.com'; -const mkdir = (value) => fs.mkdir(value, { recursive: true }); -const quote = (value = '') => JSON.stringify(String(value).replace(/\s+/g, ' ').trim()); -const csv = (value = '') => `"${String(value).replaceAll('"', '""')}"`; -const blogPreamble = /^# .+\n\n!\[[^\]]*\]\(\/assets\/images\/rula-diab-avatar\.jpg\)\n\n[^\n]+\n\n[^\n]+\n\n!\[[^\]]*\]\([^)]+\)\n\n/; - -async function walk(dir, prefix = '') { - const output = []; - for (const entry of await fs.readdir(dir, { withFileTypes: true })) { - const relative = path.join(prefix, entry.name); - if (entry.isDirectory()) output.push(...await walk(path.join(dir, entry.name), relative)); - else output.push(relative); - } - return output; -} - -function logical(file) { - return file.replace(/(?:\.html)?\?(?:hsLang=[^.]+|hs_amp=true)\.html$/, '.html'); -} - -function selectCanonical(files) { - const map = new Map(); - for (const file of files) { - if (!file.endsWith('.html') || file.includes('hs_amp=true') || /\/(?:page|author)\//.test(file)) continue; - const target = logical(file); - const key = `${target.startsWith('ar/') ? 'ar' : target.startsWith('es/') ? 'es' : 'en'}:${target}`; - if (!map.has(key) || (!file.includes('?') && map.get(key).includes('?'))) map.set(key, file); - } - return [...map.values()]; -} - -function decode(value = '') { - return value - .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) - .replace(/&#x([\da-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16))) - .replaceAll('&', '&').replaceAll('"', '"').replaceAll(''', "'") - .replaceAll('<', '<').replaceAll('>', '>').replaceAll(' ', ' '); -} - -function text(value = '') { - return decode(value.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim(); -} - -function normalizeHref(value = '') { - return decode(value) - .replace(/^https?:\/\/www\.azinstitute4autism\.com/, '') - .replace(/(?:\.html)?(?:%3F|\?)(?:hsLang=[^.#]+|hs_amp=true)(?:\.html)?$/, '') - .replace(/\.html$/, '') - .replace(/^index$/, '/'); -} - -function inline(value = '') { - return text(value.replace(/]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, href, label) => { - const cleanLabel = text(label); - return cleanLabel ? `[${cleanLabel}](${normalizeHref(href)})` : ''; - })); -} - -function meta(html, name, property = false) { - const key = property ? 'property' : 'name'; - const pattern = new RegExp(`]+${key}=["']${name}["'][^>]+content=["']([^"']*)`, 'i'); - const reverse = new RegExp(`]+content=["']([^"']*)["'][^>]+${key}=["']${name}["']`, 'i'); - return decode(html.match(pattern)?.[1] || html.match(reverse)?.[1] || ''); -} - -function asset(value = '') { - const match = value.match(/(?:hs-fs\/)?hubfs\/([^?#]+)/); - return match ? `/assets/images/${path.basename(decodeURIComponent(match[1])).split('?')[0]}` : undefined; -} - -function normalizeTable(value = '') { - return value - .replace(/(]*\bhref=["'])([^"']+)(["'])/gi, - (_, prefix, href, suffix) => `${prefix}${normalizeHref(href)}${suffix}`) - .replace(/(]*\bsrc=["'])([^"']+)(["'])/gi, - (_, prefix, src, suffix) => `${prefix}${asset(src) || src}${suffix}`); -} - -function markdownFrom(html) { - let body = html.match(//i)?.[0] || - html.match(/blog-post__body[\s\S]*?(?=)/i)?.[0] || - ''; - const tables = []; - body = body - .replace(/<(script|style|noscript|header|footer|nav|form)\b[\s\S]*?<\/\1>/gi, '') - .replace(//gi, (match) => { - const token = `__TABLE_${tables.length}__`; - tables.push(normalizeTable(match)); - return `\n\n${token}\n\n`; - }) - .replace(/]*>([\s\S]*?)<\/blockquote>/gi, (_, value) => `\n\n> ${inline(value)}\n\n`) - .replace(/]+src=["']([^"']+)["'][^>]*alt=["']([^"']*)["'][^>]*>/gi, (_, src, alt) => { - const local = asset(src); - return local ? `\n\n![${text(alt)}](${local})\n\n` : ''; - }) - .replace(/]*>([\s\S]*?)<\/h\1>/gi, (_, level, value) => `\n\n${'#'.repeat(Number(level))} ${inline(value)}\n\n`) - .replace(/]*>([\s\S]*?)<\/li>/gi, (_, value) => `\n- ${inline(value)}`) - .replace(/]*>([\s\S]*?)<\/p>/gi, (_, value) => `\n\n${inline(value)}\n\n`) - .replace(//gi, '\n') - .replace(/<[^>]+>/g, ' '); - body = decode(body).replace(/[ \t]+/g, ' ').replace(/^\s+$/gm, '').replace(/\n{3,}/g, '\n\n').trim(); - return tables.reduce((output, table, index) => output.replaceAll(`__TABLE_${index}__`, table), body); -} - -function record(file, html) { - const target = logical(file); - const lang = target.startsWith('ar/') ? 'ar' : target.startsWith('es/') ? 'es' : 'en'; - const url = target === 'index.html' ? '/' : `/${target.replace(/\/index\.html$/, '').replace(/\.html$/, '')}`; - const title = text(html.match(/]*>([\s\S]*?)<\/title>/i)?.[1] || url); - const h1 = text(html.match(/]*>([\s\S]*?)<\/h1>/i)?.[1] || title); - const published = meta(html, 'article:published_time', true) || - html.match(/]+datetime=["']([^"']+)/i)?.[1] || ''; - return { - file, lang, url, title, h1, - description: meta(html, 'description'), - image: asset(meta(html, 'og:image', true)), - date: /^\d{4}-\d{2}-\d{2}/.test(published) ? published.slice(0, 10) : '2024-01-01', - markdown: markdownFrom(html) - }; -} - -function frontmatter(item, blog) { - const slug = ['/', '/ar', '/es'].includes(item.url) ? 'index' : item.url.split('/').filter(Boolean).at(-1); - const lines = [ - '---', - `title: ${quote(item.title)}`, - `description: ${quote(item.description)}`, - `slug: ${quote(slug)}`, - `canonical: ${quote(`${site}${item.url}`)}`, - `lang: ${quote(item.lang)}`, - `translationKey: ${quote(slug)}`, - item.image ? `featuredImage: ${quote(item.image)}` : '', - blog ? `date: "${item.date}"` : '', - blog ? 'author: "rula-diab"' : '', - blog ? 'category: "Library"' : '', - blog ? 'tags: []' : '', - 'draft: false', - '---', - '', - (blog ? item.markdown.replace(blogPreamble, '') : item.markdown) || `# ${item.h1}`, - '' - ].filter((line) => line !== ''); - return `${lines.join('\n')}\n`; -} - -async function copySourceAssets(files) { - const rows = ['source,target,size,kind']; - const seen = new Set(); - await Promise.all(['images', 'fonts', 'downloads'].map((kind) => fs.rm(path.join(assets, kind), { recursive: true, force: true }))); - for (const file of files.filter((name) => name.startsWith('hubfs/') || name.startsWith('hs-fs/hubfs/') || name.startsWith('_hcms/googlefonts/'))) { - const font = file.startsWith('_hcms/googlefonts/'); - const raw = font ? file.replace('_hcms/googlefonts/', '') : path.basename(file).split('?')[0]; - if (!font && !/\.(?:avif|gif|jpe?g|png|svg|webp|pdf)$/i.test(raw)) continue; - const key = `${font}:${raw}`; - if (!raw || seen.has(key)) continue; - seen.add(key); - const kind = font ? 'fonts' : raw.endsWith('.pdf') ? 'downloads' : 'images'; - const target = path.join(assets, kind, raw); - await mkdir(path.dirname(target)); - await fs.copyFile(path.join(mirror, file), target); - const size = (await fs.stat(target)).size; - rows.push([file, path.relative(root, target), size, kind].map(csv).join(',')); - } - await fs.writeFile(path.join(reports, 'asset-inventory.csv'), `${rows.join('\n')}\n`); -} - -async function removeMissingFeaturedImages() { - for (const type of ['pages', 'blog']) { - for (const file of await walk(path.join(content, type))) { - const target = path.join(content, type, file); - let source = await fs.readFile(target, 'utf8'); - const image = source.match(/^featuredImage:\s*"\/assets\/images\/([^"]+)"/m)?.[1]; - if (!image) continue; - if (!await fs.access(path.join(assets, 'images', image)).then(() => true).catch(() => false)) { - source = source.replace(/^featuredImage:.*\n/m, ''); - await fs.writeFile(target, source); - } - } - } -} - -async function main() { - const files = await walk(mirror); - const selected = selectCanonical(files); - const records = []; - for (const file of selected) records.push(record(file, await fs.readFile(path.join(mirror, file), 'utf8'))); - await Promise.all(['pages', 'blog', 'authors'].map((type) => fs.rm(path.join(content, type), { recursive: true, force: true }))); - for (const lang of ['en', 'ar', 'es']) { - for (const type of ['pages', 'blog', 'authors']) await mkdir(path.join(content, type, lang)); - } - for (const item of records) { - const blog = item.url.includes('/library/') && !item.url.endsWith('/library'); - const slug = ['/', '/ar', '/es'].includes(item.url) ? 'index' : item.url.split('/').filter(Boolean).at(-1); - await fs.writeFile(path.join(content, blog ? 'blog' : 'pages', item.lang, `${slug}.md`), frontmatter(item, blog)); - } - for (const lang of ['en', 'ar', 'es']) { - await fs.writeFile(path.join(content, 'authors', lang, 'rula-diab.md'), `---\nname: "Rula Diab"\nslug: "rula-diab"\ndescription: "Founder and clinical leader at Arizona Institute for Autism."\navatar: "/assets/images/rula-diab-avatar.jpg"\nlang: "${lang}"\ntranslationKey: "rula-diab"\n---\n`); - } - if (!records.some((item) => item.lang === 'ar' && item.url === '/ar')) { - await fs.writeFile(path.join(content, 'pages/ar/index.md'), `---\ntitle: "معهد أريزونا للتوحد"\ndescription: "صفحة عربية تمهيدية لمعهد أريزونا للتوحد."\nslug: "index"\ncanonical: "https://www.azinstitute4autism.com/ar"\nlang: "ar"\ntranslationKey: "home"\ndraft: false\n---\n\n\n\nتتوفر المقالات العربية الحالية في المكتبة. يرجى مراجعة المحتوى العربي قبل النشر.\n`); - } - const rows = ['source_file,url,language,type,title,description,h1']; - for (const item of records) rows.push([item.file, item.url, item.lang, item.url.includes('/library/') ? 'blog' : 'page', item.title, item.description, item.h1].map(csv).join(',')); - await mkdir(reports); - await fs.writeFile(path.join(reports, 'url-inventory.csv'), `${rows.join('\n')}\n`); - await copySourceAssets(files); - await removeMissingFeaturedImages(); - console.log(`Fallback extraction completed: ${records.length} canonical records.`); -} - -await main(); diff --git a/www/tools/extract-site.mjs b/www/tools/extract-site.mjs deleted file mode 100644 index ebac68b..0000000 --- a/www/tools/extract-site.mjs +++ /dev/null @@ -1,198 +0,0 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import fg from 'fast-glob'; -import { load } from 'cheerio'; -import TurndownService from 'turndown'; -import matter from 'gray-matter'; - -const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); -const mirror = path.resolve(root, '../www.azinstitute4autism.com'); -const site = 'https://www.azinstitute4autism.com'; -const contentRoot = path.join(root, 'src/content'); -const publicRoot = path.join(root, 'public/assets'); -const reportsRoot = path.join(root, 'reports'); -const turndown = new TurndownService({ headingStyle: 'atx', bulletListMarker: '-' }); -turndown.remove(['script', 'style', 'noscript', 'iframe']); - -const mkdir = (dir) => fs.mkdir(dir, { recursive: true }); -const clean = (value = '') => value.replace(/\s+/g, ' ').trim(); -const csv = (value = '') => `"${String(value).replaceAll('"', '""')}"`; -const yaml = (value = '') => JSON.stringify(String(value)); -const blogPreamble = /^# .+\n\n!\[[^\]]*\]\(\/assets\/images\/rula-diab-avatar\.jpg\)\n\n[^\n]+\n\n[^\n]+\n\n!\[[^\]]*\]\([^)]+\)\n\n/; - -function contentMarkdown(markdown, isBlog) { - return isBlog ? markdown.replace(blogPreamble, '') : markdown; -} - -function logicalFile(file) { - return file.replace(/(?:\.html)?\?(?:hsLang=[^.]+|hs_amp=true)\.html$/, '.html'); -} - -function canonicalFiles(files) { - const selected = new Map(); - for (const file of files) { - if (!file.endsWith('.html') || /\/(?:page|author)\//.test(file) || file.includes('hs_amp=true')) continue; - const logical = logicalFile(file); - const lang = logical.startsWith('ar/') ? 'ar' : logical.startsWith('es/') ? 'es' : 'en'; - const key = `${lang}:${logical}`; - const current = selected.get(key); - if (!current || (!file.includes('?') && current.includes('?'))) selected.set(key, file); - } - return [...selected.values()]; -} - -function sourceUrl(file) { - const logical = logicalFile(file); - if (logical === 'index.html') return '/'; - return `/${logical.replace(/\/index\.html$/, '').replace(/\.html$/, '')}`; -} - -function isBlogFile(file) { - return file.startsWith('library/') || file.includes('/library/'); -} - -function languageFor(file, $) { - if (file.startsWith('ar/')) return 'ar'; - if (file.startsWith('es/')) return 'es'; - return $('html').attr('lang')?.split('-')[0] || 'en'; -} - -function localizeAsset(value = '') { - const decoded = value.replace(/^https?:\/\/[^/]+/, '').replace(/^\.\.\//, '/'); - const match = decoded.match(/(?:\/)?(?:hs-fs\/)?hubfs\/([^?#]+)/); - if (!match) return undefined; - return `/assets/images/${path.basename(decodeURIComponent(match[1])).split('?')[0]}`; -} - -function extractRecord(file, html) { - const $ = load(html); - const lang = languageFor(file, $); - const url = sourceUrl(file); - const title = clean($('title').first().text()) || clean($('h1').first().text()) || url; - const description = $('meta[name="description"]').attr('content') || ''; - const h1 = clean($('h1').first().text()); - const image = localizeAsset( - $('meta[property="og:image"]').attr('content') || - $('.blog-post__body img, main img, .body-container-wrapper img').first().attr('src') || - '' - ); - const alt = clean($('.blog-post__body img, main img, .body-container-wrapper img').first().attr('alt') || ''); - const dateText = $('meta[property="article:published_time"]').attr('content') || - $('time').first().attr('datetime') || '2024-01-01'; - const date = /^\d{4}-\d{2}-\d{2}/.test(dateText) ? dateText.slice(0, 10) : '2024-01-01'; - const selector = isBlogFile(file) ? '.blog-post__body' : 'main, .body-container-wrapper'; - const body = $(selector).first().clone(); - body.find('header, footer, nav, form, script, style, noscript, .hs_cos_wrapper_type_form').remove(); - body.find('*').removeAttr('style').removeAttr('id').removeAttr('data-hs-cos-general-type').removeAttr('data-hs-cos-type'); - body.find('a').each((_, element) => { - const href = $(element).attr('href'); - if (!href) return; - $(element).attr('href', href - .replace(/\.html(?:%3F|\?)[^"]*$/, '') - .replace(/\.html$/, '') - .replace(/^index$/, '/')); - }); - body.find('img').each((_, element) => { - const src = localizeAsset($(element).attr('src')); - if (src) $(element).attr('src', src); - }); - const tables = []; - body.find('table').each((_, element) => { - const token = `__TABLE_${tables.length}__`; - tables.push($(element).prop('outerHTML')); - $(element).replaceWith(`\n\n${token}\n\n`); - }); - const markdown = contentMarkdown( - tables.reduce((output, table, index) => output.replaceAll(`__TABLE_${index}__`, table), - turndown.turndown(body.html() || '').replace(/\n{3,}/g, '\n\n').trim()), - isBlogFile(file) - ); - return { file, lang, url, title, description, h1, image, alt, date, markdown }; -} - -function frontmatter(record, type) { - const slug = ['/', '/ar', '/es'].includes(record.url) ? 'index' : record.url.split('/').filter(Boolean).at(-1); - const data = { - title: record.title, - description: record.description, - slug, - canonical: `${site}${record.url}`, - lang: record.lang, - translationKey: slug, - draft: false - }; - if (record.image) data.featuredImage = record.image; - if (record.alt) data.alt = record.alt; - if (type === 'blog') Object.assign(data, { - date: record.date, - author: 'rula-diab', - category: 'Library', - tags: [] - }); - return matter.stringify(record.markdown || `# ${record.h1 || record.title}\n`, data); -} - -async function copyAssets() { - const assets = await fg(['hubfs/*', 'hs-fs/hubfs/*', '_hcms/googlefonts/**/*'], { cwd: mirror, onlyFiles: true }); - const inventory = ['source,target,size,kind']; - const seen = new Set(); - await Promise.all(['images', 'fonts', 'downloads'].map((kind) => fs.rm(path.join(publicRoot, kind), { recursive: true, force: true }))); - for (const source of assets) { - const original = path.join(mirror, source); - const stat = await fs.stat(original); - const isFont = source.startsWith('_hcms/googlefonts/'); - const rawName = isFont ? source.replace('_hcms/googlefonts/', '') : path.basename(source).split('?')[0]; - if (!isFont && !/\.(?:avif|gif|jpe?g|png|svg|webp|pdf)$/i.test(rawName)) continue; - if (!rawName || seen.has(`${isFont}:${rawName}`)) continue; - seen.add(`${isFont}:${rawName}`); - const target = path.join(publicRoot, isFont ? 'fonts' : rawName.endsWith('.pdf') ? 'downloads' : 'images', rawName); - await mkdir(path.dirname(target)); - await fs.copyFile(original, target); - inventory.push([source, path.relative(root, target), stat.size, isFont ? 'font' : path.extname(rawName).slice(1)].map(csv).join(',')); - } - await fs.writeFile(path.join(reportsRoot, 'asset-inventory.csv'), `${inventory.join('\n')}\n`); -} - -async function main() { - await Promise.all([ - mkdir(contentRoot), mkdir(publicRoot), mkdir(reportsRoot), - ...['en', 'ar', 'es'].flatMap((lang) => [ - mkdir(path.join(contentRoot, 'pages', lang)), - mkdir(path.join(contentRoot, 'blog', lang)), - mkdir(path.join(contentRoot, 'authors', lang)) - ]) - ]); - const files = canonicalFiles(await fg(['**/*.html'], { cwd: mirror, onlyFiles: true })); - const records = []; - for (const file of files) records.push(extractRecord(file, await fs.readFile(path.join(mirror, file), 'utf8'))); - const corePages = new Set(records.filter((r) => !isBlogFile(r.file)).map((r) => r.file)); - for (const record of records) { - const isBlog = isBlogFile(record.file); - if (!isBlog && !corePages.has(record.file)) continue; - const slug = ['/', '/ar', '/es'].includes(record.url) ? 'index' : record.url.split('/').filter(Boolean).at(-1); - const target = path.join(contentRoot, isBlog ? 'blog' : 'pages', record.lang, `${slug}.md`); - await fs.writeFile(target, frontmatter(record, isBlog ? 'blog' : 'page')); - } - for (const lang of ['en', 'ar', 'es']) { - await fs.writeFile(path.join(contentRoot, 'authors', lang, 'rula-diab.md'), matter.stringify('', { - name: 'Rula Diab', - slug: 'rula-diab', - description: 'Founder and clinical leader at Arizona Institute for Autism.', - avatar: '/assets/images/rula-diab-avatar.jpg', - lang, - translationKey: 'rula-diab' - })); - } - const urlRows = ['source_file,url,language,type,title,description,h1']; - for (const record of records) urlRows.push([ - record.file, record.url, record.lang, - isBlogFile(record.file) ? 'blog' : 'page', - record.title, record.description, record.h1 - ].map(csv).join(',')); - await fs.writeFile(path.join(reportsRoot, 'url-inventory.csv'), `${urlRows.join('\n')}\n`); - await copyAssets(); - console.log(`Extracted ${records.length} canonical pages and posts.`); -} - -await main(); diff --git a/www/tools/generate-redirects.mjs b/www/tools/generate-redirects.mjs index 1f1dbf4..2359ea9 100644 --- a/www/tools/generate-redirects.mjs +++ b/www/tools/generate-redirects.mjs @@ -1,9 +1,9 @@ import fs from 'node:fs/promises'; const redirects = [ - { from: '/aba', to: '/aba-therapy', status: 301, reason: 'Brief alias to preserved mirror URL' }, - { from: '/autismevaluations', to: '/autism-evaluations', status: 301, reason: 'Brief alias to preserved mirror URL' }, - { from: '/learnersocialclub', to: '/learner-social-club', status: 301, reason: 'Brief alias to preserved mirror URL' } + { from: '/aba', to: '/aba-therapy', status: 301, reason: 'Brief legacy alias to preserved URL' }, + { from: '/autismevaluations', to: '/autism-evaluations', status: 301, reason: 'Brief legacy alias to preserved URL' }, + { from: '/learnersocialclub', to: '/learner-social-club', status: 301, reason: 'Brief legacy alias to preserved URL' } ]; await fs.writeFile(new URL('../src/data/redirects.json', import.meta.url), `${JSON.stringify(redirects, null, 2)}\n`); await fs.writeFile(new URL('../reports/redirect-map.csv', import.meta.url),