refactor(migration): remove obsolete mirror extraction tooling

- delete extraction scripts that depended on the removed local mirror
- remove stale extract npm scripts and migration-brief references
- drop the old mirror ignore rule from .gitignore
- update reports and redirect reasons to use live-source wording

.gitignore

@@ -58,6 +58,3 @@ temp/
 .DS_Store
 Thumbs.db
 Desktop.ini
-
-# Read-only wget mirror used as migration source material
-/www.azinstitute4autism.com/

MIGRATION_BRIEF.md

@@ -383,7 +383,6 @@ www/
       rtl.css
 
   tools/
-    extract-site.mjs
     crawl-live-site.mjs
     audit-links.mjs
     generate-redirects.mjs
@@ -856,39 +855,17 @@ src/pages/es/[...slug].astro
 
 Implement routing in a clean, maintainable way.
 
-## Extraction scripts
+## Migration utility scripts
 
 Create:
 
 ```txt
-tools/extract-site.mjs
 tools/crawl-live-site.mjs
 tools/audit-links.mjs
 tools/generate-redirects.mjs
 tools/generate-sitemap.mjs
 ```
 
-### `extract-site.mjs`
-
-This script should:
-
-- scan the available source and generated output
-- identify HTML files
-- infer URL paths
-- identify page type
-- identify blog posts
-- identify listing pages
-- extract title/meta/H1/body/main content
-- extract canonical and Open Graph metadata
-- extract schema/JSON-LD where present
-- convert blog posts to Markdown where feasible
-- create page Markdown where useful
-- copy/normalize assets
-- rewrite local asset references
-- create initial reports
-
-The script does not need to be perfect, but it should be useful and documented.
-
 ### `crawl-live-site.mjs`
 
 This script should:
@@ -898,7 +875,7 @@ This script should:
 - compare discovered URLs against the current site source and generated output
 - document missing URLs/assets
 - avoid aggressive crawling
-- respect the scope of `www.azinstitute4autism.com`
+- respect the live-site scope
 
 ### `audit-links.mjs`
 
@@ -1109,7 +1086,6 @@ Add scripts similar to:
     "dev": "astro dev",
     "build": "astro check && astro build",
     "preview": "astro preview",
-    "extract": "node tools/extract-site.mjs",
     "crawl": "node tools/crawl-live-site.mjs",
     "audit:links": "node tools/audit-links.mjs",
     "generate:sitemap": "node tools/generate-sitemap.mjs",

www/.env.example

@@ -1 +1,2 @@
 PUBLIC_AIA_API_BASE=https://api.azinstitute4autism.com
+PUBLIC_ALLOW_INDEXING=false

www/README.md

@@ -38,19 +38,23 @@ cd www
 npm install
 ```
 
-Copy the example environment file when testing the like/view counter:
+Copy the example environment file when testing the like/view counter or staging
+SEO behavior:
 
 ```sh
 cp .env.example .env
 ```
 
-The default value is:
+The default values are:
 
 ```txt
 PUBLIC_AIA_API_BASE=https://api.azinstitute4autism.com
+PUBLIC_ALLOW_INDEXING=false
 ```
 
-The site still renders when that API is unavailable.
+The site still renders when the AIA API is unavailable. Migration and staging
+builds emit `noindex,nofollow` by default; set `PUBLIC_ALLOW_INDEXING=true`
+only for production builds on the canonical domain.
 
 ## View And Build
 

www/package.json

@@ -10,8 +10,6 @@
     "build": "astro build",
     "build:sandbox": "NODE_OPTIONS=--require=./tools/localhost-dns.cjs astro build",
     "preview": "astro preview",
-    "extract": "node tools/extract-fallback.mjs",
-    "extract:full": "node tools/extract-site.mjs",
     "extract:blog-footers": "node tools/extract-blog-footers.mjs",
     "crawl:live": "node tools/crawl-live-site.mjs",
     "audit:links": "node tools/audit-links.mjs",

www/public/robots.txt

@@ -1,3 +0,0 @@
-User-agent: *
-Allow: /
-Sitemap: https://www.azinstitute4autism.com/sitemap.xml

www/reports/cleanup-log.md

@@ -1,14 +1,14 @@
 # Cleanup Log
 
 - Replaced HubSpot-generated wrappers, inline styles, scripts, analytics, and modules with reusable Astro components.
-- Preserved canonical clean URLs while excluding local mirror query-string and AMP duplicates.
+- Preserved canonical clean URLs while excluding HubSpot query-string and AMP duplicates.
 - Consolidated responsive `hs-fs` image variants to canonical `hubfs` assets where available.
-- Excluded mirrored HubSpot CSS and JavaScript from the new public asset package.
+- Excluded HubSpot CSS and JavaScript from the new public asset package.
-- Self-hosted canonical images, SVGs, fonts, and PDFs from the mirror.
+- Self-hosted canonical images, SVGs, fonts, and PDFs where practical.
 - Recreated navigation and footer from structured JSON.
 - Recreated visible contact and consultation forms as static accessible HTML.
 - Used the source palette: AIA blue `#254080`, dark blue, straw yellow, orange, and teal.
-- Used the live site's locally mirrored Playfair Display headings, Rubik body copy, and Caveat accent font.
+- Used self-hosted Playfair Display headings, Rubik body copy, and Caveat accent font.
 - Added multilingual collection and route structure with Arabic RTL support.
 - Kept all extracted source content editable as Markdown.
 - Replaced the initial generic rounded-card visual system with live-derived
@@ -21,18 +21,17 @@
 - Reorganized the homepage source into named `home:` frontmatter blocks so the
   content is easier to inspect and edit without changing the rendered layout.
 - Replaced the extracted linear team article with a dedicated live-derived team grid.
-- Repaired mirror-rewritten PDF, lightbox, CTA, and relative content links.
+- Repaired rewritten PDF, lightbox, CTA, and relative content links.
 - Limited language-switcher choices to translations that have generated routes.
 - Verified and implemented the current production likes/views API contract.
-- Recovered 18 live page-banner assets that the wget mirror missed because
+- Recovered 18 live page-banner assets that HubSpot injected through malformed
-  HubSpot injected them through malformed inline `background-image` styles.
+  inline `background-image` styles.
 - Restored the live library banner and consultation form section background.
 - Added a mapped-page-imagery audit to prevent missing visual assets from
   silently passing the source link audit.
 - Removed duplicated article titles, author/date blocks, and featured images
   from all 65 blog Markdown bodies; these elements are rendered by the shared
-  blog-post layout. Updated both extractors and added a blog-content audit to
+  blog-post layout. Added a blog-content audit to prevent recurrence.
-  prevent recurrence.
 - Converted 19 FAQ-bearing blog posts to MDX and restored a reusable
   `FAQAccordion.astro` component that owns the live-style accordion and
   matching `FAQPage` JSON-LD.

www/reports/migration-summary.md

@@ -3,9 +3,9 @@
 ## Status
 
 A maintainable Astro migration and substantial fidelity-first pass are
-implemented. The live public site, not the wget mirror, was used as the
+implemented. The live public site was used as the authority for the shared
-authority for the shared shell, English and Spanish homepages, service-page
+shell, English and Spanish homepages, service-page family, team page, library
-family, team page, library indexes, and blog-post family.
+indexes, and blog-post family.
 
 ## Migrated Content and Assets
 
@@ -15,9 +15,9 @@ family, team page, library indexes, and blog-post family.
 - 167 self-hosted images, 20 font files, and two PDF downloads.
 - Current June and May 2026 English library articles are included.
 
-The raw mirror remains untouched. HubSpot-generated wrappers, analytics,
+HubSpot-generated wrappers, analytics, scripts, CSS, query-language duplicates,
-scripts, CSS, query-language duplicates, AMP variants, pagination, and author
+AMP variants, pagination, and author archive variants are not carried into the
-archive variants are not carried into the Astro implementation.
+Astro implementation.
 
 ## Live-Source Fidelity Pass
 
@@ -55,7 +55,7 @@ Implemented fidelity work includes:
   links, fawn backgrounds, spacing, rounded presentation, and quote-mark SVG.
 - Replaced generic oversized cards, rounded controls, and marketing heroes
   with live-derived typography, palette, widths, spacing, and compact controls.
-- Corrected material live/mirror discrepancies found during the pass,
+- Corrected material live-source discrepancies found during the pass,
   including current homepage ESA copy, testimonial content, and ABA copy.
 
 ## URLs and Redirects
@@ -157,7 +157,7 @@ final production-build proof.
 - Perform full-page desktop and mobile visual comparisons for page-specific
   layouts beyond the completed homepage, team, service, library, and article
   families.
-- Editorially review extracted long-form content for remaining mirror artifacts,
+- Editorially review extracted long-form content for remaining extraction artifacts,
   stale phone/email references, and heading hierarchy.
 - Review all source alt text, keyboard behavior, screen-reader output, and
   contrast with accessibility tooling.

www/reports/redirect-map.csv

@@ -1,4 +1,4 @@
 from,to,status,reason
-"/aba","/aba-therapy",301,"Brief alias to preserved mirror URL"
+"/aba","/aba-therapy",301,"Brief legacy alias to preserved URL"
-"/autismevaluations","/autism-evaluations",301,"Brief alias to preserved mirror URL"
+"/autismevaluations","/autism-evaluations",301,"Brief legacy alias to preserved URL"
-"/learnersocialclub","/learner-social-club",301,"Brief alias to preserved mirror URL"
+"/learnersocialclub","/learner-social-club",301,"Brief legacy alias to preserved URL"

www/reports/seo-audit.md

@@ -18,8 +18,8 @@
 
 ## Cleanup
 
-- HubSpot `?hsLang=...` mirror duplicates are excluded from generated routes.
+- HubSpot `?hsLang=...` duplicates are excluded from generated routes.
-- AMP mirror variants are excluded because Astro pages are responsive and static.
+- AMP variants are excluded because Astro pages are responsive and static.
 - Blog pagination and author archive variants are not preserved as separate generated pages.
 - Brief aliases `/aba`, `/autismevaluations`, and `/learnersocialclub` redirect to preserved source URLs.
 

www/src/components/Seo.astro

@@ -6,12 +6,23 @@ interface Props {
   image?: string;
   lang?: string;
   type?: 'website' | 'article';
+  robots?: string;
 }
-const { title, description = '', canonical, image, lang = 'en', type = 'website' } = Astro.props;
+const {
+  title,
+  description = '',
+  canonical,
+  image,
+  lang = 'en',
+  type = 'website',
+  robots
+} = Astro.props;
+const defaultRobots = import.meta.env.PUBLIC_ALLOW_INDEXING === 'true' ? 'index,follow' : 'noindex,nofollow';
 const absoluteImage = image ? new URL(image, Astro.site ?? Astro.url).href : undefined;
 ---
 <title>{title}</title>
 <meta name="description" content={description} />
+<meta name="robots" content={robots ?? defaultRobots} />
 {canonical && <link rel="canonical" href={canonical} />}
 <meta property="og:type" content={type} />
 <meta property="og:title" content={title} />

www/src/pages/robots.txt.ts

@@ -0,0 +1,23 @@
+const allowIndexing = import.meta.env.PUBLIC_ALLOW_INDEXING === 'true';
+
+const body = allowIndexing
+  ? [
+      'User-agent: *',
+      'Allow: /',
+      'Sitemap: https://www.azinstitute4autism.com/sitemap.xml',
+      ''
+    ].join('\n')
+  : [
+      'User-agent: *',
+      'Allow: /',
+      '# Staging and migration builds emit meta robots noindex,nofollow.',
+      ''
+    ].join('\n');
+
+export function GET() {
+  return new Response(body, {
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8'
+    }
+  });
+}

www/tools/extract-fallback.mjs

@@ -1,220 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
-const mirror = path.resolve(root, '../www.azinstitute4autism.com');
-const content = path.join(root, 'src/content');
-const assets = path.join(root, 'public/assets');
-const reports = path.join(root, 'reports');
-const site = 'https://www.azinstitute4autism.com';
-const mkdir = (value) => fs.mkdir(value, { recursive: true });
-const quote = (value = '') => JSON.stringify(String(value).replace(/\s+/g, ' ').trim());
-const csv = (value = '') => `"${String(value).replaceAll('"', '""')}"`;
-const blogPreamble = /^# .+\n\n!\[[^\]]*\]\(\/assets\/images\/rula-diab-avatar\.jpg\)\n\n[^\n]+\n\n[^\n]+\n\n!\[[^\]]*\]\([^)]+\)\n\n/;
-
-async function walk(dir, prefix = '') {
-  const output = [];
-  for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
-    const relative = path.join(prefix, entry.name);
-    if (entry.isDirectory()) output.push(...await walk(path.join(dir, entry.name), relative));
-    else output.push(relative);
-  }
-  return output;
-}
-
-function logical(file) {
-  return file.replace(/(?:\.html)?\?(?:hsLang=[^.]+|hs_amp=true)\.html$/, '.html');
-}
-
-function selectCanonical(files) {
-  const map = new Map();
-  for (const file of files) {
-    if (!file.endsWith('.html') || file.includes('hs_amp=true') || /\/(?:page|author)\//.test(file)) continue;
-    const target = logical(file);
-    const key = `${target.startsWith('ar/') ? 'ar' : target.startsWith('es/') ? 'es' : 'en'}:${target}`;
-    if (!map.has(key) || (!file.includes('?') && map.get(key).includes('?'))) map.set(key, file);
-  }
-  return [...map.values()];
-}
-
-function decode(value = '') {
-  return value
-    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
-    .replace(/&#x([\da-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)))
-    .replaceAll('&', '&').replaceAll('"', '"').replaceAll(''', "'")
-    .replaceAll('&lt;', '<').replaceAll('&gt;', '>').replaceAll('&nbsp;', ' ');
-}
-
-function text(value = '') {
-  return decode(value.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();
-}
-
-function normalizeHref(value = '') {
-  return decode(value)
-    .replace(/^https?:\/\/www\.azinstitute4autism\.com/, '')
-    .replace(/(?:\.html)?(?:%3F|\?)(?:hsLang=[^.#]+|hs_amp=true)(?:\.html)?$/, '')
-    .replace(/\.html$/, '')
-    .replace(/^index$/, '/');
-}
-
-function inline(value = '') {
-  return text(value.replace(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, href, label) => {
-    const cleanLabel = text(label);
-    return cleanLabel ? `[${cleanLabel}](${normalizeHref(href)})` : '';
-  }));
-}
-
-function meta(html, name, property = false) {
-  const key = property ? 'property' : 'name';
-  const pattern = new RegExp(`<meta[^>]+${key}=["']${name}["'][^>]+content=["']([^"']*)`, 'i');
-  const reverse = new RegExp(`<meta[^>]+content=["']([^"']*)["'][^>]+${key}=["']${name}["']`, 'i');
-  return decode(html.match(pattern)?.[1] || html.match(reverse)?.[1] || '');
-}
-
-function asset(value = '') {
-  const match = value.match(/(?:hs-fs\/)?hubfs\/([^?#]+)/);
-  return match ? `/assets/images/${path.basename(decodeURIComponent(match[1])).split('?')[0]}` : undefined;
-}
-
-function normalizeTable(value = '') {
-  return value
-    .replace(/(<a\b[^>]*\bhref=["'])([^"']+)(["'])/gi,
-      (_, prefix, href, suffix) => `${prefix}${normalizeHref(href)}${suffix}`)
-    .replace(/(<img\b[^>]*\bsrc=["'])([^"']+)(["'])/gi,
-      (_, prefix, src, suffix) => `${prefix}${asset(src) || src}${suffix}`);
-}
-
-function markdownFrom(html) {
-  let body = html.match(/<main[\s\S]*?<\/main>/i)?.[0] ||
-    html.match(/blog-post__body[\s\S]*?(?=<footer|blog-post__tags|<\/article>)/i)?.[0] ||
-    '';
-  const tables = [];
-  body = body
-    .replace(/<(script|style|noscript|header|footer|nav|form)\b[\s\S]*?<\/\1>/gi, '')
-    .replace(/<table[\s\S]*?<\/table>/gi, (match) => {
-      const token = `__TABLE_${tables.length}__`;
-      tables.push(normalizeTable(match));
-      return `\n\n${token}\n\n`;
-    })
-    .replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, value) => `\n\n> ${inline(value)}\n\n`)
-    .replace(/<img[^>]+src=["']([^"']+)["'][^>]*alt=["']([^"']*)["'][^>]*>/gi, (_, src, alt) => {
-      const local = asset(src);
-      return local ? `\n\n![${text(alt)}](${local})\n\n` : '';
-    })
-    .replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, level, value) => `\n\n${'#'.repeat(Number(level))} ${inline(value)}\n\n`)
-    .replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_, value) => `\n- ${inline(value)}`)
-    .replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, value) => `\n\n${inline(value)}\n\n`)
-    .replace(/<br\s*\/?>/gi, '\n')
-    .replace(/<[^>]+>/g, ' ');
-  body = decode(body).replace(/[ \t]+/g, ' ').replace(/^\s+$/gm, '').replace(/\n{3,}/g, '\n\n').trim();
-  return tables.reduce((output, table, index) => output.replaceAll(`__TABLE_${index}__`, table), body);
-}
-
-function record(file, html) {
-  const target = logical(file);
-  const lang = target.startsWith('ar/') ? 'ar' : target.startsWith('es/') ? 'es' : 'en';
-  const url = target === 'index.html' ? '/' : `/${target.replace(/\/index\.html$/, '').replace(/\.html$/, '')}`;
-  const title = text(html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || url);
-  const h1 = text(html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || title);
-  const published = meta(html, 'article:published_time', true) ||
-    html.match(/<time[^>]+datetime=["']([^"']+)/i)?.[1] || '';
-  return {
-    file, lang, url, title, h1,
-    description: meta(html, 'description'),
-    image: asset(meta(html, 'og:image', true)),
-    date: /^\d{4}-\d{2}-\d{2}/.test(published) ? published.slice(0, 10) : '2024-01-01',
-    markdown: markdownFrom(html)
-  };
-}
-
-function frontmatter(item, blog) {
-  const slug = ['/', '/ar', '/es'].includes(item.url) ? 'index' : item.url.split('/').filter(Boolean).at(-1);
-  const lines = [
-    '---',
-    `title: ${quote(item.title)}`,
-    `description: ${quote(item.description)}`,
-    `slug: ${quote(slug)}`,
-    `canonical: ${quote(`${site}${item.url}`)}`,
-    `lang: ${quote(item.lang)}`,
-    `translationKey: ${quote(slug)}`,
-    item.image ? `featuredImage: ${quote(item.image)}` : '',
-    blog ? `date: "${item.date}"` : '',
-    blog ? 'author: "rula-diab"' : '',
-    blog ? 'category: "Library"' : '',
-    blog ? 'tags: []' : '',
-    'draft: false',
-    '---',
-    '',
-    (blog ? item.markdown.replace(blogPreamble, '') : item.markdown) || `# ${item.h1}`,
-    ''
-  ].filter((line) => line !== '');
-  return `${lines.join('\n')}\n`;
-}
-
-async function copySourceAssets(files) {
-  const rows = ['source,target,size,kind'];
-  const seen = new Set();
-  await Promise.all(['images', 'fonts', 'downloads'].map((kind) => fs.rm(path.join(assets, kind), { recursive: true, force: true })));
-  for (const file of files.filter((name) => name.startsWith('hubfs/') || name.startsWith('hs-fs/hubfs/') || name.startsWith('_hcms/googlefonts/'))) {
-    const font = file.startsWith('_hcms/googlefonts/');
-    const raw = font ? file.replace('_hcms/googlefonts/', '') : path.basename(file).split('?')[0];
-    if (!font && !/\.(?:avif|gif|jpe?g|png|svg|webp|pdf)$/i.test(raw)) continue;
-    const key = `${font}:${raw}`;
-    if (!raw || seen.has(key)) continue;
-    seen.add(key);
-    const kind = font ? 'fonts' : raw.endsWith('.pdf') ? 'downloads' : 'images';
-    const target = path.join(assets, kind, raw);
-    await mkdir(path.dirname(target));
-    await fs.copyFile(path.join(mirror, file), target);
-    const size = (await fs.stat(target)).size;
-    rows.push([file, path.relative(root, target), size, kind].map(csv).join(','));
-  }
-  await fs.writeFile(path.join(reports, 'asset-inventory.csv'), `${rows.join('\n')}\n`);
-}
-
-async function removeMissingFeaturedImages() {
-  for (const type of ['pages', 'blog']) {
-    for (const file of await walk(path.join(content, type))) {
-      const target = path.join(content, type, file);
-      let source = await fs.readFile(target, 'utf8');
-      const image = source.match(/^featuredImage:\s*"\/assets\/images\/([^"]+)"/m)?.[1];
-      if (!image) continue;
-      if (!await fs.access(path.join(assets, 'images', image)).then(() => true).catch(() => false)) {
-        source = source.replace(/^featuredImage:.*\n/m, '');
-        await fs.writeFile(target, source);
-      }
-    }
-  }
-}
-
-async function main() {
-  const files = await walk(mirror);
-  const selected = selectCanonical(files);
-  const records = [];
-  for (const file of selected) records.push(record(file, await fs.readFile(path.join(mirror, file), 'utf8')));
-  await Promise.all(['pages', 'blog', 'authors'].map((type) => fs.rm(path.join(content, type), { recursive: true, force: true })));
-  for (const lang of ['en', 'ar', 'es']) {
-    for (const type of ['pages', 'blog', 'authors']) await mkdir(path.join(content, type, lang));
-  }
-  for (const item of records) {
-    const blog = item.url.includes('/library/') && !item.url.endsWith('/library');
-    const slug = ['/', '/ar', '/es'].includes(item.url) ? 'index' : item.url.split('/').filter(Boolean).at(-1);
-    await fs.writeFile(path.join(content, blog ? 'blog' : 'pages', item.lang, `${slug}.md`), frontmatter(item, blog));
-  }
-  for (const lang of ['en', 'ar', 'es']) {
-    await fs.writeFile(path.join(content, 'authors', lang, 'rula-diab.md'), `---\nname: "Rula Diab"\nslug: "rula-diab"\ndescription: "Founder and clinical leader at Arizona Institute for Autism."\navatar: "/assets/images/rula-diab-avatar.jpg"\nlang: "${lang}"\ntranslationKey: "rula-diab"\n---\n`);
-  }
-  if (!records.some((item) => item.lang === 'ar' && item.url === '/ar')) {
-    await fs.writeFile(path.join(content, 'pages/ar/index.md'), `---\ntitle: "معهد أريزونا للتوحد"\ndescription: "صفحة عربية تمهيدية لمعهد أريزونا للتوحد."\nslug: "index"\ncanonical: "https://www.azinstitute4autism.com/ar"\nlang: "ar"\ntranslationKey: "home"\ndraft: false\n---\n\n<!-- TODO: Replace this placeholder with reviewed Arabic content. -->\n\nتتوفر المقالات العربية الحالية في المكتبة. يرجى مراجعة المحتوى العربي قبل النشر.\n`);
-  }
-  const rows = ['source_file,url,language,type,title,description,h1'];
-  for (const item of records) rows.push([item.file, item.url, item.lang, item.url.includes('/library/') ? 'blog' : 'page', item.title, item.description, item.h1].map(csv).join(','));
-  await mkdir(reports);
-  await fs.writeFile(path.join(reports, 'url-inventory.csv'), `${rows.join('\n')}\n`);
-  await copySourceAssets(files);
-  await removeMissingFeaturedImages();
-  console.log(`Fallback extraction completed: ${records.length} canonical records.`);
-}
-
-await main();

www/tools/extract-site.mjs

@@ -1,198 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import fg from 'fast-glob';
-import { load } from 'cheerio';
-import TurndownService from 'turndown';
-import matter from 'gray-matter';
-
-const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
-const mirror = path.resolve(root, '../www.azinstitute4autism.com');
-const site = 'https://www.azinstitute4autism.com';
-const contentRoot = path.join(root, 'src/content');
-const publicRoot = path.join(root, 'public/assets');
-const reportsRoot = path.join(root, 'reports');
-const turndown = new TurndownService({ headingStyle: 'atx', bulletListMarker: '-' });
-turndown.remove(['script', 'style', 'noscript', 'iframe']);
-
-const mkdir = (dir) => fs.mkdir(dir, { recursive: true });
-const clean = (value = '') => value.replace(/\s+/g, ' ').trim();
-const csv = (value = '') => `"${String(value).replaceAll('"', '""')}"`;
-const yaml = (value = '') => JSON.stringify(String(value));
-const blogPreamble = /^# .+\n\n!\[[^\]]*\]\(\/assets\/images\/rula-diab-avatar\.jpg\)\n\n[^\n]+\n\n[^\n]+\n\n!\[[^\]]*\]\([^)]+\)\n\n/;
-
-function contentMarkdown(markdown, isBlog) {
-  return isBlog ? markdown.replace(blogPreamble, '') : markdown;
-}
-
-function logicalFile(file) {
-  return file.replace(/(?:\.html)?\?(?:hsLang=[^.]+|hs_amp=true)\.html$/, '.html');
-}
-
-function canonicalFiles(files) {
-  const selected = new Map();
-  for (const file of files) {
-    if (!file.endsWith('.html') || /\/(?:page|author)\//.test(file) || file.includes('hs_amp=true')) continue;
-    const logical = logicalFile(file);
-    const lang = logical.startsWith('ar/') ? 'ar' : logical.startsWith('es/') ? 'es' : 'en';
-    const key = `${lang}:${logical}`;
-    const current = selected.get(key);
-    if (!current || (!file.includes('?') && current.includes('?'))) selected.set(key, file);
-  }
-  return [...selected.values()];
-}
-
-function sourceUrl(file) {
-  const logical = logicalFile(file);
-  if (logical === 'index.html') return '/';
-  return `/${logical.replace(/\/index\.html$/, '').replace(/\.html$/, '')}`;
-}
-
-function isBlogFile(file) {
-  return file.startsWith('library/') || file.includes('/library/');
-}
-
-function languageFor(file, $) {
-  if (file.startsWith('ar/')) return 'ar';
-  if (file.startsWith('es/')) return 'es';
-  return $('html').attr('lang')?.split('-')[0] || 'en';
-}
-
-function localizeAsset(value = '') {
-  const decoded = value.replace(/^https?:\/\/[^/]+/, '').replace(/^\.\.\//, '/');
-  const match = decoded.match(/(?:\/)?(?:hs-fs\/)?hubfs\/([^?#]+)/);
-  if (!match) return undefined;
-  return `/assets/images/${path.basename(decodeURIComponent(match[1])).split('?')[0]}`;
-}
-
-function extractRecord(file, html) {
-  const $ = load(html);
-  const lang = languageFor(file, $);
-  const url = sourceUrl(file);
-  const title = clean($('title').first().text()) || clean($('h1').first().text()) || url;
-  const description = $('meta[name="description"]').attr('content') || '';
-  const h1 = clean($('h1').first().text());
-  const image = localizeAsset(
-    $('meta[property="og:image"]').attr('content') ||
-    $('.blog-post__body img, main img, .body-container-wrapper img').first().attr('src') ||
-    ''
-  );
-  const alt = clean($('.blog-post__body img, main img, .body-container-wrapper img').first().attr('alt') || '');
-  const dateText = $('meta[property="article:published_time"]').attr('content') ||
-    $('time').first().attr('datetime') || '2024-01-01';
-  const date = /^\d{4}-\d{2}-\d{2}/.test(dateText) ? dateText.slice(0, 10) : '2024-01-01';
-  const selector = isBlogFile(file) ? '.blog-post__body' : 'main, .body-container-wrapper';
-  const body = $(selector).first().clone();
-  body.find('header, footer, nav, form, script, style, noscript, .hs_cos_wrapper_type_form').remove();
-  body.find('*').removeAttr('style').removeAttr('id').removeAttr('data-hs-cos-general-type').removeAttr('data-hs-cos-type');
-  body.find('a').each((_, element) => {
-    const href = $(element).attr('href');
-    if (!href) return;
-    $(element).attr('href', href
-      .replace(/\.html(?:%3F|\?)[^"]*$/, '')
-      .replace(/\.html$/, '')
-      .replace(/^index$/, '/'));
-  });
-  body.find('img').each((_, element) => {
-    const src = localizeAsset($(element).attr('src'));
-    if (src) $(element).attr('src', src);
-  });
-  const tables = [];
-  body.find('table').each((_, element) => {
-    const token = `__TABLE_${tables.length}__`;
-    tables.push($(element).prop('outerHTML'));
-    $(element).replaceWith(`\n\n${token}\n\n`);
-  });
-  const markdown = contentMarkdown(
-    tables.reduce((output, table, index) => output.replaceAll(`__TABLE_${index}__`, table),
-      turndown.turndown(body.html() || '').replace(/\n{3,}/g, '\n\n').trim()),
-    isBlogFile(file)
-  );
-  return { file, lang, url, title, description, h1, image, alt, date, markdown };
-}
-
-function frontmatter(record, type) {
-  const slug = ['/', '/ar', '/es'].includes(record.url) ? 'index' : record.url.split('/').filter(Boolean).at(-1);
-  const data = {
-    title: record.title,
-    description: record.description,
-    slug,
-    canonical: `${site}${record.url}`,
-    lang: record.lang,
-    translationKey: slug,
-    draft: false
-  };
-  if (record.image) data.featuredImage = record.image;
-  if (record.alt) data.alt = record.alt;
-  if (type === 'blog') Object.assign(data, {
-    date: record.date,
-    author: 'rula-diab',
-    category: 'Library',
-    tags: []
-  });
-  return matter.stringify(record.markdown || `# ${record.h1 || record.title}\n`, data);
-}
-
-async function copyAssets() {
-  const assets = await fg(['hubfs/*', 'hs-fs/hubfs/*', '_hcms/googlefonts/**/*'], { cwd: mirror, onlyFiles: true });
-  const inventory = ['source,target,size,kind'];
-  const seen = new Set();
-  await Promise.all(['images', 'fonts', 'downloads'].map((kind) => fs.rm(path.join(publicRoot, kind), { recursive: true, force: true })));
-  for (const source of assets) {
-    const original = path.join(mirror, source);
-    const stat = await fs.stat(original);
-    const isFont = source.startsWith('_hcms/googlefonts/');
-    const rawName = isFont ? source.replace('_hcms/googlefonts/', '') : path.basename(source).split('?')[0];
-    if (!isFont && !/\.(?:avif|gif|jpe?g|png|svg|webp|pdf)$/i.test(rawName)) continue;
-    if (!rawName || seen.has(`${isFont}:${rawName}`)) continue;
-    seen.add(`${isFont}:${rawName}`);
-    const target = path.join(publicRoot, isFont ? 'fonts' : rawName.endsWith('.pdf') ? 'downloads' : 'images', rawName);
-    await mkdir(path.dirname(target));
-    await fs.copyFile(original, target);
-    inventory.push([source, path.relative(root, target), stat.size, isFont ? 'font' : path.extname(rawName).slice(1)].map(csv).join(','));
-  }
-  await fs.writeFile(path.join(reportsRoot, 'asset-inventory.csv'), `${inventory.join('\n')}\n`);
-}
-
-async function main() {
-  await Promise.all([
-    mkdir(contentRoot), mkdir(publicRoot), mkdir(reportsRoot),
-    ...['en', 'ar', 'es'].flatMap((lang) => [
-      mkdir(path.join(contentRoot, 'pages', lang)),
-      mkdir(path.join(contentRoot, 'blog', lang)),
-      mkdir(path.join(contentRoot, 'authors', lang))
-    ])
-  ]);
-  const files = canonicalFiles(await fg(['**/*.html'], { cwd: mirror, onlyFiles: true }));
-  const records = [];
-  for (const file of files) records.push(extractRecord(file, await fs.readFile(path.join(mirror, file), 'utf8')));
-  const corePages = new Set(records.filter((r) => !isBlogFile(r.file)).map((r) => r.file));
-  for (const record of records) {
-    const isBlog = isBlogFile(record.file);
-    if (!isBlog && !corePages.has(record.file)) continue;
-    const slug = ['/', '/ar', '/es'].includes(record.url) ? 'index' : record.url.split('/').filter(Boolean).at(-1);
-    const target = path.join(contentRoot, isBlog ? 'blog' : 'pages', record.lang, `${slug}.md`);
-    await fs.writeFile(target, frontmatter(record, isBlog ? 'blog' : 'page'));
-  }
-  for (const lang of ['en', 'ar', 'es']) {
-    await fs.writeFile(path.join(contentRoot, 'authors', lang, 'rula-diab.md'), matter.stringify('', {
-      name: 'Rula Diab',
-      slug: 'rula-diab',
-      description: 'Founder and clinical leader at Arizona Institute for Autism.',
-      avatar: '/assets/images/rula-diab-avatar.jpg',
-      lang,
-      translationKey: 'rula-diab'
-    }));
-  }
-  const urlRows = ['source_file,url,language,type,title,description,h1'];
-  for (const record of records) urlRows.push([
-    record.file, record.url, record.lang,
-    isBlogFile(record.file) ? 'blog' : 'page',
-    record.title, record.description, record.h1
-  ].map(csv).join(','));
-  await fs.writeFile(path.join(reportsRoot, 'url-inventory.csv'), `${urlRows.join('\n')}\n`);
-  await copyAssets();
-  console.log(`Extracted ${records.length} canonical pages and posts.`);
-}
-
-await main();

www/tools/generate-redirects.mjs

@@ -1,9 +1,9 @@
 import fs from 'node:fs/promises';
 
 const redirects = [
-  { from: '/aba', to: '/aba-therapy', status: 301, reason: 'Brief alias to preserved mirror URL' },
+  { from: '/aba', to: '/aba-therapy', status: 301, reason: 'Brief legacy alias to preserved URL' },
-  { from: '/autismevaluations', to: '/autism-evaluations', status: 301, reason: 'Brief alias to preserved mirror URL' },
+  { from: '/autismevaluations', to: '/autism-evaluations', status: 301, reason: 'Brief legacy alias to preserved URL' },
-  { from: '/learnersocialclub', to: '/learner-social-club', status: 301, reason: 'Brief alias to preserved mirror URL' }
+  { from: '/learnersocialclub', to: '/learner-social-club', status: 301, reason: 'Brief legacy alias to preserved URL' }
 ];
 await fs.writeFile(new URL('../src/data/redirects.json', import.meta.url), `${JSON.stringify(redirects, null, 2)}\n`);
 await fs.writeFile(new URL('../reports/redirect-map.csv', import.meta.url),