PIG-Demo/scripts/sitemap.mjs

#!/usr/bin/env node
/**
 * `dist/sitemap.xml` and `robots.txt`.
 *
 * PIG's own site is `noindex` — it is a private tool. This one is the exact
 * inverse: it exists to be found, shared and quoted, so the robots file opens
 * everything and points at a sitemap listing every route the router actually
 * produces.
 *
 * The route list is the same one `prerender.mjs` bakes, from the same reader,
 * so a sitemap entry cannot point at a URL that has no prerendered document.
 * Those two drifting apart is how you end up submitting 14 URLs and having 13
 * of them indexed as the homepage.
 *
 * `robots.txt` is written into `public/` (source, committed) AND into `dist/`
 * (because this runs after the build that would have copied it). The sitemap is
 * a build artifact only — it lists what this build contains.
 */

import fs from 'node:fs';
import path from 'node:path';

import { SITE_ORIGIN, abs, die, exists, expandRoutes, green, loadAllMetas, loadVerticals, rel } from './_lib.mjs';

const DIST = abs('dist');
const PUBLIC = abs('public');

const { metas } = loadAllMetas();
const verticals = loadVerticals();
if (verticals.error) die(`could not read the verticals: ${verticals.error}`);

const { routes, errors } = expandRoutes({ metas, verticals: verticals.list });
if (errors.length) die(`route enumeration failed:\n  - ${errors.join('\n  - ')}`);

/* ------------------------------------------------------------- robots.txt */

// Kept byte-identical to the committed public/robots.txt so re-running this
// script is a no-op in the diff. If you change the wording, change it here —
// this is the generator, and the committed file is its output.
const robots = [
  '# The inverse of primeintellectgrowth.com, deliberately: being found is the',
  '# entire point of this site.',
  'User-agent: *',
  'Allow: /',
  '',
  `Sitemap: ${SITE_ORIGIN}/sitemap.xml`,
  '',
].join('\n');

fs.mkdirSync(PUBLIC, { recursive: true });
fs.writeFileSync(path.join(PUBLIC, 'robots.txt'), robots, 'utf8');

/* ------------------------------------------------------------ sitemap.xml */

if (!exists(path.join(DIST, 'index.html'))) {
  die(
    `${rel(DIST)}/index.html does not exist, so there is no build to describe. ` +
      'Run `pnpm build` first. (public/robots.txt has been written.)',
  );
}

// One date for the whole build. Per-file mtimes would claim the vertical pages
// changed whenever anything in the bundle did, which is true of a hash-named
// asset and useless to a crawler.
const lastmod = new Date().toISOString().slice(0, 10);

const xml = [
  '<?xml version="1.0" encoding="UTF-8"?>',
  '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
  ...routes.map((route) =>
    [
      '  <url>',
      `    <loc>${escapeXml(`${SITE_ORIGIN}${route.path === '/' ? '/' : route.path}`)}</loc>`,
      `    <lastmod>${lastmod}</lastmod>`,
      '  </url>',
    ].join('\n'),
  ),
  '</urlset>',
  '',
].join('\n');

fs.writeFileSync(path.join(DIST, 'sitemap.xml'), xml, 'utf8');
fs.writeFileSync(path.join(DIST, 'robots.txt'), robots, 'utf8');

console.log(green(`sitemap: ${routes.length} URLs -> ${rel(path.join(DIST, 'sitemap.xml'))}`));
console.log(green(`robots:  ${rel(path.join(PUBLIC, 'robots.txt'))} and ${rel(path.join(DIST, 'robots.txt'))}`));

/** Sitemap URLs are XML text; a bare `&` in a query string invalidates the file. */
function escapeXml(value) {
  return value
    .replace(/&/g, '&amp;')
    .replace(/</g, '&lt;')
    .replace(/>/g, '&gt;')
    .replace(/"/g, '&quot;')
    .replace(/'/g, '&apos;');
}