#!/usr/bin/env node /** * `dist/sitemap.xml` and `robots.txt`. * * PIG's own site is `noindex` — it is a private tool. This one is the exact * inverse: it exists to be found, shared and quoted, so the robots file opens * everything and points at a sitemap listing every route the router actually * produces. * * The route list is the same one `prerender.mjs` bakes, from the same reader, * so a sitemap entry cannot point at a URL that has no prerendered document. * Those two drifting apart is how you end up submitting 14 URLs and having 13 * of them indexed as the homepage. * * `robots.txt` is written into `public/` (source, committed) AND into `dist/` * (because this runs after the build that would have copied it). The sitemap is * a build artifact only — it lists what this build contains. */ import fs from 'node:fs'; import path from 'node:path'; import { SITE_ORIGIN, abs, die, exists, expandRoutes, green, loadAllMetas, loadVerticals, rel } from './_lib.mjs'; const DIST = abs('dist'); const PUBLIC = abs('public'); const { metas } = loadAllMetas(); const verticals = loadVerticals(); if (verticals.error) die(`could not read the verticals: ${verticals.error}`); const { routes, errors } = expandRoutes({ metas, verticals: verticals.list }); if (errors.length) die(`route enumeration failed:\n - ${errors.join('\n - ')}`); /* ------------------------------------------------------------- robots.txt */ const robots = [ '# demo.primeintellectgrowth.com', '#', '# This site is meant to be found. Every page is public, static and safe to', '# crawl; the source it documents is public too.', '', 'User-agent: *', 'Allow: /', '', `Sitemap: ${SITE_ORIGIN}/sitemap.xml`, '', ].join('\n'); fs.mkdirSync(PUBLIC, { recursive: true }); fs.writeFileSync(path.join(PUBLIC, 'robots.txt'), robots, 'utf8'); /* ------------------------------------------------------------ sitemap.xml */ if (!exists(path.join(DIST, 'index.html'))) { die( `${rel(DIST)}/index.html does not exist, so there is no build to describe. ` + 'Run `pnpm build` first. (public/robots.txt has been written.)', ); } // One date for the whole build. Per-file mtimes would claim the vertical pages // changed whenever anything in the bundle did, which is true of a hash-named // asset and useless to a crawler. const lastmod = new Date().toISOString().slice(0, 10); const xml = [ '', '', ...routes.map((route) => [ ' ', ` ${escapeXml(`${SITE_ORIGIN}${route.path === '/' ? '/' : route.path}`)}`, ` ${lastmod}`, ' ', ].join('\n'), ), '', '', ].join('\n'); fs.writeFileSync(path.join(DIST, 'sitemap.xml'), xml, 'utf8'); fs.writeFileSync(path.join(DIST, 'robots.txt'), robots, 'utf8'); console.log(green(`sitemap: ${routes.length} URLs -> ${rel(path.join(DIST, 'sitemap.xml'))}`)); console.log(green(`robots: ${rel(path.join(PUBLIC, 'robots.txt'))} and ${rel(path.join(DIST, 'robots.txt'))}`)); /** Sitemap URLs are XML text; a bare `&` in a query string invalidates the file. */ function escapeXml(value) { return value .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, '''); }