Add reference-only build and pre-built data.json input for PR previews

.gitignore

@@ -1,5 +1,7 @@
 # build output
 dist/
+# reference-only build stash (npm run build:reference-only)
+.reference-only-stash/
 # generated types
 .astro/
 

astro.config.mjs

@@ -18,6 +18,10 @@ if (shouldSkipCompress) {
 
 const isA11yTest = Boolean(process.env.A11Y_TEST);
 
+// Reference-only preview builds (processing/p5.js#9303) skip the service
+// worker: it globs and precaches the whole dist, which a preview doesn't need.
+const isReferenceOnly = Boolean(process.env.REFERENCE_ONLY);
+
 // https://astro.build/config
 export default defineConfig({
   site: 'https://p5js.org',
@@ -32,27 +36,31 @@ export default defineConfig({
     }),
     mdx(),
     fast(),
-    serviceWorker({
-      workbox: {
-        globPatterns: [
-          "**/*.{css,js,jpg,json,png,svg,ico,woff,woff2}", // Cache all assets accept HTML
-        ],
-        clientsClaim: true,
-        runtimeCaching: [
-          {
-            urlPattern: ({ url }) => url.pathname.endsWith(".html"), // Caches HTML pages
-            handler: "CacheFirst", // Tries the cache first, then falls back to network if offline
-            options: {
-              cacheName: "html-pages-cache",
-              expiration: {
-                maxEntries: 50, // Limits the number of HTML pages cached
-                maxAgeSeconds: 24 * 60 * 60 * 7, // Cache for 1 week
-              },
+    ...(isReferenceOnly
+      ? []
+      : [
+          serviceWorker({
+            workbox: {
+              globPatterns: [
+                "**/*.{css,js,jpg,json,png,svg,ico,woff,woff2}", // Cache all assets accept HTML
+              ],
+              clientsClaim: true,
+              runtimeCaching: [
+                {
+                  urlPattern: ({ url }) => url.pathname.endsWith(".html"), // Caches HTML pages
+                  handler: "CacheFirst", // Tries the cache first, then falls back to network if offline
+                  options: {
+                    cacheName: "html-pages-cache",
+                    expiration: {
+                      maxEntries: 50, // Limits the number of HTML pages cached
+                      maxAgeSeconds: 24 * 60 * 60 * 7, // Cache for 1 week
+                    },
+                  },
+                },
+              ],
             },
-          },
-        ],
-      },
-    }),
+          }),
+        ]),
   ],
   prefetch: {
     defaultStrategy: "viewport",

docs/scripts.md

@@ -56,6 +56,20 @@ When you're done, you can run this command to reset your changes:
 npm run custom:cleanup
 ```
 
+## Building the reference from a pre-built data.json
+
+`npm run build:reference` normally clones p5.js and runs its docs build to produce the YUIDoc `data.json` it works from. If you already have that file — for example from running `npm run docs` in a p5.js checkout, or from a CI artifact — you can hand it over directly with the `P5_DATA_JSON_PATH` environment variable, which skips the clone and the docs build entirely:
+
+```sh
+P5_DATA_JSON_PATH=../p5.js/docs/reference/data.json npm run build:reference
+```
+
+This is mainly useful in CI, where the p5.js build that produced the `data.json` has already run.
+
+## Reference-only site build
+
+`npm run build:reference-only` builds only the English reference pages (`/reference/**`); every other page 404s. It works by temporarily moving the non-reference routes out of `src/pages`, running `astro build`, and restoring them afterwards. Setting `REFERENCE_SKIP_LOWERCASE=1` additionally skips the lowercase alias of each reference route (for example `/reference/p5/loadimage/` alongside `/reference/p5/loadImage/`), which nearly halves the page count. This build is what the p5.js PR preview uses; combined with `P5_DATA_JSON_PATH` above it produces a reference preview without cloning or building p5.js.
+
 ## After a p5.js release
 
 To update the content on the p5.js website following a new release of p5.js, you should run all of these commands to pull in the latest content. **Be sure to run the search indices script last as it depends on the results of the other scripts to be accurate.**

package.json

@@ -17,6 +17,7 @@
     "build:contributor-docs": "tsx ./src/scripts/builders/contribute.ts",
     "build:contributors": "tsx ./src/scripts/builders/people.ts",
     "build:reference": "tsx ./src/scripts/builders/reference.ts",
+    "build:reference-only": "tsx ./src/scripts/referenceOnlyBuild.ts",
     "build:search": "tsx ./src/scripts/builders/search.ts",
     "build:p5-version": "tsx ./src/scripts/p5-version.ts",
     "custom:dev": "tsx ./src/scripts/branchTest.ts",

src/pages/reference/[...id].astro

@@ -9,7 +9,7 @@ import {
 export async function getStaticPaths() {
   const referenceEntries = await getCollectionInDefaultLocale("reference");
 
-  return referenceEntries
+  const paths = referenceEntries
     // @ts-expect-error - Casting to the reference item schema
     .filter((entry) => entry.module !== 'Constants')
     .map((entry) => ({
@@ -22,7 +22,16 @@ export async function getStaticPaths() {
           return relatedEntry.data.submodule === entry.data.submodule;
         }),
       },
-    }))
+    }));
+
+  // The lowercase alias of every route nearly doubles the reference page
+  // count; preview builds (processing/p5.js#9303) skip them to keep the
+  // output small.
+  if (process.env.REFERENCE_SKIP_LOWERCASE) {
+    return paths;
+  }
+
+  return paths
     .reduce<GetStaticPathsResult>((acc, entry) => {
       acc.push(entry);
       acc.push(Object.assign({}, entry, {

src/scripts/parsers/reference.ts

@@ -30,28 +30,7 @@ export const parseLibraryReference =
     const useExternalP5Sound = !!process.env.P5_REPO_BRANCH ||
       latestRelease.startsWith('v2');
 
-    // Clone p5.js
-    await cloneLibraryRepo(
-      localPath,
-      process.env.P5_REPO_URL || p5RepoUrl,
-      process.env.P5_BRANCH || latestRelease,
-      { shouldFixAbsolutePathInPreprocessor: false },
-    );
-
-    // Install dependencies and build docs in the p5 repo
-    await createP5Docs('p5.js', 'data-p5');
-
-    // If we're using a custom build of p5 instead of a public release, create
-    // a build and copy it to the specified path
-    if (process.env.PUBLIC_P5_LIBRARY_PATH) {
-      await createP5Build('p5.js', `../../../public${process.env.PUBLIC_P5_LIBRARY_PATH}`);
-    }
-    if (process.env.PUBLIC_P5_WEBGPU_LIBRARY_PATH) {
-      await fs.cp(
-        path.join(__dirname, 'in', 'p5.js', 'lib', 'p5.webgpu.js'),
-        path.join(__dirname, `../../../public${process.env.PUBLIC_P5_WEBGPU_LIBRARY_PATH}`),
-      );
-    }
+    await prepareP5Data(latestRelease);
 
     // Copy the reference output so we can process it
     const p5Data = await getYuidocOutput('data-p5');
@@ -145,6 +124,66 @@ export const parseLibraryReference =
     return result;
   };
 
+/**
+ * Puts the p5.js YUIDoc data where getYuidocOutput('data-p5') reads it.
+ *
+ * If P5_DATA_JSON_PATH is set, that file is used directly and the p5.js
+ * clone, `npm install`, and `npm run docs` are all skipped. This lets a CI
+ * job that already built p5.js docs (e.g. a p5.js PR build) hand its
+ * data.json over instead of this script rebuilding it from a fresh clone.
+ *
+ * Otherwise p5.js is cloned (honoring P5_REPO_URL / P5_BRANCH) and its docs
+ * are built in the clone, exactly as before.
+ */
+export const prepareP5Data = async (latestRelease: string): Promise<void> => {
+  const prebuiltDataPath = process.env.P5_DATA_JSON_PATH;
+  if (prebuiltDataPath) {
+    let stats;
+    try {
+      stats = await fs.stat(prebuiltDataPath);
+    } catch {
+      throw new Error(
+        `P5_DATA_JSON_PATH is set to "${prebuiltDataPath}" but no file exists there`,
+      );
+    }
+    if (!stats.isFile()) {
+      throw new Error(
+        `P5_DATA_JSON_PATH is set to "${prebuiltDataPath}" but that path is not a file`,
+      );
+    }
+    console.log(
+      `Skipping p5.js clone and docs build; using pre-built data.json from ${prebuiltDataPath}`,
+    );
+    const dataP5Dir = path.join(yuidocOutputPath, 'data-p5');
+    await fs.mkdir(dataP5Dir, { recursive: true });
+    await fs.copyFile(prebuiltDataPath, path.join(dataP5Dir, 'data.json'));
+    return;
+  }
+
+  // Clone p5.js
+  await cloneLibraryRepo(
+    localPath,
+    process.env.P5_REPO_URL || p5RepoUrl,
+    process.env.P5_BRANCH || latestRelease,
+    { shouldFixAbsolutePathInPreprocessor: false },
+  );
+
+  // Install dependencies and build docs in the p5 repo
+  await createP5Docs('p5.js', 'data-p5');
+
+  // If we're using a custom build of p5 instead of a public release, create
+  // a build and copy it to the specified path
+  if (process.env.PUBLIC_P5_LIBRARY_PATH) {
+    await createP5Build('p5.js', `../../../public${process.env.PUBLIC_P5_LIBRARY_PATH}`);
+  }
+  if (process.env.PUBLIC_P5_WEBGPU_LIBRARY_PATH) {
+    await fs.cp(
+      path.join(__dirname, 'in', 'p5.js', 'lib', 'p5.webgpu.js'),
+      path.join(__dirname, `../../../public${process.env.PUBLIC_P5_WEBGPU_LIBRARY_PATH}`),
+    );
+  }
+};
+
 /**
  * Gets the parsed YUIDoc output from the saved file and parses it as JSON
  * returns the parsed YUIDoc output

src/scripts/referenceOnlyBuild.ts

@@ -0,0 +1,131 @@
+/**
+ * Builds only the English reference pages (/reference/**), for PR previews
+ * (processing/p5.js#9303). Every other route 404s in the output.
+ *
+ * Astro has no config option to exclude routes from a static build, so this
+ * wrapper temporarily moves every non-reference entry out of src/pages before
+ * running `astro build`, and restores them afterwards — also on failure and
+ * on SIGINT/SIGTERM. Only src/pages/reference/** plus the shared _utils
+ * modules (imported by layouts and components) stay in place.
+ *
+ * Run with: npm run build:reference-only
+ * Set REFERENCE_SKIP_LOWERCASE=1 to also skip the lowercase alias routes.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const root = path.join(__dirname, "..", "..");
+const pagesDir = path.join(root, "src", "pages");
+export const stashDir = path.join(root, ".reference-only-stash");
+
+// Entries in src/pages that the reference build needs
+const keep = new Set(["reference", "_utils.ts", "_utils-node.ts"]);
+
+/**
+ * A non-empty stash dir means a previous run died before restoring
+ * src/pages (e.g. kill -9; normal failures and Ctrl-C restore on their own).
+ * Refuse to run rather than mix two runs' stashes, and say how to recover.
+ */
+export const assertStashDirUsable = (dir: string) => {
+  let stashed: string[];
+  try {
+    stashed = fs.readdirSync(dir);
+  } catch {
+    return; // no stash dir: nothing to recover
+  }
+  if (stashed.length === 0) return;
+  throw new Error(
+    `Refusing to run: ${dir} already exists and is not empty.\n` +
+      `A previous run was probably interrupted before it could restore ` +
+      `src/pages. Recover with:\n\n` +
+      `  mv ${path.relative(root, dir)}/* src/pages/ && rmdir ${path.relative(root, dir)}\n`,
+  );
+};
+
+// Sync so it can also run inside a signal handler.
+const restoreStash = () => {
+  let stashed: string[];
+  try {
+    stashed = fs.readdirSync(stashDir);
+  } catch {
+    return;
+  }
+  for (const entry of stashed) {
+    fs.renameSync(path.join(stashDir, entry), path.join(pagesDir, entry));
+  }
+  fs.rmdirSync(stashDir);
+  console.log("Restored non-reference pages.");
+};
+
+let restored = false;
+const restoreOnce = () => {
+  if (restored) return;
+  restored = true;
+  restoreStash();
+};
+
+let currentChild: ReturnType<typeof spawn> | undefined;
+
+const runAstroBuild = () =>
+  new Promise<number>((resolve) => {
+    // `astro build` directly rather than `npm run build`, which would also
+    // run `astro check` — previews don't need type checking and it costs time.
+    // detached puts npx and the astro process it spawns in their own process
+    // group, so the signal handler below can kill both; signalling only the
+    // npx wrapper leaves the actual build running.
+    const child = spawn("npx", ["astro", "build"], {
+      cwd: root,
+      stdio: "inherit",
+      detached: true,
+      env: { ...process.env, REFERENCE_ONLY: "1" },
+    });
+    currentChild = child;
+    child.on("exit", (code, signal) => resolve(signal ? 1 : (code ?? 1)));
+    child.on("error", (err) => {
+      console.error(`Failed to start astro build: ${err}`);
+      resolve(1);
+    });
+  });
+
+const main = async () => {
+  assertStashDirUsable(stashDir);
+  fs.mkdirSync(stashDir, { recursive: true });
+
+  // Kill the build's process group, restore src/pages, and exit with the
+  // conventional signal exit code. The child is detached, so this group-kill
+  // is what stops the build on Ctrl-C too.
+  for (const signal of ["SIGINT", "SIGTERM"] as const) {
+    process.on(signal, () => {
+      if (currentChild?.pid) {
+        try {
+          process.kill(-currentChild.pid, signal);
+        } catch {
+          // the group already exited
+        }
+      }
+      restoreOnce();
+      process.exit(signal === "SIGINT" ? 130 : 143);
+    });
+  }
+
+  try {
+    let moved = 0;
+    for (const entry of fs.readdirSync(pagesDir)) {
+      if (keep.has(entry)) continue;
+      fs.renameSync(path.join(pagesDir, entry), path.join(stashDir, entry));
+      moved++;
+    }
+    console.log(`Moved ${moved} non-reference page entries aside.`);
+    process.exitCode = await runAstroBuild();
+  } finally {
+    restoreOnce();
+  }
+};
+
+// Only run when executed as a script, so tests can import the helpers.
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main();
+}

test/scripts/reference-only-build.test.ts

@@ -0,0 +1,34 @@
+/** Covers the stash-dir safety check in src/scripts/referenceOnlyBuild.ts. */
+import { afterEach, beforeEach, describe, expect, test } from "vitest";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { assertStashDirUsable } from "../../src/scripts/referenceOnlyBuild";
+
+let tempDir: string;
+
+describe("assertStashDirUsable", () => {
+  beforeEach(async () => {
+    tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ref-only-stash-"));
+  });
+
+  afterEach(async () => {
+    await fs.rm(tempDir, { recursive: true, force: true });
+  });
+
+  test("refuses a non-empty stash dir and says how to recover", async () => {
+    // The leftovers of a run that was killed before restoring src/pages;
+    // starting a new run over them would mix two stashes.
+    await fs.writeFile(path.join(tempDir, "about.astro"), "");
+
+    expect(() => assertStashDirUsable(tempDir)).toThrow(/Refusing to run/);
+    expect(() => assertStashDirUsable(tempDir)).toThrow(/mv .*src\/pages/);
+  });
+
+  test("accepts a missing or empty stash dir", async () => {
+    expect(() =>
+      assertStashDirUsable(path.join(tempDir, "does-not-exist")),
+    ).not.toThrow();
+    expect(() => assertStashDirUsable(tempDir)).not.toThrow();
+  });
+});