ci(deploy): prune expired deployment prefixes

.github/workflows/prod-build.yml

@@ -39,6 +39,10 @@ permissions:
 
 jobs:
   build:
+    # Queue runs so a delayed older run can't deploy over a newer one.
+    concurrency:
+      group: prod-deploy
+      cancel-in-progress: false
     environment: prod
     runs-on: ubuntu-latest-dex-builder
 
@@ -421,6 +425,40 @@ jobs:
             wait $pid
           done
 
+      - name: Read active deployment prefixes
+        id: active-deployments
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        working-directory: mdn/dex
+        env:
+          GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }}
+        run: |
+          prefixes="$(node scripts/cleanup-deployments.js get-active "$GCP_BUCKET_NAME" \
+            mdn-prod-prod-europe-west1@europe-west1 \
+            mdn-prod-prod-us-west1@us-west1 \
+            mdn-prod-prod-asia-east1@asia-east1)"
+          echo "prefixes=$prefixes" >> "$GITHUB_OUTPUT"
+
+      - name: Authenticate for deployment cleanup
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
+        with:
+          token_format: access_token
+          service_account: deploy-prod-content@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
+          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions
+
+      - name: Clean up old deployments
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        working-directory: mdn/dex
+        env:
+          GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }}
+          DEPLOYMENTS_RETENTION_DAYS: ${{ vars.DEPLOYMENTS_RETENTION_DAYS || '30' }}
+          ACTIVE_DEPLOYMENTS: ${{ steps.active-deployments.outputs.prefixes }}
+        run: |
+          node scripts/cleanup-deployments.js cleanup "$GCP_BUCKET_NAME" "$DEPLOYMENTS_RETENTION_DAYS"
+
       - name: Update AI Help index with macros
         working-directory: mdn/dex
         run: npm run ai-help-macros -- update-index "$FRED_ROOT/out/en-us/docs"

.github/workflows/stage-build.yml

@@ -49,6 +49,10 @@ jobs:
           GH_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }}
 
   build:
+    # Queue runs so a delayed older run can't deploy over a newer one.
+    concurrency:
+      group: stage-deploy
+      cancel-in-progress: false
     environment: stage
     runs-on: ubuntu-latest-dex-builder
 
@@ -438,6 +442,40 @@ jobs:
             wait $pid
           done
 
+      - name: Read active deployment prefixes
+        id: active-deployments
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        working-directory: mdn/dex
+        env:
+          GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }}
+        run: |
+          prefixes="$(node scripts/cleanup-deployments.js get-active "$GCP_BUCKET_NAME" \
+            mdn-nonprod-stage-europe-west1@europe-west1 \
+            mdn-nonprod-stage-us-west1@us-west1 \
+            mdn-nonprod-stage-asia-east1@asia-east1)"
+          echo "prefixes=$prefixes" >> "$GITHUB_OUTPUT"
+
+      - name: Authenticate for deployment cleanup
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
+        with:
+          token_format: access_token
+          service_account: deploy-stage-content@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
+          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions
+
+      - name: Clean up old deployments
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        working-directory: mdn/dex
+        env:
+          GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }}
+          DEPLOYMENTS_RETENTION_DAYS: ${{ vars.DEPLOYMENTS_RETENTION_DAYS || '30' }}
+          ACTIVE_DEPLOYMENTS: ${{ steps.active-deployments.outputs.prefixes }}
+        run: |
+          node scripts/cleanup-deployments.js cleanup "$GCP_BUCKET_NAME" "$DEPLOYMENTS_RETENTION_DAYS"
+
       - name: Update AI Help index with macros
         working-directory: mdn/dex
         run: npm run ai-help-macros -- update-index "$FRED_ROOT/out/en-us/docs"

.github/workflows/test-build.yml

@@ -64,6 +64,10 @@ jobs:
           GH_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }}
 
   build:
+    # Queue runs so a delayed older run can't deploy over a newer one.
+    concurrency:
+      group: test-deploy
+      cancel-in-progress: false
     environment: test
     runs-on: ubuntu-latest-dex-builder
 
@@ -467,6 +471,38 @@ jobs:
             wait $pid
           done
 
+      - name: Read active deployment prefixes
+        id: active-deployments
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        working-directory: mdn/dex
+        env:
+          GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }}
+        run: |
+          prefixes="$(node scripts/cleanup-deployments.js get-active "$GCP_BUCKET_NAME" \
+            mdn-nonprod-test-europe-west3@europe-west3)"
+          echo "prefixes=$prefixes" >> "$GITHUB_OUTPUT"
+
+      - name: Authenticate for deployment cleanup
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
+        with:
+          token_format: access_token
+          service_account: deploy-test-content@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
+          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions
+
+      - name: Clean up old deployments
+        if: ${{ ! vars.SKIP_FUNCTION }}
+        continue-on-error: true
+        working-directory: mdn/dex
+        env:
+          GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }}
+          DEPLOYMENTS_RETENTION_DAYS: ${{ vars.DEPLOYMENTS_RETENTION_DAYS || '30' }}
+          ACTIVE_DEPLOYMENTS: ${{ steps.active-deployments.outputs.prefixes }}
+        run: |
+          node scripts/cleanup-deployments.js cleanup "$GCP_BUCKET_NAME" "$DEPLOYMENTS_RETENTION_DAYS"
+
       - name: Invalidate CDN
         if: ${{ github.event.inputs.invalidate }}
         run: gcloud compute url-maps invalidate-cdn-cache ${{ secrets.GCP_LOAD_BALANCER_NAME }} --path "/*" --async

scripts/cleanup-deployments.js

@@ -0,0 +1,226 @@
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const DEPLOYMENTS_PREFIX = "deployments/";
+const STDERR_TAIL_LINES = 20;
+
+/**
+ * Runs gcloud and returns its stdout. Only the tail of stderr is kept, since
+ * bulk storage operations print a status line per object.
+ * @param {string[]} args
+ * @returns {Promise<string>}
+ */
+function runGcloud(args) {
+  return new Promise((resolve, reject) => {
+    const child = spawn("gcloud", args, { stdio: ["ignore", "pipe", "pipe"] });
+    /** @type {Buffer[]} */
+    const stdout = [];
+    /** @type {string[]} */
+    let stderrTail = [];
+    child.stdout.on("data", (chunk) => stdout.push(chunk));
+    child.stderr.on("data", (chunk) => {
+      stderrTail = [...stderrTail, ...String(chunk).split("\n")].slice(
+        -STDERR_TAIL_LINES
+      );
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve(Buffer.concat(stdout).toString("utf8").trim());
+      } else {
+        reject(
+          new Error(
+            `gcloud ${args.slice(0, 3).join(" ")} exited with ${code}:\n${stderrTail.join("\n")}`
+          )
+        );
+      }
+    });
+  });
+}
+
+/** @param {string} bucketName @param {string} target @returns {Promise<string>} */
+async function getActiveDeployment(bucketName, target) {
+  const separator = target.lastIndexOf("@");
+  if (separator < 1 || separator === target.length - 1) {
+    throw new Error(`Expected function@region, got: ${target}`);
+  }
+
+  const functionName = target.slice(0, separator);
+  const region = target.slice(separator + 1);
+  const description = await runGcloud([
+    "beta",
+    "functions",
+    "describe",
+    functionName,
+    "--gen2",
+    `--region=${region}`,
+    "--format=json",
+  ]);
+  const { serviceConfig } = JSON.parse(description);
+  const sourceContent = serviceConfig?.environmentVariables?.SOURCE_CONTENT;
+  const deploymentBaseUrl = `https://storage.googleapis.com/${bucketName}/${DEPLOYMENTS_PREFIX}`;
+
+  if (!sourceContent?.startsWith(deploymentBaseUrl)) {
+    throw new Error(
+      `${functionName} in ${region} does not use a deployment prefix: ${sourceContent ?? "missing"}`
+    );
+  }
+
+  const deployment = sourceContent
+    .slice(deploymentBaseUrl.length)
+    .replace(/\/$/, "");
+  if (getDeploymentTime(deployment) === null) {
+    throw new Error(
+      `Unexpected deployment prefix for ${functionName} in ${region}: ${deployment}`
+    );
+  }
+  return deployment;
+}
+
+/** @param {string} deployment @returns {number | null} */
+function getDeploymentTime(deployment) {
+  const match = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})-\d+-\d+$/.exec(
+    deployment
+  );
+  if (!match) {
+    return null;
+  }
+
+  const [, year, month, day, hour, minute, second] = match.map(Number);
+  return Date.UTC(year, month - 1, day, hour, minute, second);
+}
+
+/** @param {string} bucketName @param {string[]} targets @returns {Promise<string[]>} */
+async function getActiveDeployments(bucketName, targets) {
+  const deployments = await Promise.all(
+    targets.map((target) => getActiveDeployment(bucketName, target))
+  );
+  return [...new Set(deployments)];
+}
+
+/**
+ * Lists the immediate children of `deployments/` as bucket-relative prefixes.
+ * @param {string} bucketName
+ * @returns {Promise<string[]>}
+ */
+async function listDeploymentFolders(bucketName) {
+  const baseUrl = `gs://${bucketName}/`;
+  const listing = await runGcloud([
+    "storage",
+    "ls",
+    `${baseUrl}${DEPLOYMENTS_PREFIX}`,
+  ]);
+  return listing
+    .split("\n")
+    .filter((line) => line.startsWith(baseUrl) && line.endsWith("/"))
+    .map((line) => line.slice(baseUrl.length));
+}
+
+/**
+ * Sorts deployment folders into those to delete and those to keep.
+ * @param {string[]} folderPrefixes bucket-relative, e.g. `deployments/20261009-120000-1-1/`
+ * @param {{ cutoff: number, activeDeployments: Set<string> }} options
+ * @returns {{ expired: string[], retained: string[], unrecognized: string[] }}
+ */
+export function classifyDeployments(
+  folderPrefixes,
+  { cutoff, activeDeployments }
+) {
+  /** @type {{ expired: string[], retained: string[], unrecognized: string[] }} */
+  const result = { expired: [], retained: [], unrecognized: [] };
+
+  for (const folderPrefix of folderPrefixes) {
+    if (!folderPrefix.startsWith(DEPLOYMENTS_PREFIX)) {
+      throw new Error(
+        `Unexpected deployment prefix from Cloud Storage: ${folderPrefix}`
+      );
+    }
+
+    const deployment = folderPrefix
+      .slice(DEPLOYMENTS_PREFIX.length)
+      .replace(/\/$/, "");
+    const deploymentTime = getDeploymentTime(deployment);
+    if (deploymentTime === null) {
+      result.unrecognized.push(folderPrefix);
+    } else if (deploymentTime < cutoff && !activeDeployments.has(deployment)) {
+      result.expired.push(folderPrefix);
+    } else {
+      result.retained.push(folderPrefix);
+    }
+  }
+
+  return result;
+}
+
+/** @param {string} bucketName @param {number} retentionDays @param {Set<string>} activeDeployments */
+async function cleanupDeployments(
+  bucketName,
+  retentionDays,
+  activeDeployments
+) {
+  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+  const { expired, retained, unrecognized } = classifyDeployments(
+    await listDeploymentFolders(bucketName),
+    { cutoff, activeDeployments }
+  );
+
+  for (const folderPrefix of unrecognized) {
+    console.log(
+      `Skipping unrecognized deployment folder: gs://${bucketName}/${folderPrefix}`
+    );
+  }
+
+  for (const folderPrefix of expired) {
+    const url = `gs://${bucketName}/${folderPrefix}`;
+    console.log(`Deleting expired deployment: ${url}`);
+    await runGcloud(["storage", "rm", "--recursive", url]);
+  }
+
+  console.log(
+    `Deleted ${expired.length} expired, retained ${retained.length}, skipped ${unrecognized.length} deployment folders`
+  );
+}
+
+async function main() {
+  const [command, bucketName, ...args] = process.argv.slice(2);
+  if (!bucketName) {
+    throw new Error("A bucket name is required");
+  }
+
+  if (command === "get-active") {
+    if (args.length === 0) {
+      throw new Error("At least one function@region target is required");
+    }
+    console.log(JSON.stringify(await getActiveDeployments(bucketName, args)));
+    return;
+  }
+
+  if (command === "cleanup") {
+    const [retentionDaysValue] = args;
+    const retentionDays = Number(retentionDaysValue);
+    if (!Number.isInteger(retentionDays) || retentionDays < 1) {
+      throw new Error(
+        `Retention days must be a positive integer: ${retentionDaysValue}`
+      );
+    }
+    const activeDeployments = new Set(
+      JSON.parse(process.env["ACTIVE_DEPLOYMENTS"] ?? "[]")
+    );
+    if (activeDeployments.size === 0) {
+      throw new Error("No active deployments provided, refusing cleanup");
+    }
+    await cleanupDeployments(bucketName, retentionDays, activeDeployments);
+    return;
+  }
+
+  throw new Error(
+    "Usage: cleanup-deployments.js <get-active|cleanup> <bucket> ..."
+  );
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  main().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+  });
+}