Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion actions/setup/js/check_daily_aic_workflow_guardrail.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const { getErrorMessage } = require("./error_helpers.cjs");
const { createRateLimitAwareGithub } = require("./github_rate_limit_logger.cjs");
const { scanDailyAIC } = require("./daily_aic_scan.cjs");
const { API_READ_OPERATIONS, createAPIBudget, retryNotBefore, safeResponseHeaders, isQuotaError, getAPIReadFailure } = require("./daily_aic_api_budget.cjs");
const { getGitHubAPIDiagnostic } = require("./github_api_diagnostics.cjs");
const { loadBillableJobs, allBillableJobsNotStarted, allBillableJobsSkipped, sumCoveredComponents } = require("./daily_aic_component_coverage.cjs");
const { readLedgerEntries } = require("./daily_aic_repo_memory_ledger.cjs");

Expand Down Expand Up @@ -843,7 +844,8 @@ async function main(options = {}) {
const reason = readFailure
? `${readFailure.operation} read stopped (${readFailure.status ? `HTTP ${readFailure.status}` : "transport or read error"}; attempts: ${readFailure.attempts}; reason: ${readFailure.reason}). Accounting remains unverified; agent startup is blocked.`
: getErrorMessage(error);
const message = `Daily workflow AI Credits are unknown: ${reason}`;
const diagnostic = getGitHubAPIDiagnostic(error);
const message = `Daily workflow AI Credits are unknown: ${reason}${diagnostic ? ` ${diagnostic.message}` : ""}`;
core.setOutput("daily_ai_credits_guardrail_status", status);
core.setOutput("daily_ai_credits_guardrail_error", message);
logDailyGuardrail("AIC inspection failed", { status, error: readFailure || getErrorMessage(error) });
Expand Down
46 changes: 45 additions & 1 deletion actions/setup/js/check_daily_aic_workflow_guardrail.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,7 @@ describe("check_daily_aic_workflow_guardrail", () => {
["repository_history", 502, {}, 3, "retry_exhausted", "transient_error"],
["workflow_history", 503, { "retry-after": "120" }, 1, "retry_restricted", "transient_error"],
["workflow_history", 502, { "x-ratelimit-remaining": "100" }, 1, "quota_stop", "transient_error"],
["workflow_history", 403, { "x-ratelimit-remaining": "0" }, 1, "quota_stop", "transient_error"],
["workflow_history", 403, {}, 1, "non_transient", "structural_error"],
])("main() safely attributes %s HTTP %i while keeping unknown history blocked", async (operation, httpStatus, headers, attempts, reason, guardrailStatus) => {
vi.useFakeTimers();
Expand Down Expand Up @@ -534,7 +535,13 @@ describe("check_daily_aic_workflow_guardrail", () => {
const pending = runMain();
await vi.runAllTimersAsync();
await pending;
const message = `Daily workflow AI Credits are unknown: ${operation} read stopped (HTTP ${httpStatus}; attempts: ${attempts}; reason: ${reason}). Accounting remains unverified; agent startup is blocked.`;
const diagnostic =
reason === "quota_stop"
? ` GitHub API ${headers["x-ratelimit-remaining"] === "0" ? "quota_exhausted" : "quota_reserve"} (HTTP ${httpStatus}; resource=unknown; remaining=${headers["x-ratelimit-remaining"]}; limit=unknown; used=unknown; reset=unknown; retry_not_before=unknown)`
: httpStatus === 403
? " GitHub API permission_denied (HTTP 403; Check the credential's access to this operation; quota exhaustion was not established.)"
: "";
const message = `Daily workflow AI Credits are unknown: ${operation} read stopped (HTTP ${httpStatus}; attempts: ${attempts}; reason: ${reason}). Accounting remains unverified; agent startup is blocked.${diagnostic}`;
expect(outputs).toMatchObject({ daily_ai_credits_guardrail_status: guardrailStatus, daily_ai_credits_guardrail_error: message, daily_ai_credits_total: "", daily_ai_credits_estimated: "", daily_ai_credits_exceeded: "false" });
expect(global.core.setFailed).toHaveBeenCalledWith(message);
expect(failedRead).toHaveBeenCalledTimes(attempts);
Expand All @@ -552,6 +559,43 @@ describe("check_daily_aic_workflow_guardrail", () => {
}
});

it("main() keeps wrapped quota failures classified and blocks unknown accounting", async () => {
const outputs = {};
const quotaError = httpError(403, "API rate limit exceeded for installation; private-payload");
quotaError.response.headers = { "x-ratelimit-remaining": "0", "x-ratelimit-reset": "1791972360" };
const wrapped = new Error("unable to download artifact: private-payload", { cause: quotaError });
const getCurrent = vi.fn().mockRejectedValue(wrapped);
global.core = {
setOutput: (key, value) => {
outputs[key] = value;
},
setFailed: vi.fn(),
info: vi.fn(),
warning: vi.fn(),
};
global.github = { rest: { actions: { getWorkflowRun: getCurrent } } };
global.context = { repo: { owner: "test-owner", repo: "test-repo" }, runId: 42 };
process.env.GH_AW_MAX_DAILY_AI_CREDITS = "1000";
process.env.GH_AW_GITHUB_TOKEN = "synthetic";
try {
await runMain();
expect(outputs.daily_ai_credits_guardrail_status).toBe("transient_error");
expect(outputs.daily_ai_credits_total).toBe("");
expect(outputs.daily_ai_credits_guardrail_error).toContain("current_run read stopped");
expect(outputs.daily_ai_credits_guardrail_error).toContain("quota_exhausted");
expect(outputs.daily_ai_credits_guardrail_error).toContain("reset=2026-10-14T10:06:00.000Z");
expect(global.core.setFailed).toHaveBeenCalledWith(outputs.daily_ai_credits_guardrail_error);
expect(getCurrent).toHaveBeenCalledOnce();
expect(JSON.stringify([outputs, global.core.info.mock.calls, global.core.setFailed.mock.calls])).not.toContain("private-payload");
} finally {
delete global.core;
delete global.github;
delete global.context;
delete process.env.GH_AW_MAX_DAILY_AI_CREDITS;
delete process.env.GH_AW_GITHUB_TOKEN;
}
});

it("main() counts business requests instead of unrelated rate-limit snapshots", async () => {
// The unrelated quota endpoint must not supply the business-request budget.
const coreInfos = [];
Expand Down
34 changes: 19 additions & 15 deletions actions/setup/js/daily_aic_api_budget.cjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// @ts-check

const SAFE_HEADERS = ["x-ratelimit-limit", "x-ratelimit-remaining", "x-ratelimit-used", "x-ratelimit-reset", "retry-after"];
const { getGitHubAPIQuota, getGitHubAPIDiagnostic, getSanitizedGitHubAPICause } = require("./github_api_diagnostics.cjs");

const SAFE_HEADERS = ["x-ratelimit-limit", "x-ratelimit-remaining", "x-ratelimit-used", "x-ratelimit-reset", "x-ratelimit-resource", "retry-after"];
const API_READ_OPERATIONS = Object.freeze({
CURRENT_RUN: "current_run",
WORKFLOW_HISTORY: "workflow_history",
Expand Down Expand Up @@ -37,12 +39,7 @@ function apiError(status, headers, message) {
}

function retryNotBefore(headers, now = Date.now(), quotaStopped = false) {
const safe = safeResponseHeaders(headers);
const reset = quotaStopped || safe["x-ratelimit-remaining"] === "0" ? Number(safe["x-ratelimit-reset"]) * 1000 : NaN;
const retry = safe["retry-after"];
const retryAt = retry == null ? NaN : /^\d+$/.test(retry) ? now + Number(retry) * 1000 : Date.parse(retry);
const times = [reset, retryAt].filter(value => Number.isFinite(value) && value > now);
return times.length ? new Date(Math.max(...times)).toISOString() : "";
return getGitHubAPIQuota(headers, now, quotaStopped).retryNotBefore;
}

function getErrorStatus(error) {
Expand All @@ -52,7 +49,7 @@ function getErrorStatus(error) {
function isQuotaError(error) {
const headers = safeResponseHeaders(error?.response?.headers);
const status = getErrorStatus(error);
return error?.code === "DAILY_AIC_QUOTA_STOP" || status === 429 || (status === 403 && (headers["x-ratelimit-remaining"] === "0" || headers["retry-after"] != null));
return error?.code === "DAILY_AIC_QUOTA_STOP" || status === 429 || (status === 403 && (headers["x-ratelimit-remaining"] === "0" || headers["retry-after"] != null)) || getGitHubAPIDiagnostic(error)?.quota === true;
}

function isTransientError(error) {
Expand All @@ -72,13 +69,17 @@ function createAPIBudget(reserve = 100) {
let requests = 0;
let latest = {};
const budget = {
observe(response) {
observe(response, cause = undefined) {
requests++;
latest = safeResponseHeaders(response.headers);
const remaining = latest["x-ratelimit-remaining"];
const limited = response.status === 429 || (response.status === 403 && (remaining === "0" || latest["retry-after"] != null));
if (limited || (remaining != null && Number(remaining) <= reserve)) {
throw Object.assign(apiError(response.status, latest, "Daily AIC inspection stopped to preserve GitHub API quota"), { code: "DAILY_AIC_QUOTA_STOP" });
const sanitizedCause = getSanitizedGitHubAPICause(cause);
const error = Object.assign(apiError(response.status, latest, "Daily AIC inspection stopped to preserve GitHub API quota"), { code: "DAILY_AIC_QUOTA_STOP", ...(sanitizedCause ? { cause: sanitizedCause } : {}) });
const diagnostic = getGitHubAPIDiagnostic(error);
if (diagnostic) error.message += `: ${diagnostic.message}`;
throw error;
}
},
async request(operation, observe = true, readOperation) {
Expand All @@ -92,7 +93,7 @@ function createAPIBudget(reserve = 100) {
try {
response = await operation();
} catch (error) {
if (observe && error?.response) budget.observe(error.response);
if (observe && error?.response) budget.observe(error.response, error);
throw error;
}
if (observe) budget.observe(response);
Expand All @@ -111,6 +112,8 @@ function createAPIBudget(reserve = 100) {
const failure = Object.freeze({ attempts: attempt + 1, status: status !== null && Number.isInteger(status) && status >= 100 && status <= 599 ? status : null, reason, requests, operation: operationName });
if (error && typeof error === "object") READ_FAILURES.set(error, failure);
core.info(`[daily-workflow-aic] Stopped daily AIC read: ${JSON.stringify(failure)}`);
const diagnostic = getGitHubAPIDiagnostic(error);
if (diagnostic) core.info(`[daily-workflow-aic] ${operationName}: ${diagnostic.message}`);
throw error;
}
const delayMs = 1000 * 2 ** attempt;
Expand All @@ -120,12 +123,13 @@ function createAPIBudget(reserve = 100) {
}
},
snapshot() {
const metadata = getGitHubAPIQuota(latest);
return {
requests,
remaining: Number(latest["x-ratelimit-remaining"] || 0),
limit: Number(latest["x-ratelimit-limit"] || 0),
used: Number(latest["x-ratelimit-used"] || 0),
reset: latest["x-ratelimit-reset"] ? new Date(Number(latest["x-ratelimit-reset"]) * 1000).toISOString() : "",
remaining: metadata.remaining ?? 0,
limit: metadata.limit ?? 0,
used: metadata.used ?? 0,
reset: metadata.reset,
};
},
};
Expand Down
86 changes: 86 additions & 0 deletions actions/setup/js/daily_aic_api_budget.test.cjs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { createRequire } from "node:module";
import { inspect } from "node:util";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";

const require = createRequire(import.meta.url);
const { API_READ_OPERATIONS, apiError, createAPIBudget, isTransientError, retryNotBefore, getAPIReadFailure } = require("./daily_aic_api_budget.cjs");
const { getGitHubAPIDiagnostic } = require("./github_api_diagnostics.cjs");

beforeEach(() => {
vi.useFakeTimers();
Expand All @@ -16,6 +18,27 @@ afterEach(() => {
const response = (status = 200, headers = { "x-ratelimit-remaining": "4000" }) => ({ status, headers });

describe("bounded daily AIC read retries", () => {
it("classifies a headerless installation rate limit without retrying or publishing its payload", async () => {
const error = apiError(403, {}, "API rate limit exceeded for installation; secret-test-value");
const request = vi.fn().mockRejectedValue(error);
await expect(createAPIBudget().request(request, true, API_READ_OPERATIONS.CURRENT_RUN)).rejects.toBe(error);
expect(getAPIReadFailure(error)).toMatchObject({ attempts: 1, reason: "quota_stop", operation: "current_run" });
expect(request).toHaveBeenCalledOnce();
const logs = global.core.info.mock.calls.flat().join("\n");
expect(logs).toContain("current_run: GitHub API rate_limited");
expect(logs).toContain("remaining=unknown");
expect(logs).not.toContain("secret-test-value");
});

it("does not let malformed reset metadata replace the original failure", async () => {
const request = vi.fn().mockRejectedValue(apiError(403, { "x-ratelimit-remaining": "0", "x-ratelimit-reset": "9999999999999999" }, "private payload"));
const budget = createAPIBudget();
await expect(budget.request(request)).rejects.toMatchObject({ code: "DAILY_AIC_QUOTA_STOP" });
expect(budget.snapshot().reset).toBe("");
expect(retryNotBefore({ "x-ratelimit-remaining": "0", "x-ratelimit-reset": "9999999999999999" })).toBe("");
expect(request).toHaveBeenCalledOnce();
});

it("retries only the failed read and observes each response once", async () => {
const budget = createAPIBudget();
const request = vi
Expand Down Expand Up @@ -58,6 +81,69 @@ describe("bounded daily AIC read retries", () => {
expect(global.core.info).toHaveBeenCalledWith(`[daily-workflow-aic] Stopped daily AIC read: {"attempts":1,"status":${status},"reason":"quota_stop","requests":1,"operation":"unknown"}`);
});

it("classifies a synthetic HTTP 403 reserve stop without permission evidence as quota_reserve", () => {
let stopped;
try {
createAPIBudget(100).observe(response(403, { "x-ratelimit-remaining": "50" }));
} catch (error) {
stopped = error;
}
expect(stopped).toMatchObject({ code: "DAILY_AIC_QUOTA_STOP", status: 403 });
expect(getGitHubAPIDiagnostic(stopped)).toMatchObject({ quota: true, classification: "quota_reserve" });
});

it.each([
[401, "Bad credentials", "50", "authentication_failed"],
[403, "Resource not accessible by integration", "50", "permission_denied"],
[403, "Resource not accessible by integration", "0", "permission_denied"],
[403, "API rate limit exceeded for installation", "0", "quota_exhausted"],
[403, "Unspecified error", "50", "quota_reserve"],
])("preserves original HTTP %i access evidence while stopping quota-restricted reads", async (status, message, remaining, classification) => {
const original = apiError(status, { "x-ratelimit-remaining": remaining }, `${message}; secret-test-value`);
const request = vi.fn().mockRejectedValue(original);
const stopped = await createAPIBudget()
.request(request)
.catch(error => error);
expect(stopped).toMatchObject({ code: "DAILY_AIC_QUOTA_STOP", status });
expect(getGitHubAPIDiagnostic(stopped)).toMatchObject({ quota: true, classification });
if (classification === "quota_exhausted" || classification === "quota_reserve") {
expect(stopped.cause).toBeUndefined();
} else {
expect(stopped.cause).not.toBe(original);
expect(stopped.cause).toMatchObject({ status, message: status === 401 ? "Bad credentials" : "Permission denied" });
expect(Object.keys(stopped.cause)).toEqual(["status"]);
}
expect(request).toHaveBeenCalledOnce();
expect(vi.getTimerCount()).toBe(0);
const logs = global.core.info.mock.calls.flat().join("\n");
expect(logs).toContain(`GitHub API ${classification}`);
expect(logs).toContain('"reason":"quota_stop"');
expect(logs).not.toContain("secret-test-value");
});

it.each([
[401, "Bad credentials", "authentication_failed"],
[403, "Resource not accessible by integration", "permission_denied"],
[403, "Resource not accessible by personal access token", "permission_denied"],
[403, "API rate limit exceeded", "quota_exhausted"],
])("never exposes original HTTP %i error properties during terminal inspection", async (status, message, classification) => {
const original = apiError(status, { "x-ratelimit-remaining": "0" }, `${message}; secret-message-value`);
original.request = { url: "https://example.invalid/?token=secret-url-value", headers: { authorization: "secret-request-header-value" }, body: "secret-request-body-value" };
original.response.headers.authorization = "secret-response-header-value";
original.response.headers.location = "https://example.invalid/?token=secret-location-value";
original.response.body = "secret-response-body-value";
original.cause = new Error("secret-nested-cause-value");
const stopped = await createAPIBudget()
.request(() => Promise.reject(original))
.catch(error => error);
expect(stopped).toMatchObject({ code: "DAILY_AIC_QUOTA_STOP", status });
expect(getGitHubAPIDiagnostic(stopped)).toMatchObject({ quota: true, classification });
const terminal = inspect(stopped, { depth: null, showHidden: true });
expect(terminal).not.toMatch(/secret-[\w-]+-value|example\.invalid/);
expect(terminal).toContain("DAILY_AIC_QUOTA_STOP");
expect(vi.getTimerCount()).toBe(0);
});

it("does not retry a server error with an explicit retry restriction", async () => {
const request = vi.fn(async () => response(503, { "retry-after": "120" }));
await expect(createAPIBudget().request(request)).rejects.toMatchObject({ status: 503 });
Expand Down
Loading
Loading