From 2fded7bcd9906db25be1ddc36ffc68b92a47610a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 13:17:37 +0000 Subject: [PATCH 1/6] ci: report client requests that do not match the OpenAPI specification Add `tools/openapi-request-validator`, a request-only validator built on `express-openapi-validator`: an Express middleware the unit-test mock server mounts when `OPENAPI_SPEC_PATH` is set, a validating reverse proxy the integration tests run through, and a report script that aggregates the findings and emits GitHub Actions warning annotations. Both CI jobs fetch the published specification, run the tests with validation on, and print the report. Nothing is blocked or failed on a finding; the result is a warning while the specification and the client catch up with each other. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017ocuALGHVYTScW9Ht9Ytjo --- .github/workflows/check.yaml | 30 +++ oxlint.config.ts | 2 +- package.json | 3 + pnpm-lock.yaml | 121 ++++++++++- test/mock_server/server.ts | 13 ++ tools/openapi-request-validator/README.md | 26 +++ tools/openapi-request-validator/proxy.mts | 136 ++++++++++++ tools/openapi-request-validator/report.mts | 111 ++++++++++ tools/openapi-request-validator/validator.mts | 202 ++++++++++++++++++ 9 files changed, 640 insertions(+), 4 deletions(-) create mode 100644 tools/openapi-request-validator/README.md create mode 100644 tools/openapi-request-validator/proxy.mts create mode 100644 tools/openapi-request-validator/report.mts create mode 100644 tools/openapi-request-validator/validator.mts diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index e0c6b2266..ecebddde3 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -48,8 +48,19 @@ jobs: - name: Install Chrome for puppeteer run: pnpm exec puppeteer browsers install chrome + - name: Fetch the OpenAPI specification + run: pnpm spec:fetch + + # The mock server also checks every request against the specification, see `tools/openapi-request-validator`. - name: Run Tests run: pnpm test + env: + OPENAPI_SPEC_PATH: tmp/openapi.json + OPENAPI_REPORT_FILE: openapi-findings.jsonl + + - name: Report requests that do not match the OpenAPI specification + if: always() + run: pnpm openapi:report openapi-findings.jsonl integration_tests: name: Integration tests (Node ${{ matrix.node-version }}) @@ -92,9 +103,28 @@ jobs: # A workflow that calls this one as a reusable workflow - the release flow does - gets no # secrets unless it opts in with `secrets: inherit`, so the token can legitimately be empty. # Skip rather than fail there, otherwise a missing token blocks the release. + - name: Fetch the OpenAPI specification + if: ${{ env.APIFY_TEST_USER_API_TOKEN != '' }} + run: pnpm spec:fetch + + # The tests talk to the live API through a proxy that checks every request against the + # specification, see `tools/openapi-request-validator`. The proxy keeps running into the next steps. + - name: Start the OpenAPI validating proxy + if: ${{ env.APIFY_TEST_USER_API_TOKEN != '' }} + run: | + pnpm openapi:proxy --spec tmp/openapi.json --port 3999 --report openapi-findings.jsonl > openapi-proxy.log 2>&1 & + for _ in $(seq 1 30); do grep -q listening openapi-proxy.log && break; sleep 1; done + cat openapi-proxy.log + - name: Run integration tests if: ${{ env.APIFY_TEST_USER_API_TOKEN != '' }} run: pnpm test:integration + env: + APIFY_INTEGRATION_TESTS_API_URL: http://127.0.0.1:3999 + + - name: Report requests that do not match the OpenAPI specification + if: ${{ always() && env.APIFY_TEST_USER_API_TOKEN != '' }} + run: pnpm openapi:report openapi-findings.jsonl - name: Report the skipped integration tests if: ${{ env.APIFY_TEST_USER_API_TOKEN == '' }} diff --git a/oxlint.config.ts b/oxlint.config.ts index 72195d634..1bde542cf 100644 --- a/oxlint.config.ts +++ b/oxlint.config.ts @@ -25,7 +25,7 @@ export default defineConfig({ }, { // Maintainer-facing CLI scripts, so reporting progress on stdout is the point. - files: ['scripts/**'], + files: ['scripts/**', 'tools/**'], rules: { 'no-console': 'off', }, diff --git a/package.json b/package.json index 0d550f72a..aa0a266fa 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,8 @@ "build:node": "tsc", "build:browser": "rsbuild build", "spec:fetch": "node scripts/openapi_spec.mts fetch", + "openapi:proxy": "node tools/openapi-request-validator/proxy.mts", + "openapi:report": "node tools/openapi-request-validator/report.mts", "generate:models": "node scripts/openapi_spec.mts fetch && node scripts/generate_types.mts && node scripts/generate_schemas.mts && node scripts/openapi_spec.mts record-version" }, "apify": { @@ -113,6 +115,7 @@ "compression": "^1.8.2", "esbuild": "0.28.2", "express": "^5.2.1", + "express-openapi-validator": "5.6.2", "oxfmt": "0.73.0", "oxlint": "1.89.0", "oxlint-tsgolint": "7.0.2005", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3a281b144..c9111e9c2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -253,6 +253,9 @@ importers: express: specifier: ^5.2.1 version: 5.2.1(supports-color@10.2.2) + express-openapi-validator: + specifier: 5.6.2 + version: 5.6.2(@types/json-schema@7.0.15)(express@5.2.1(supports-color@10.2.2)) oxfmt: specifier: 0.73.0 version: 0.73.0 @@ -513,6 +516,12 @@ packages: '@algolia/transporter@4.27.0': resolution: {integrity: sha512-PvSbELU4VjN3xSX79ki+zIdOGhTxyJXWvRDzkUjfTx2iNfPWDdTjzKbP1o+268coJztxrkuBwJz90Urek7o1Kw==} + '@apidevtools/json-schema-ref-parser@14.2.1': + resolution: {integrity: sha512-HmdFw9CDYqM6B25pqGBpNeLCKvGPlIx1EbLrVL0zPvj50CJQUHyBNBw45Muk0kEIkogo1VZvOKHajdMuAzSxRg==} + engines: {node: '>= 20'} + peerDependencies: + '@types/json-schema': ^7.0.15 + '@apify/consts@3.0.1': resolution: {integrity: sha512-ZPmfdb4qt143R+6/jFYVCQDoHGInl24Ntz8ltEAMW7tFHSdiXnQoZtDgydZYT06vw6gOtPa/jEXgSneq8FM+uA==} engines: {node: '>=22'} @@ -2416,6 +2425,9 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@jsdevtools/ono@7.1.3': + resolution: {integrity: sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==} + '@jsonjoy.com/base64@1.1.2': resolution: {integrity: sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==} engines: {node: '>=10.0'} @@ -4101,6 +4113,9 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/multer@2.3.0': + resolution: {integrity: sha512-i7STIm9V4K2MPH4ZYMtrZAwNxs3kglk2LgleaTuB9pUXgADBcQYQuYMd7+6xgTIxfoggIkFA/DkkvycdZpIARA==} + '@types/node@17.0.45': resolution: {integrity: sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==} @@ -4426,8 +4441,21 @@ packages: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} + ajv-draft-04@1.0.0: + resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} + peerDependencies: + ajv: ^8.5.0 + peerDependenciesMeta: + ajv: + optional: true + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true ajv-keywords@3.5.2: resolution: {integrity: sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==} @@ -4493,6 +4521,9 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} @@ -4729,6 +4760,10 @@ packages: resolution: {integrity: sha512-DdH81/zPLVS11EUgWq3tEu/xn+EzljlMYooDNdzWEnFha3R3NBMpMV1UqYIpjYHV/SgpFKMIX1Oh7o06SjM/oA==} engines: {node: '>=18'} + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + bytes@3.0.0: resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} engines: {node: '>= 0.8'} @@ -5613,6 +5648,11 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + express-openapi-validator@5.6.2: + resolution: {integrity: sha512-fkDn4+ImUC4HTJ1g0cek/ItqYhmEO19AglJd2Iw2OJco0jLIbxIlDGVazmXbvvYeziU4Bnah2h+S2tb6NtWg8w==} + peerDependencies: + express: '*' + express@4.22.3: resolution: {integrity: sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw==} engines: {node: '>= 0.10.0'} @@ -6640,9 +6680,16 @@ packages: resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lodash.clonedeep@4.5.0: + resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==} + lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} + lodash.get@4.4.2: + resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} + deprecated: This package is deprecated. Use the optional chaining (?.) operator instead. + lodash.isequal@4.5.0: resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead. @@ -7124,6 +7171,10 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + multer@2.5.0: + resolution: {integrity: sha512-BsqOzYGVq49JcW2IzoSAxeKYRevb+dGGPPWleOR8Lrlk4igwxu1/9pwSXlByhCb7MVZDrNJFC0IBP5/O5nLYRw==} + engines: {node: '>= 10.16.0'} + multicast-dns@7.2.5: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true @@ -7234,6 +7285,9 @@ packages: oniguruma-to-es@2.3.0: resolution: {integrity: sha512-bwALDxriqfKGfUufKGGepCzu9x7nJQuoRoAFp4AnwehhC2crqrDIAP/uN2qdlsAvSMpeRC3+Yzhqc7hLmle5+g==} + ono@7.1.3: + resolution: {integrity: sha512-9jnfVriq7uJM4o5ganUY54ntUm+5EK21EGaQ5NWnkWg3zz5ywbbonlBguRcnmF1/HDiIe3zxNxXcO1YPBmPcQQ==} + open@10.2.0: resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} engines: {node: '>=18'} @@ -8757,6 +8811,10 @@ packages: stream-json@3.7.0: resolution: {integrity: sha512-rCSBdcBP/bPk6T8QFcxAj1MSzAuc5i49cYW6IE7sYObNEPccBJIUiL6fU9c9BWt40aJK0siVynLX7lWaW8alXw==} + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -9871,6 +9929,11 @@ snapshots: '@algolia/logger-common': 4.27.0 '@algolia/requester-common': 4.27.0 + '@apidevtools/json-schema-ref-parser@14.2.1(@types/json-schema@7.0.15)': + dependencies: + '@types/json-schema': 7.0.15 + js-yaml: 5.4.2 + '@apify/consts@3.0.1': {} '@apify/datastructures@3.0.1': {} @@ -12974,6 +13037,8 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.6.0 + '@jsdevtools/ono@7.1.3': {} + '@jsonjoy.com/base64@1.1.2(tslib@2.8.1)': dependencies: tslib: 2.8.1 @@ -14462,6 +14527,10 @@ snapshots: '@types/ms@2.1.0': {} + '@types/multer@2.3.0': + dependencies: + '@types/express': 5.0.6 + '@types/node@17.0.45': {} '@types/node@24.19.0': @@ -14755,8 +14824,12 @@ snapshots: clean-stack: 2.2.0 indent-string: 4.0.0 - ajv-formats@3.0.1: - dependencies: + ajv-draft-04@1.0.0(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + + ajv-formats@3.0.1(ajv@8.20.0): + optionalDependencies: ajv: 8.20.0 ajv-keywords@3.5.2(ajv@6.15.0): @@ -14847,6 +14920,8 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.2 + append-field@1.0.0: {} + arg@5.0.2: {} argparse@2.0.1: {} @@ -15175,6 +15250,10 @@ snapshots: dependencies: run-applescript: 7.1.0 + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + bytes@3.0.0: {} bytes@3.1.2: {} @@ -16121,6 +16200,26 @@ snapshots: expect-type@1.4.0: {} + express-openapi-validator@5.6.2(@types/json-schema@7.0.15)(express@5.2.1(supports-color@10.2.2)): + dependencies: + '@apidevtools/json-schema-ref-parser': 14.2.1(@types/json-schema@7.0.15) + '@types/multer': 2.3.0 + ajv: 8.20.0 + ajv-draft-04: 1.0.0(ajv@8.20.0) + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + express: 5.2.1(supports-color@10.2.2) + json-schema-traverse: 1.0.0 + lodash.clonedeep: 4.5.0 + lodash.get: 4.4.2 + media-typer: 1.1.1 + multer: 2.5.0 + ono: 7.1.3 + path-to-regexp: 8.4.2 + qs: 6.16.0 + transitivePeerDependencies: + - '@types/json-schema' + express@4.22.3(supports-color@10.2.2): dependencies: accepts: 1.3.8 @@ -17370,8 +17469,12 @@ snapshots: dependencies: p-locate: 6.0.0 + lodash.clonedeep@4.5.0: {} + lodash.debounce@4.0.8: {} + lodash.get@4.4.2: {} + lodash.isequal@4.5.0: {} lodash.memoize@4.1.2: {} @@ -18173,6 +18276,12 @@ snapshots: ms@2.1.3: {} + multer@2.5.0: + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + type-is: 1.6.18 + multicast-dns@7.2.5: dependencies: dns-packet: 5.6.1 @@ -18270,6 +18379,10 @@ snapshots: regex: 5.1.1 regex-recursion: 5.1.1 + ono@7.1.3: + dependencies: + '@jsdevtools/ono': 7.1.3 + open@10.2.0: dependencies: default-browser: 5.5.1 @@ -19907,7 +20020,7 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 ajv: 8.20.0 - ajv-formats: 3.0.1 + ajv-formats: 3.0.1(ajv@8.20.0) ajv-keywords: 5.1.0(ajv@8.20.0) search-insights@2.17.3: {} @@ -20280,6 +20393,8 @@ snapshots: dependencies: stream-chain: 4.2.6 + streamsearch@1.1.0: {} + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 diff --git a/test/mock_server/server.ts b/test/mock_server/server.ts index cd27e9e91..de09e6260 100644 --- a/test/mock_server/server.ts +++ b/test/mock_server/server.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs'; import type { Server } from 'node:http'; import http from 'node:http'; import path from 'node:path'; @@ -7,6 +8,8 @@ import bodyParser from 'body-parser'; import compression from 'compression'; import express from 'express'; +import { createJsonlSink } from '../../tools/openapi-request-validator/report.mts'; +import { createRequestValidator } from '../../tools/openapi-request-validator/validator.mts'; import * as fixtures from './fixtures.js'; // Routers import { actors } from './routes/actors.js'; @@ -93,6 +96,16 @@ export function createDefaultApp(v2Router = express.Router()) { mockServer.requests.push(req); next(); }); + // With `OPENAPI_SPEC_PATH` set, every request the client sends is also checked against the OpenAPI + // specification. Findings are appended to `OPENAPI_REPORT_FILE`, see `tools/openapi-request-validator`. + if (process.env.OPENAPI_SPEC_PATH) { + const { OPENAPI_SPEC_PATH: specPath, OPENAPI_REPORT_FILE: reportFile } = process.env; + const validator = createRequestValidator({ + spec: JSON.parse(readFileSync(specPath, 'utf8')), + onFinding: reportFile ? createJsonlSink(reportFile) : undefined, + }); + app.use(validator.middleware); + } app.use('/v2', v2Router); app.use('/external', external); diff --git a/tools/openapi-request-validator/README.md b/tools/openapi-request-validator/README.md new file mode 100644 index 000000000..86c651cf4 --- /dev/null +++ b/tools/openapi-request-validator/README.md @@ -0,0 +1,26 @@ +# OpenAPI request validator + +Checks the requests the client sends against the Apify API OpenAPI specification and reports every mismatch. +Only requests are validated; responses are passed through untouched. Nothing is ever blocked, the result is a +report, so it works as a warning in CI while the specification and the client catch up with each other. + +- `validator.mts` is an Express middleware. The unit-test mock server mounts it when `OPENAPI_SPEC_PATH` is set + and appends findings to `OPENAPI_REPORT_FILE`. +- `proxy.mts` is a validating reverse proxy for the integration tests. It forwards every request to the live API + and records the findings on the way. `GET /__openapi-report` returns them as JSON. +- `report.mts` aggregates a findings file by operation and error, prints it, and emits GitHub Actions warning + annotations in CI. `--fail` makes it exit non-zero when there is any finding. + +```sh +pnpm spec:fetch +# Unit tests +OPENAPI_SPEC_PATH=tmp/openapi.json OPENAPI_REPORT_FILE=openapi-findings.jsonl pnpm test +# Integration tests +pnpm openapi:proxy --spec tmp/openapi.json --port 3999 --report openapi-findings.jsonl & +APIFY_INTEGRATION_TESTS_API_URL=http://127.0.0.1:3999 pnpm test:integration +pnpm openapi:report openapi-findings.jsonl +``` + +Known false positives of the underlying `express-openapi-validator` are filtered, the same way the API server's +own monitoring does it: `0` and `1` for boolean query parameters, internal query parameters, the undocumented +legacy `/v2/acts` alias, and `+` encoding a space in a query value. diff --git a/tools/openapi-request-validator/proxy.mts b/tools/openapi-request-validator/proxy.mts new file mode 100644 index 000000000..e04ffdf37 --- /dev/null +++ b/tools/openapi-request-validator/proxy.mts @@ -0,0 +1,136 @@ +/** + * Validating reverse proxy for the Apify API. + * + * Every request is validated against the OpenAPI specification (requests only) and then forwarded unchanged to the + * upstream API; the upstream response is passed back to the client. Point a client's integration tests at the + * proxy to collect every request they make that does not match the specification. + * + * node proxy.mts --spec tmp/openapi.json --upstream https://api.apify.com --port 3999 --report findings.jsonl + * + * `GET /__openapi-report` returns the findings collected so far as JSON. The spec may be a local path or a URL. + */ +import { readFileSync } from 'node:fs'; +import { Readable } from 'node:stream'; +import { parseArgs } from 'node:util'; + +import express from 'express'; +import type { Request } from 'express'; + +import { createJsonlSink } from './report.mts'; +import { createRequestValidator } from './validator.mts'; + +const DEFAULT_SPEC_URL = 'https://docs.apify.com/api/openapi.json'; +const DEFAULT_UPSTREAM = 'https://api.apify.com'; +const BODY_LIMIT = '100mb'; + +/** Headers that describe the connection or the encoded body, which the proxy re-creates itself. */ +const HOP_BY_HOP_HEADERS = new Set([ + 'host', + 'connection', + 'keep-alive', + 'transfer-encoding', + 'content-length', + 'content-encoding', + 'expect', + 'proxy-connection', + 'upgrade', +]); + +type RequestWithRawBody = Request & { rawBody?: Buffer }; + +const loadSpec = async (source: string): Promise => { + if (/^https?:\/\//.test(source)) { + const response = await fetch(source); + if (!response.ok) throw new Error(`Failed to download the specification: ${response.status} ${source}`); + return (await response.json()) as object; + } + return JSON.parse(readFileSync(source, 'utf8')) as object; +}; + +const main = async (): Promise => { + const { values } = parseArgs({ + options: { + spec: { type: 'string', default: DEFAULT_SPEC_URL }, + upstream: { type: 'string', default: DEFAULT_UPSTREAM }, + port: { type: 'string', default: '0' }, + report: { type: 'string' }, + }, + }); + const upstream = values.upstream.replace(/\/$/, ''); + const spec = await loadSpec(values.spec); + const validator = createRequestValidator({ + spec, + onFinding: values.report ? createJsonlSink(values.report) : undefined, + }); + let validatedRequests = 0; + + const keepRawBody = (req: RequestWithRawBody, _res: unknown, buffer: Buffer): void => { + req.rawBody = buffer; + }; + + const app = express(); + app.get('/__openapi-report', (_req, res) => { + res.json({ validatedRequests, findings: validator.findings }); + }); + // Bodies arrive decompressed from the parsers (`inflate`), so the validator sees the payload and the proxy + // forwards it without the original `Content-Encoding`. + app.use( + express.json({ + limit: BODY_LIMIT, + inflate: true, + strict: false, + type: 'application/json', + verify: keepRawBody, + }), + ); + app.use(express.raw({ limit: BODY_LIMIT, inflate: true, type: () => true, verify: keepRawBody })); + app.use((req, _res, next) => { + if (req.path.startsWith('/v2')) validatedRequests += 1; + next(); + }); + app.use(validator.middleware); + app.use(async (req: RequestWithRawBody, res, next) => { + try { + const headers = new Headers(); + for (const [name, value] of Object.entries(req.headers)) { + if (HOP_BY_HOP_HEADERS.has(name) || value === undefined) continue; + headers.set(name, Array.isArray(value) ? value.join(', ') : value); + } + const hasBody = !['GET', 'HEAD'].includes(req.method) && req.rawBody && req.rawBody.length > 0; + const upstreamResponse = await fetch(`${upstream}${req.originalUrl}`, { + method: req.method, + headers, + body: hasBody ? req.rawBody : undefined, + redirect: 'manual', + }); + res.status(upstreamResponse.status); + upstreamResponse.headers.forEach((value, name) => { + if (!HOP_BY_HOP_HEADERS.has(name)) res.setHeader(name, value); + }); + if (upstreamResponse.body) { + Readable.fromWeb(upstreamResponse.body as never).pipe(res); + } else { + res.end(); + } + } catch (error) { + next(error); + } + }); + + const server = app.listen(Number(values.port), () => { + const address = server.address(); + const port = typeof address === 'object' && address ? address.port : values.port; + console.log(`OpenAPI validating proxy listening on http://127.0.0.1:${port}, forwarding to ${upstream}`); + }); + + const shutdown = (): void => { + console.log( + `OpenAPI validating proxy: ${validator.findings.length} finding(s) in ${validatedRequests} request(s)`, + ); + server.close(() => process.exit(0)); + }; + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); +}; + +await main(); diff --git a/tools/openapi-request-validator/report.mts b/tools/openapi-request-validator/report.mts new file mode 100644 index 000000000..5dc3b4080 --- /dev/null +++ b/tools/openapi-request-validator/report.mts @@ -0,0 +1,111 @@ +/** + * Aggregate and print request validation findings. + * + * Findings are stored as JSON lines so that several processes (vitest workers, the proxy) can append to the same + * file. `node report.mts ` prints a summary grouped by operation and error, and emits GitHub Actions + * `::warning::` annotations when running in CI. Pass `--fail` to exit non-zero when there is any finding. + */ +import { appendFileSync, existsSync, readFileSync } from 'node:fs'; + +import type { Finding } from './validator.mts'; + +const MAX_EXAMPLE_URLS = 3; + +/** Return a sink that appends each finding to a JSON lines file. */ +export const createJsonlSink = + (file: string) => + (finding: Finding): void => { + appendFileSync(file, `${JSON.stringify(finding)}\n`); + }; + +export const loadFindings = (file: string): Finding[] => { + if (!existsSync(file)) return []; + return readFileSync(file, 'utf8') + .split('\n') + .filter((line) => line.trim() !== '') + .map((line) => JSON.parse(line) as Finding); +}; + +export interface FindingGroup { + kind: Finding['kind']; + operation: string; + message: string; + count: number; + exampleUrls: string[]; +} + +/** Replace path segments that look like resource IDs so that the same operation groups together. */ +const normalizePath = (url: string): string => + url + .split('?')[0] + .split('/') + .map((segment) => (/^[A-Za-z0-9]{17}$|~/.test(segment) ? '{id}' : segment)) + .join('/'); + +export const aggregate = (findings: Finding[]): FindingGroup[] => { + const groups = new Map(); + for (const finding of findings) { + const operation = `${finding.method} ${finding.route ?? normalizePath(finding.url)}`; + for (const error of finding.errors) { + const message = + error.path && finding.kind === 'invalid-request' ? `${error.path}: ${error.message}` : error.message; + const key = `${finding.kind}|${operation}|${message}`; + const group = groups.get(key) ?? { kind: finding.kind, operation, message, count: 0, exampleUrls: [] }; + group.count += 1; + if (group.exampleUrls.length < MAX_EXAMPLE_URLS && !group.exampleUrls.includes(finding.url)) { + group.exampleUrls.push(finding.url); + } + groups.set(key, group); + } + } + return [...groups.values()].sort( + (a, b) => a.operation.localeCompare(b.operation) || a.message.localeCompare(b.message), + ); +}; + +const KIND_LABELS: Record = { + 'invalid-request': 'Request does not match the specification', + 'undocumented-operation': 'Operation is not in the specification', + 'validator-failure': 'Validator failed', +}; + +export const formatReport = (groups: FindingGroup[], totalRequests?: number): string => { + const lines: string[] = []; + const total = groups.reduce((sum, group) => sum + group.count, 0); + const scope = totalRequests === undefined ? '' : ` out of ${totalRequests} validated requests`; + lines.push(`OpenAPI request validation: ${total} finding(s) in ${groups.length} group(s)${scope}`); + let currentOperation = ''; + for (const group of groups) { + if (group.operation !== currentOperation) { + currentOperation = group.operation; + lines.push('', currentOperation); + } + lines.push(` [${group.kind}] ${group.message} (${group.count}x)`); + for (const url of group.exampleUrls) lines.push(` e.g. ${url}`); + } + return lines.join('\n'); +}; + +export const formatAnnotations = (groups: FindingGroup[]): string[] => + groups.map( + (group) => + `::warning title=OpenAPI request validation::${group.operation}: ${KIND_LABELS[group.kind]}: ${group.message} (${group.count}x, e.g. ${group.exampleUrls[0]})`, + ); + +const main = (): void => { + const args = process.argv.slice(2); + const fail = args.includes('--fail'); + const file = args.find((arg) => !arg.startsWith('--')); + if (!file) { + console.error('Usage: node report.mts [--fail]'); + process.exit(2); + } + const groups = aggregate(loadFindings(file)); + console.log(formatReport(groups)); + if (process.env.GITHUB_ACTIONS) { + for (const line of formatAnnotations(groups)) console.log(line); + } + if (fail && groups.length > 0) process.exit(1); +}; + +if (process.argv[1] && import.meta.url === new URL(process.argv[1], 'file://').href) main(); diff --git a/tools/openapi-request-validator/validator.mts b/tools/openapi-request-validator/validator.mts new file mode 100644 index 000000000..ef63aea9d --- /dev/null +++ b/tools/openapi-request-validator/validator.mts @@ -0,0 +1,202 @@ +/** + * Request-only OpenAPI validation middleware. + * + * Validates every incoming request against the Apify API OpenAPI specification and records each mismatch as a + * `Finding`. It never blocks or alters a request: validation runs against a shadow copy of the request, so the + * handlers downstream see the request exactly as the client sent it. Responses are not validated at all. + * + * This is a deliberately small subset of the monitoring middleware the API server runs, keeping only what is + * needed to report client requests that do not match the published specification. + */ +import type { Request, RequestHandler, Response } from 'express'; +import { error as openApiErrors, middleware as openApiMiddleware } from 'express-openapi-validator'; + +export interface ValidationError { + path: string; + message: string; + errorCode?: string; +} + +export type FindingKind = 'invalid-request' | 'undocumented-operation' | 'validator-failure'; + +export interface Finding { + kind: FindingKind; + method: string; + /** The request URL as the client sent it, including the query string. */ + url: string; + /** The specification path template the request matched, when it matched one. */ + route?: string; + errors: ValidationError[]; + timestamp: string; +} + +export interface RequestValidatorOptions { + /** The parsed OpenAPI specification document. */ + spec: object; + /** Only requests whose path starts with this prefix are validated. */ + validatedPathPrefix?: string; + /** Paths intentionally absent from the public specification. */ + ignoredPathPrefixes?: string[]; + /** Query parameters intentionally left out of the public specification. */ + internalQueryParameters?: string[]; + /** Called for every finding, for example to stream findings to a file. */ + onFinding?: (finding: Finding) => void; +} + +export interface RequestValidator { + middleware: RequestHandler; + findings: Finding[]; +} + +const DEFAULT_VALIDATED_PATH_PREFIX = '/v2'; + +/** Undocumented legacy alias of `/v2/actors`. */ +const DEFAULT_IGNORED_PATH_SUFFIXES = ['/acts']; + +const DEFAULT_INTERNAL_QUERY_PARAMETERS = ['_useNewStoreBackend', '_useCase', '_traceId']; + +/** Query parameter values the API coerces to booleans, which the validator does not recognize. */ +const BOOLEAN_COERCIBLE_VALUES = new Set(['0', '1']); + +const queryParameterFromErrorPath = (path: string | undefined): string | undefined => + path?.match(/^\/query\/(.+)$/)?.[1]; + +const isFalsePositiveBooleanCoercionError = (error: ValidationError, req: Request): boolean => { + if (error.message !== 'must be boolean') return false; + const parameterName = queryParameterFromErrorPath(error.path); + if (!parameterName) return false; + const value = req.query[parameterName]; + return typeof value === 'string' && BOOLEAN_COERCIBLE_VALUES.has(value); +}; + +type OpenApiError = InstanceType<(typeof openApiErrors)[keyof typeof openApiErrors]>; + +const isOpenApiError = (error: unknown): error is OpenApiError => + Object.values(openApiErrors).some((ErrorClass) => error instanceof ErrorClass); + +/** Reserved characters the validator rejects in a query value, without `+`, which encodes a space. */ +const RESERVED_QUERY_CHARACTERS = /[:/?#[\]@!$&'()*,;=]/; + +/** + * The validator rejects `+` in query values as a reserved character, but a client encoding a space as `+` follows + * the form encoding the API accepts. + */ +const isFalsePositiveReservedCharacterError = (error: ValidationError, req: Request): boolean => { + if (!error.message.includes('must be url encoded')) return false; + const parameterName = queryParameterFromErrorPath(error.path); + if (!parameterName) return false; + const rawQuery = req.originalUrl.split('?')[1] ?? ''; + const rawValues = rawQuery + .split('&') + .filter((pair) => pair.startsWith(`${parameterName}=`)) + .map((pair) => pair.slice(parameterName.length + 1)); + return rawValues.length > 0 && rawValues.every((value) => !RESERVED_QUERY_CHARACTERS.test(value)); +}; + +/** + * Build a request object the validator may freely mutate. The validator rewrites `req.query` into its parsed form + * and attaches `req.openapi`, neither of which must leak to the handlers that process the real request. + */ +const createShadowRequest = (req: Request): Request => { + const shadow = Object.create(req) as Request; + const contentType = req.headers['content-type'] ?? ''; + // A raw text body arrives as a Buffer; the specification describes it as a string. + const body = Buffer.isBuffer(req.body) && /^text\//i.test(contentType) ? req.body.toString('utf8') : req.body; + const ownProperties = { + query: structuredClone(req.query), + params: { ...req.params }, + body, + openapi: undefined, + }; + for (const [name, value] of Object.entries(ownProperties)) { + Object.defineProperty(shadow, name, { value, writable: true, configurable: true, enumerable: true }); + } + return shadow; +}; + +/** Run an express middleware chain to completion and resolve with the error it passed to `next`, if any. */ +const runChain = async (handlers: RequestHandler[], req: Request, res: Response): Promise => { + for (const handler of handlers) { + const error = await new Promise((resolve) => { + try { + const result = handler(req, res, resolve) as unknown; + if (result instanceof Promise) result.catch(resolve); + } catch (err) { + resolve(err); + } + }); + if (error) return error; + } + return undefined; +}; + +export function createRequestValidator(options: RequestValidatorOptions): RequestValidator { + const { + spec, + validatedPathPrefix = DEFAULT_VALIDATED_PATH_PREFIX, + ignoredPathPrefixes = DEFAULT_IGNORED_PATH_SUFFIXES.map((suffix) => `${validatedPathPrefix}${suffix}`), + internalQueryParameters = DEFAULT_INTERNAL_QUERY_PARAMETERS, + onFinding, + } = options; + const internalQueryParameterSet = new Set(internalQueryParameters); + const findings: Finding[] = []; + + const validators = openApiMiddleware({ + apiSpec: spec as never, + validateApiSpec: false, + validateSecurity: false, + validateResponses: false, + validateRequests: { + allowUnknownQueryParameters: false, + coerceTypes: false, + removeAdditional: false, + allErrors: true, + }, + }); + + const shouldValidate = (path: string): boolean => + path.startsWith(validatedPathPrefix) && !ignoredPathPrefixes.some((prefix) => path.startsWith(prefix)); + + const record = (req: Request, shadow: Request, error: unknown): void => { + const base = { + method: req.method, + url: req.originalUrl, + route: (shadow as Request & { openapi?: { expressRoute?: string } }).openapi?.expressRoute, + timestamp: new Date().toISOString(), + }; + let finding: Finding; + if (error instanceof openApiErrors.NotFound || error instanceof openApiErrors.MethodNotAllowed) { + finding = { ...base, kind: 'undocumented-operation', errors: [{ path: req.path, message: error.message }] }; + } else if (isOpenApiError(error)) { + const errors = (error.errors as ValidationError[]).filter( + (e) => + !isFalsePositiveBooleanCoercionError(e, req) && + !isFalsePositiveReservedCharacterError(e, req) && + !internalQueryParameterSet.has(queryParameterFromErrorPath(e.path) ?? ''), + ); + if (errors.length === 0) return; + finding = { ...base, kind: 'invalid-request', errors }; + } else { + const message = error instanceof Error ? error.message : String(error); + finding = { ...base, kind: 'validator-failure', errors: [{ path: req.path, message }] }; + } + findings.push(finding); + onFinding?.(finding); + }; + + const middleware: RequestHandler = (req, res, next) => { + if (!shouldValidate(req.path)) { + next(); + return; + } + const shadow = createShadowRequest(req); + runChain(validators, shadow, res) + .then((error) => { + if (error) record(req, shadow, error); + }) + .catch((error: unknown) => record(req, shadow, error)) + .finally(() => next()); + }; + + return { middleware, findings }; +} From 9682a02182df075c8a89d45c044e1d8c487733ec Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 13:29:36 +0000 Subject: [PATCH 2/6] ci: print one warning line per OpenAPI validation finding The report script prints only the warning lines, as GitHub Actions annotations in CI, instead of a grouped text block followed by them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017ocuALGHVYTScW9Ht9Ytjo --- tools/openapi-request-validator/README.md | 4 +-- tools/openapi-request-validator/report.mts | 41 ++++++++-------------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/tools/openapi-request-validator/README.md b/tools/openapi-request-validator/README.md index 86c651cf4..c4a856b01 100644 --- a/tools/openapi-request-validator/README.md +++ b/tools/openapi-request-validator/README.md @@ -8,8 +8,8 @@ report, so it works as a warning in CI while the specification and the client ca and appends findings to `OPENAPI_REPORT_FILE`. - `proxy.mts` is a validating reverse proxy for the integration tests. It forwards every request to the live API and records the findings on the way. `GET /__openapi-report` returns them as JSON. -- `report.mts` aggregates a findings file by operation and error, prints it, and emits GitHub Actions warning - annotations in CI. `--fail` makes it exit non-zero when there is any finding. +- `report.mts` aggregates a findings file by operation and error and prints one warning line per group, as + GitHub Actions warning annotations in CI. `--fail` makes it exit non-zero when there is any finding. ```sh pnpm spec:fetch diff --git a/tools/openapi-request-validator/report.mts b/tools/openapi-request-validator/report.mts index 5dc3b4080..2b7e89645 100644 --- a/tools/openapi-request-validator/report.mts +++ b/tools/openapi-request-validator/report.mts @@ -2,8 +2,8 @@ * Aggregate and print request validation findings. * * Findings are stored as JSON lines so that several processes (vitest workers, the proxy) can append to the same - * file. `node report.mts ` prints a summary grouped by operation and error, and emits GitHub Actions - * `::warning::` annotations when running in CI. Pass `--fail` to exit non-zero when there is any finding. + * file. `node report.mts ` prints one warning line per operation and error, as GitHub Actions `::warning::` + * annotations when running in CI. Pass `--fail` to exit non-zero when there is any finding. */ import { appendFileSync, existsSync, readFileSync } from 'node:fs'; @@ -69,27 +69,15 @@ const KIND_LABELS: Record = { 'validator-failure': 'Validator failed', }; -export const formatReport = (groups: FindingGroup[], totalRequests?: number): string => { - const lines: string[] = []; - const total = groups.reduce((sum, group) => sum + group.count, 0); - const scope = totalRequests === undefined ? '' : ` out of ${totalRequests} validated requests`; - lines.push(`OpenAPI request validation: ${total} finding(s) in ${groups.length} group(s)${scope}`); - let currentOperation = ''; - for (const group of groups) { - if (group.operation !== currentOperation) { - currentOperation = group.operation; - lines.push('', currentOperation); - } - lines.push(` [${group.kind}] ${group.message} (${group.count}x)`); - for (const url of group.exampleUrls) lines.push(` e.g. ${url}`); - } - return lines.join('\n'); -}; +const describeGroup = (group: FindingGroup): string => + `${group.operation}: ${KIND_LABELS[group.kind]}: ${group.message} (${group.count}x, e.g. ${group.exampleUrls[0]})`; -export const formatAnnotations = (groups: FindingGroup[]): string[] => - groups.map( - (group) => - `::warning title=OpenAPI request validation::${group.operation}: ${KIND_LABELS[group.kind]}: ${group.message} (${group.count}x, e.g. ${group.exampleUrls[0]})`, +/** One warning line per group; GitHub Actions renders the annotated form in the job summary and on the PR. */ +export const formatWarnings = (groups: FindingGroup[], { annotations = false } = {}): string[] => + groups.map((group) => + annotations + ? `::warning title=OpenAPI request validation::${describeGroup(group)}` + : `Warning: ${describeGroup(group)}`, ); const main = (): void => { @@ -101,11 +89,12 @@ const main = (): void => { process.exit(2); } const groups = aggregate(loadFindings(file)); - console.log(formatReport(groups)); - if (process.env.GITHUB_ACTIONS) { - for (const line of formatAnnotations(groups)) console.log(line); + if (groups.length === 0) { + console.log('OpenAPI request validation: every request matched the specification.'); + return; } - if (fail && groups.length > 0) process.exit(1); + for (const line of formatWarnings(groups, { annotations: Boolean(process.env.GITHUB_ACTIONS) })) console.log(line); + if (fail) process.exit(1); }; if (process.argv[1] && import.meta.url === new URL(process.argv[1], 'file://').href) main(); From abd5fecb7411c88f0cfe0bba9663c347cbb6e5ba Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 13:43:22 +0000 Subject: [PATCH 3/6] ci: type-check the OpenAPI validator with the maintainer scripts Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017ocuALGHVYTScW9Ht9Ytjo --- tsconfig.scripts.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.scripts.json b/tsconfig.scripts.json index 6ff50f667..c90809aaf 100644 --- a/tsconfig.scripts.json +++ b/tsconfig.scripts.json @@ -27,5 +27,5 @@ "types": ["node"], "allowImportingTsExtensions": true }, - "include": ["scripts/**/*"] + "include": ["scripts/**/*", "tools/**/*"] } From 7bbaa773f68644912495f90c367b6c56bff7a22e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 13:57:51 +0000 Subject: [PATCH 4/6] ci: run OpenAPI request validation only from the manual Run tests workflow The Check workflow and local test runs behave as before. The new workflow_dispatch workflow runs the unit and integration tests with the validation option, on by default. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017ocuALGHVYTScW9Ht9Ytjo --- .github/workflows/check.yaml | 30 ------- .github/workflows/run_tests.yaml | 104 ++++++++++++++++++++++ tools/openapi-request-validator/README.md | 7 +- 3 files changed, 110 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/run_tests.yaml diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index ecebddde3..e0c6b2266 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -48,19 +48,8 @@ jobs: - name: Install Chrome for puppeteer run: pnpm exec puppeteer browsers install chrome - - name: Fetch the OpenAPI specification - run: pnpm spec:fetch - - # The mock server also checks every request against the specification, see `tools/openapi-request-validator`. - name: Run Tests run: pnpm test - env: - OPENAPI_SPEC_PATH: tmp/openapi.json - OPENAPI_REPORT_FILE: openapi-findings.jsonl - - - name: Report requests that do not match the OpenAPI specification - if: always() - run: pnpm openapi:report openapi-findings.jsonl integration_tests: name: Integration tests (Node ${{ matrix.node-version }}) @@ -103,28 +92,9 @@ jobs: # A workflow that calls this one as a reusable workflow - the release flow does - gets no # secrets unless it opts in with `secrets: inherit`, so the token can legitimately be empty. # Skip rather than fail there, otherwise a missing token blocks the release. - - name: Fetch the OpenAPI specification - if: ${{ env.APIFY_TEST_USER_API_TOKEN != '' }} - run: pnpm spec:fetch - - # The tests talk to the live API through a proxy that checks every request against the - # specification, see `tools/openapi-request-validator`. The proxy keeps running into the next steps. - - name: Start the OpenAPI validating proxy - if: ${{ env.APIFY_TEST_USER_API_TOKEN != '' }} - run: | - pnpm openapi:proxy --spec tmp/openapi.json --port 3999 --report openapi-findings.jsonl > openapi-proxy.log 2>&1 & - for _ in $(seq 1 30); do grep -q listening openapi-proxy.log && break; sleep 1; done - cat openapi-proxy.log - - name: Run integration tests if: ${{ env.APIFY_TEST_USER_API_TOKEN != '' }} run: pnpm test:integration - env: - APIFY_INTEGRATION_TESTS_API_URL: http://127.0.0.1:3999 - - - name: Report requests that do not match the OpenAPI specification - if: ${{ always() && env.APIFY_TEST_USER_API_TOKEN != '' }} - run: pnpm openapi:report openapi-findings.jsonl - name: Report the skipped integration tests if: ${{ env.APIFY_TEST_USER_API_TOKEN == '' }} diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml new file mode 100644 index 000000000..5b988edc4 --- /dev/null +++ b/.github/workflows/run_tests.yaml @@ -0,0 +1,104 @@ +# Runs the whole test suite (unit and integration) on demand. Unlike the `Check` workflow, every request the tests +# send can be checked against the published OpenAPI specification; mismatches are reported as warnings and never +# fail the run. See `tools/openapi-request-validator`. + +name: Run tests + +on: + workflow_dispatch: + inputs: + validate_openapi: + description: Check every request against the OpenAPI specification and report mismatches as warnings + type: boolean + default: true + +permissions: + contents: read + +env: + NODE_VERSION: 24 + # Where the unit-test mock server and the integration-test proxy append their findings. + OPENAPI_FINDINGS_FILE: openapi-findings.jsonl + +jobs: + unit_tests: + name: Unit tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Use Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install pnpm and dependencies + uses: apify/actions/pnpm-install@v1.6.2 + + - name: Install Chrome for puppeteer + run: pnpm exec puppeteer browsers install chrome + + - name: Fetch the OpenAPI specification + if: ${{ inputs.validate_openapi }} + run: pnpm spec:fetch + + # With validation off both variables are empty, and the mock server runs as it does everywhere else. + - name: Run unit tests + run: pnpm test + env: + OPENAPI_SPEC_PATH: ${{ inputs.validate_openapi && 'tmp/openapi.json' || '' }} + OPENAPI_REPORT_FILE: ${{ inputs.validate_openapi && env.OPENAPI_FINDINGS_FILE || '' }} + + - name: Report requests that do not match the OpenAPI specification + if: ${{ always() && inputs.validate_openapi }} + run: pnpm openapi:report ${{ env.OPENAPI_FINDINGS_FILE }} + + integration_tests: + name: Integration tests + runs-on: ubuntu-latest + + env: + APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_PYTHON_SDK_API_TOKEN }} + OPENAPI_PROXY_PORT: 3999 + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Use Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install pnpm and dependencies + uses: apify/actions/pnpm-install@v1.6.2 + + - name: Fetch the OpenAPI specification + if: ${{ inputs.validate_openapi }} + run: pnpm spec:fetch + + # The proxy forwards every request to the live API and records the ones that do not match the + # specification. It keeps running in the background through the following steps. + - name: Start the OpenAPI validating proxy + if: ${{ inputs.validate_openapi }} + run: | + pnpm openapi:proxy --spec tmp/openapi.json --port "$OPENAPI_PROXY_PORT" --report "$OPENAPI_FINDINGS_FILE" > openapi-proxy.log 2>&1 & + for _ in $(seq 1 30); do grep -q listening openapi-proxy.log && break; sleep 1; done + cat openapi-proxy.log + + # With validation off the variable is empty and the tests talk to the API directly. + - name: Run integration tests + if: ${{ env.APIFY_TEST_USER_API_TOKEN != '' }} + run: pnpm test:integration + env: + APIFY_INTEGRATION_TESTS_API_URL: ${{ inputs.validate_openapi && format('http://127.0.0.1:{0}', env.OPENAPI_PROXY_PORT) || '' }} + + - name: Report the skipped integration tests + if: ${{ env.APIFY_TEST_USER_API_TOKEN == '' }} + run: echo "::warning::APIFY_TEST_USER_API_TOKEN is empty, the integration tests did not run." + + - name: Report requests that do not match the OpenAPI specification + if: ${{ always() && inputs.validate_openapi }} + run: pnpm openapi:report "$OPENAPI_FINDINGS_FILE" diff --git a/tools/openapi-request-validator/README.md b/tools/openapi-request-validator/README.md index c4a856b01..780f673cb 100644 --- a/tools/openapi-request-validator/README.md +++ b/tools/openapi-request-validator/README.md @@ -2,7 +2,12 @@ Checks the requests the client sends against the Apify API OpenAPI specification and reports every mismatch. Only requests are validated; responses are passed through untouched. Nothing is ever blocked, the result is a -report, so it works as a warning in CI while the specification and the client catch up with each other. +report, so it works as a warning while the specification and the client catch up with each other. + +Validation is opt-in. The regular `Check` workflow and a plain `pnpm test` or `pnpm test:integration` never run it. +The manual `Run tests` workflow (Actions, `Run workflow`) runs the whole suite with the +`Check every request against the OpenAPI specification` option on by default and prints the findings as warning +annotations. - `validator.mts` is an Express middleware. The unit-test mock server mounts it when `OPENAPI_SPEC_PATH` is set and appends findings to `OPENAPI_REPORT_FILE`. From cf13769769804aee07441f88a838d591252f7f02 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 14:35:46 +0000 Subject: [PATCH 5/6] ci: temporarily run the Run tests workflow on the pull request The workflow cannot be started by hand before it exists on master. The pull request trigger is a test aid and is removed before merging. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017ocuALGHVYTScW9Ht9Ytjo --- .github/workflows/run_tests.yaml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 5b988edc4..2a87dd64a 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -11,6 +11,9 @@ on: description: Check every request against the OpenAPI specification and report mismatches as warnings type: boolean default: true + # TEMPORARY, remove before merging: a workflow can only be started by hand once it exists on the default + # branch, so until then the pull request runs it with validation on. + pull_request: permissions: contents: read @@ -19,6 +22,8 @@ env: NODE_VERSION: 24 # Where the unit-test mock server and the integration-test proxy append their findings. OPENAPI_FINDINGS_FILE: openapi-findings.jsonl + # Decided once here: ticked when started by hand, always on for the temporary pull request trigger. + VALIDATE_OPENAPI: ${{ github.event_name == 'pull_request' || inputs.validate_openapi }} jobs: unit_tests: @@ -41,18 +46,18 @@ jobs: run: pnpm exec puppeteer browsers install chrome - name: Fetch the OpenAPI specification - if: ${{ inputs.validate_openapi }} + if: ${{ env.VALIDATE_OPENAPI == 'true' }} run: pnpm spec:fetch # With validation off both variables are empty, and the mock server runs as it does everywhere else. - name: Run unit tests run: pnpm test env: - OPENAPI_SPEC_PATH: ${{ inputs.validate_openapi && 'tmp/openapi.json' || '' }} - OPENAPI_REPORT_FILE: ${{ inputs.validate_openapi && env.OPENAPI_FINDINGS_FILE || '' }} + OPENAPI_SPEC_PATH: ${{ env.VALIDATE_OPENAPI == 'true' && 'tmp/openapi.json' || '' }} + OPENAPI_REPORT_FILE: ${{ env.VALIDATE_OPENAPI == 'true' && env.OPENAPI_FINDINGS_FILE || '' }} - name: Report requests that do not match the OpenAPI specification - if: ${{ always() && inputs.validate_openapi }} + if: ${{ always() && env.VALIDATE_OPENAPI == 'true' }} run: pnpm openapi:report ${{ env.OPENAPI_FINDINGS_FILE }} integration_tests: @@ -76,13 +81,13 @@ jobs: uses: apify/actions/pnpm-install@v1.6.2 - name: Fetch the OpenAPI specification - if: ${{ inputs.validate_openapi }} + if: ${{ env.VALIDATE_OPENAPI == 'true' }} run: pnpm spec:fetch # The proxy forwards every request to the live API and records the ones that do not match the # specification. It keeps running in the background through the following steps. - name: Start the OpenAPI validating proxy - if: ${{ inputs.validate_openapi }} + if: ${{ env.VALIDATE_OPENAPI == 'true' }} run: | pnpm openapi:proxy --spec tmp/openapi.json --port "$OPENAPI_PROXY_PORT" --report "$OPENAPI_FINDINGS_FILE" > openapi-proxy.log 2>&1 & for _ in $(seq 1 30); do grep -q listening openapi-proxy.log && break; sleep 1; done @@ -93,12 +98,12 @@ jobs: if: ${{ env.APIFY_TEST_USER_API_TOKEN != '' }} run: pnpm test:integration env: - APIFY_INTEGRATION_TESTS_API_URL: ${{ inputs.validate_openapi && format('http://127.0.0.1:{0}', env.OPENAPI_PROXY_PORT) || '' }} + APIFY_INTEGRATION_TESTS_API_URL: ${{ env.VALIDATE_OPENAPI == 'true' && format('http://127.0.0.1:{0}', env.OPENAPI_PROXY_PORT) || '' }} - name: Report the skipped integration tests if: ${{ env.APIFY_TEST_USER_API_TOKEN == '' }} run: echo "::warning::APIFY_TEST_USER_API_TOKEN is empty, the integration tests did not run." - name: Report requests that do not match the OpenAPI specification - if: ${{ always() && inputs.validate_openapi }} + if: ${{ always() && env.VALIDATE_OPENAPI == 'true' }} run: pnpm openapi:report "$OPENAPI_FINDINGS_FILE" From 16da6d4dc785f6eb229d07bc558c63c95176f83b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 14:36:09 +0000 Subject: [PATCH 6/6] ci: run the Run tests workflow on pull requests and on demand Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017ocuALGHVYTScW9Ht9Ytjo --- .github/workflows/run_tests.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 2a87dd64a..fa46bf399 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -1,6 +1,6 @@ -# Runs the whole test suite (unit and integration) on demand. Unlike the `Check` workflow, every request the tests -# send can be checked against the published OpenAPI specification; mismatches are reported as warnings and never -# fail the run. See `tools/openapi-request-validator`. +# Runs the whole test suite (unit and integration) with every request the tests send checked against the published +# OpenAPI specification. Mismatches are reported as warnings and never fail the run, see +# `tools/openapi-request-validator`. Runs on every pull request update, and on demand with the validation optional. name: Run tests @@ -11,10 +11,12 @@ on: description: Check every request against the OpenAPI specification and report mismatches as warnings type: boolean default: true - # TEMPORARY, remove before merging: a workflow can only be started by hand once it exists on the default - # branch, so until then the pull request runs it with validation on. pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read @@ -22,7 +24,7 @@ env: NODE_VERSION: 24 # Where the unit-test mock server and the integration-test proxy append their findings. OPENAPI_FINDINGS_FILE: openapi-findings.jsonl - # Decided once here: ticked when started by hand, always on for the temporary pull request trigger. + # Decided once here: always on for a pull request, as ticked when started by hand. VALIDATE_OPENAPI: ${{ github.event_name == 'pull_request' || inputs.validate_openapi }} jobs: