diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..73c00fb --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,35 @@ +name: Validate API Specs + +on: + pull_request: + paths: + - "reference/**" + - "redocly.yaml" + - "package.json" + push: + branches: [main] + paths: + - "reference/**" + - "redocly.yaml" + - "package.json" + +jobs: + validate: + name: Lint & Validate + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - run: npm ci + + - name: Lint OpenAPI specs + run: npm run lint:openapi + + - name: Validate JSON schemas + run: npm run lint:schemas diff --git a/.gitignore b/.gitignore index eaa7094..2f53ec4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea/ .worktrees +node_modules/ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..c0882b1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,108 @@ +{ + "name": "api-specs", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "api-specs", + "version": "1.0.0", + "devDependencies": { + "@redocly/cli": "^2.41.0", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" + } + }, + "node_modules/@redocly/cli": { + "version": "2.41.0", + "resolved": "https://registry.npmjs.org/@redocly/cli/-/cli-2.41.0.tgz", + "integrity": "sha512-7MjbLQ7u7zPT5sWN6PaBBOCflue//fgmmoGYui8pKmi1dw6um2pYwjElXjC0qycNEGXo3pCJc599GqEgFdKbxQ==", + "dev": true, + "license": "MIT", + "bin": { + "openapi": "bin/cli.js", + "redocly": "bin/cli.js" + }, + "engines": { + "node": ">=22.12.0 || >=20.19.0 <21.0.0", + "npm": ">=10" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..cbf63e4 --- /dev/null +++ b/package.json @@ -0,0 +1,16 @@ +{ + "name": "api-specs", + "version": "1.0.0", + "private": true, + "description": "Precog API specifications", + "scripts": { + "lint": "npm run lint:openapi && npm run lint:schemas", + "lint:openapi": "redocly lint reference/*.yaml", + "lint:schemas": "node scripts/validate-json-schemas.mjs" + }, + "devDependencies": { + "@redocly/cli": "^2.41.0", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" + } +} diff --git a/redocly.yaml b/redocly.yaml new file mode 100644 index 0000000..8b39295 --- /dev/null +++ b/redocly.yaml @@ -0,0 +1,18 @@ +extends: + - recommended + +rules: + no-ambiguous-paths: off + no-server-example.com: off + no-http-verbs-in-paths: warn + no-identical-paths: error + no-undefined-server-variable: error + operation-4xx-response: warn + path-not-include-query: error + + # Relax some recommended rules that may be too strict initially + info-contact: warn + info-license: off + operation-operationId-unique: error + operation-operationId: warn + tag-description: warn diff --git a/reference/admin-http-api.yaml b/reference/admin-http-api.yaml index af3711d..105c982 100644 --- a/reference/admin-http-api.yaml +++ b/reference/admin-http-api.yaml @@ -280,24 +280,6 @@ components: $ref: '#/components/schemas/applies-to' toOverride: $ref: '#/components/schemas/job-args-to-override' - - error-message: - title: error-message - type: object - properties: - error: - type: object - required: - - type - - details - properties: - type: - type: string - minLength: 1 - details: - type: string - required: - - error securitySchemes: BearerAuth: type: http diff --git a/reference/metadata-http-api.yaml b/reference/metadata-http-api.yaml index c2e74b4..f9b72c8 100644 --- a/reference/metadata-http-api.yaml +++ b/reference/metadata-http-api.yaml @@ -47,8 +47,8 @@ paths: primaryKey: [ id_example, array_index ] foreignKeys: example: - - local: [ id_example ] - - reference: [ id ] + local: [ id_example ] + reference: [ id ] columns: - columnName: id_example columnType: string @@ -244,24 +244,17 @@ components: type: object title: pipeline dataset properties properties: - primaryKey: - type: object - description: A primary key property - properties: - type: - oneOf: - - type: string - enum: [strict, optional] - description: Indicates whether the key is strict or optional - components: - type: array - title: Primary key components - items: - type: string - minLength: 1 - required: [type, components] - required: [primaryKey] - additionalProperties: false + type: + type: string + enum: [strict, optional] + description: Indicates whether the key is strict or optional + components: + type: array + title: Primary key components + items: + type: string + minLength: 1 + required: [type, components] securitySchemes: BearerAuth: diff --git a/reference/public-http-api.yaml b/reference/public-http-api.yaml index 544f92b..a66a858 100644 --- a/reference/public-http-api.yaml +++ b/reference/public-http-api.yaml @@ -546,7 +546,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/test-connectivity-request' + $ref: '#/components/schemas/source-validation-request' responses: '200': description: OK @@ -733,7 +733,7 @@ paths: operationId: get-pipelines-pipeline-id description: Get a particular pipeline. delete: - summary: '' + summary: Deletes the given pipeline tags: [] operationId: delete-pipelines-pipeline-id responses: @@ -741,7 +741,6 @@ paths: description: No Content '404': description: Not Found - description: Deletes the given pipeline '/pipelines/{pipeline-id}/metrics': parameters: - schema: @@ -763,6 +762,7 @@ paths: description: Unauthorized '404': description: Pipeline not found. + operationId: get-pipelines-pipeline-id-metrics description: Get the metrics from a pipeline '/pipelines/{pipeline-id}/datasets': @@ -791,7 +791,7 @@ paths: operationId: get-pipelines-pipeline-id-datasets description: Get the datasets in the provided pipeline. patch: - summary: '' + summary: Add/remove/reload datasets for the provided pipeline. tags: [] operationId: patch-pipelines-pipeline-id-datasets responses: @@ -813,7 +813,6 @@ paths: application/json: schema: $ref: '#/components/schemas/error-message' - description: Add (or remove) datasets to (or from) the provided pipeline. requestBody: content: application/json: @@ -1121,13 +1120,11 @@ paths: properties: vectorize: type: boolean - nullable: true description: >- The new value of the vectorize flag. `null` and `false` are equivalent: the pipeline does not vectorize. sentimentAnalysis: type: boolean - nullable: true description: >- Enables sentiment analysis on the pipeline. Requires vectorization to also be on, otherwise a 400 will be returned. @@ -1198,166 +1195,6 @@ paths: description: Unauthorized '404': description: Pipeline not found - '/pipelines/{pipeline-id}/semantic-contexts': - parameters: - - schema: - type: string - name: pipeline-id - in: path - required: true - get: - operationId: get-pipelines-pipeline-id-semantic-contexts - summary: Get existing semantic contexts of this pipeline - responses: - '200': - description: List of semantic contexts - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/semantic-context' - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - '401': - description: Unauthorized - '404': - description: Pipeline not found - '422': - description: Semantic error. - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - post: - operationId: post-pipelines-pipeline-id-semantic-contexts - summary: Add a semantic context to the pipeline - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/semantic-context-request' - responses: - '201': - description: Semantic context created - content: - application/json: - schema: - type: object - properties: - semanticContextId: - type: string - description: ID generated for the newly created semantic context - minLength: 1 - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - '401': - description: Unauthorized - '404': - description: Pipeline not found - '422': - description: Semantic error. - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - - '/pipelines/{pipeline-id}/semantic-contexts/{semantic-context-id}': - parameters: - - schema: - type: string - name: pipeline-id - in: path - required: true - - schema: - type: string - name: semantic-context-id - in: path - required: true - get: - operationId: get-pipelines-pipeline-id-semantic-contexts-semantic-context-id - summary: Get the details of the particular semantic context - responses: - '200': - description: Semantic context retrieved - content: - application/json: - schema: - $ref: '#/components/schemas/semantic-context' - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - '401': - description: Unauthorized - '404': - description: Semantic context with this pipeline ID/semantic context ID combination not found - '422': - description: Semantic error. - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - put: - operationId: put-pipelines-pipeline-id-semantic-contexts-semantic-context-id - summary: Replace details of a semantic context - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/semantic-context-request' - responses: - '200': - description: Semantic context updated - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - '401': - description: Unauthorized - '404': - description: Semantic context with this pipeline ID/semantic context ID combination not found - '422': - description: Semantic error. - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - delete: - operationId: delete-pipelines-pipeline-id-semantic-contexts-semantic-context-id - summary: Delete the semantic context - responses: - '200': - description: Semantic context deleted - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - '401': - description: Unauthorized - '404': - description: Semantic context with this pipeline ID/semantic context ID combination not found - '422': - description: Semantic error. - content: - application/json: - schema: - $ref: '#/components/schemas/error-message' - /semantic-model/generation-jobs: post: operationId: post-semantic-model-generation-jobs @@ -1639,6 +1476,7 @@ paths: description: Unauthorized '404': description: Load not found + operationId: get-loads-load-id-metrics description: Returns metrics for a specific pipeline load, grouped by dataset and table. /log: post: @@ -1762,7 +1600,7 @@ paths: - returnUrl description: Create a Stripe customer portal session parameters: [] - /issues/: + /issues: post: summary: Create issue operationId: post-issues @@ -2173,8 +2011,23 @@ components: required: - id - name - pipeline: - title: '' + pii-config: + type: object + properties: + type: + type: string + enum: + - passthrough + - hash + - omit + - nullify + required: + - type + description: >- + Policy applied to detected PII columns. Optional — when absent, + passthrough is assumed (PII columns are not transformed). Omitted + from responses when passthrough. + pipeline-core: type: object properties: id: @@ -2184,41 +2037,23 @@ components: name: type: string minLength: 1 - destinationId: - type: string - minLength: 1 columnNameStrategy: type: object - minLength: 1 required: - tableNamePostfix properties: tableNamePostfix: type: boolean - datasetIds: - type: array - readOnly: true - items: - type: string + required: + - id + - name + pipeline-processing-config: + type: object + properties: piiConfig: - type: object - properties: - type: - type: string - enum: - - passthrough - - hash - - omit - - nullify - required: - - type - description: >- - Policy applied to detected PII columns. Optional — when absent, - passthrough is assumed (PII columns are not transformed). Omitted - from responses when passthrough. + $ref: '#/components/schemas/pii-config' vectorize: type: boolean - nullable: true description: >- vectorization-spec-v4 / LD13. When `true`, loads of datasets in this pipeline will additionally produce per-row text embeddings @@ -2243,14 +2078,26 @@ components: until an operator deletes them by hand. sentimentAnalysis: type: boolean - nullable: true description: >- Enables sentiment analysis on the pipeline. Requires vectorization to also be on, otherwise a 400 will be returned. - required: - - id - - name - - destinationId + pipeline: + title: '' + allOf: + - $ref: '#/components/schemas/pipeline-core' + - $ref: '#/components/schemas/pipeline-processing-config' + - type: object + properties: + destinationId: + type: string + minLength: 1 + datasetIds: + type: array + readOnly: true + items: + type: string + required: + - destinationId drifted-dataset: title: '' type: object @@ -2265,7 +2112,6 @@ components: description: When the schema drift was detected. offset: type: object - nullable: true description: The offset at which schema drift was detected. properties: type: @@ -2558,24 +2404,17 @@ components: type: object title: pipeline dataset properties properties: - primaryKey: - type: object - description: A primary key property - properties: - type: - oneOf: - - type: string - enum: [strict, optional] - description: Indicates whether the key is strict or optional - components: - type: array - title: Primary key components - items: - type: string - minLength: 1 - required: [type, components] - required: [primaryKey] - additionalProperties: false + type: + type: string + enum: [strict, optional] + description: Indicates whether the key is strict or optional + components: + type: array + title: Primary key components + items: + type: string + minLength: 1 + required: [type, components] reuse-previous-primary-key: description: Legacy boolean or new strategy object. oneOf: @@ -2610,25 +2449,12 @@ components: type: object properties: pipeline: - type: object - required: - - id - - name - - columnNameStrategy - properties: - id: - type: string - minLength: 1 - name: - type: string - minLength: 1 - columnNameStrategy: - type: object + allOf: + - $ref: '#/components/schemas/pipeline-core' + - $ref: '#/components/schemas/pipeline-processing-config' + - type: object required: - - tableNamePostfix - properties: - tableNamePostfix: - type: boolean + - columnNameStrategy destination: type: object required: diff --git a/scripts/validate-json-schemas.mjs b/scripts/validate-json-schemas.mjs new file mode 100644 index 0000000..6698912 --- /dev/null +++ b/scripts/validate-json-schemas.mjs @@ -0,0 +1,33 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import Ajv from "ajv/dist/2020.js"; +import addFormats from "ajv-formats"; + +const schemasDir = resolve("reference/schemas"); +const files = readdirSync(schemasDir).filter((f) => f.endsWith(".schema.json")); + +if (files.length === 0) { + console.log("No .schema.json files found — skipping."); + process.exit(0); +} + +const ajv = new Ajv({ allErrors: true, strict: true }); +addFormats(ajv); + +let failed = false; + +for (const file of files) { + const filePath = join(schemasDir, file); + const content = JSON.parse(readFileSync(filePath, "utf-8")); + + try { + ajv.compile(content); + console.log(`✓ ${file}`); + } catch (err) { + console.error(`✗ ${file}`); + console.error(` ${err.message}`); + failed = true; + } +} + +process.exit(failed ? 1 : 0);