{
"openapi": "3.1.0",
"info": {
"title": "Data Quality Gate",
"version": "0.2.0",
"summary": "Deterministic dataset-reliability checker for AI agents.",
"description": "Deterministic post-scrape data cleaner and quality gate for AI agents. Three tiers over one engine, no LLM anywhere: the same input always produces byte-identical output.\n\n**CLEAN** (`POST /api/clean`, $0.04) - post the raw output of a scrape, get the REPAIRED data back as the response body: residual HTML stripped, mojibake decoded (\"Café\" -> \"Café\"), invisible characters removed, non-breaking spaces normalised, values trimmed, across nested objects and arrays. It repairs how data was ENCODED and never what it SAYS: a negative price or a failed extraction (\"captcha\", \"access denied\") is reported, never rewritten or deleted. Call it after every extraction run - a verdict is cached per source, but dirt is produced fresh by every run.\n\n**CLEAN + AUDIT** (`POST /api/clean/audit`, $0.12) - identical repaired data plus a complete, replayable, reversible ledger of every transformation, with a replay_id and input/output SHA-256. Applying the ledger in reverse reconstructs the input byte for byte.\n\n**VERDICT** (`POST /api`, $0.01) - score + exact facts + a RELIABLE / USABLE_WITH_CLEANING / UNRELIABLE judgement, for deciding whether to trust a source at all. Facts-only signals (price_divergence, text_cleanliness, a robust MAD cross-check) report alongside without moving the score.\n\nWhat is repaired automatically, what needs an explicit opt-in, and what is only ever reported is published in full at `GET /api/clean` - machine-readable, and auditable before you pay. Paid via x402: no account, no API key, no signup."
},
"servers": [
{
"url": "https://www.aidatatools.dev",
"description": "Production"
}
],
"paths": {
"/api": {
"get": {
"operationId": "getApiInfo",
"summary": "Usage information",
"description": "Returns a short machine-readable description of how to call POST /api. Does not run any analysis.",
"responses": {
"200": {
"description": "Usage info",
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
}
},
"tags": [
"verdict"
]
},
"post": {
"operationId": "checkDatasetQuality",
"summary": "Analyze a dataset and return a quality verdict",
"description": "Runs 8 deterministic checks (structure, completeness, nulls, types, impossible values, duplicates, outliers, cardinality) and returns facts, a weighted 0-100 score, and a verdict. Six of those checks carry the score; structure and cardinality are facts-only, as are the additional price_divergence, text_cleanliness and outliers.modified_z_mad signals. Never returns 5xx for bad *data* -- malformed, empty, or huge datasets degrade to a valid low verdict instead of an error. Only a malformed HTTP request body returns 4xx.",
"x-payment-info": {
"enabled": true,
"note": "x402 payment is implemented for two networks (see /llms-full.txt) but each activates independently only once its own payout address is set server-side; this static document cannot reflect that live state. Check GET /api's response body -- fields payment.x402Enabled and payment.activeNetworks -- for the current, real-time status before assuming a 402 will occur, and for which network(s) are actually offered.",
"x402Version": 2,
"protocols": [
"x402"
],
"price": {
"mode": "fixed",
"currency": "USD",
"amount": "0.01"
},
"scheme": "exact",
"amount": "10000",
"resource": "/api",
"mimeType": "application/json",
"facilitator": "https://facilitator.payai.network",
"networks": [
{
"network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
"asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
"payTo": null,
"envVar": "X402_PAY_TO"
},
{
"network": "eip155:8453",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"payTo": null,
"envVar": "X402_PAY_TO_BASE"
}
]
},
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/QualityGateRequest"
},
"examples": {
"bareArray": {
"summary": "Bare dataset array (convenience form)",
"value": [
{
"id": 1,
"price": 10
},
{
"id": 2,
"price": null
},
{
"id": 1,
"price": 10
}
]
},
"envelope": {
"summary": "Envelope form",
"value": {
"rawJson": [
{
"id": 1,
"price": 10
},
{
"id": 2,
"price": -5
}
]
}
}
}
}
}
},
"responses": {
"200": {
"description": "A quality verdict was produced (an UNRELIABLE verdict is a successful analysis, not an error). If x402 payment is active, includes a PAYMENT-RESPONSE header with the settlement receipt.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/QualityGateResponse"
}
}
}
},
"402": {
"description": "Payment required (only when x402 is active -- see x-payment-info above and GET /api). Body is an x402 PaymentRequired object; also mirrored in the PAYMENT-REQUIRED header, base64-encoded.",
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
},
"400": {
"description": "Request body is empty or not valid JSON (transport-level error, not a data-quality judgment).",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"413": {
"description": "Request body exceeds the size limit (4 MB).",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"tags": [
"verdict"
]
}
},
"/api/clean": {
"get": {
"operationId": "getRepairBoundary",
"tags": [
"clean"
],
"summary": "The repair boundary, machine-readable and free",
"description": "Free. Returns the complete ruleset: all 20 rules with their action (AUTO / OPT_IN / FLAG), what each does and WHY it sits at that action level, plus the 7 engine invariants. Published so a caller can audit exactly what will happen to their data before paying for it.",
"responses": {
"200": {
"description": "Usage information plus the full ruleset manifest.",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
}
},
"post": {
"operationId": "cleanScrapedData",
"tags": [
"clean"
],
"summary": "Repair scraped data and return it",
"description": "Call this on the output of any scrape or extraction, before you use, store or reason over the data. Returns the REPAIRED data as the response body, in the shape you posted it, with nested objects and arrays cleaned too.\n\nDeterministic and idempotent: the same input, options and ruleset_version always yield byte-identical output, and cleaning the result again changes nothing.\n\nSafe by construction. Seven AUTO rules are information-preserving and applied silently (mojibake round-trip iterated to a fixpoint; HTML tags stripped with a block/inline distinction; entities decoded exactly once with a re-entrancy guard; zero-width, BOM, soft-hyphen and control characters removed - except ZWNJ/ZWJ where a script or an emoji sequence needs them; the non-breaking space family folded to a plain space; whitespace trimmed). Five more require an explicit opt-in because they change the row count, a type, or the schema. Eight cases are only ever REPORTED with a proposal and no option can turn them on: near-duplicate merging, ambiguous placeholder nulling ('None' is a surname, 'NA' is Namibia), full NFKC (it rewrites 10^2 to 102), and failed extractions ('captcha', 'access denied') which are never deleted because that value tells you the record must be re-scraped.\n\nRead the full boundary at GET /api/clean before paying.",
"x-payment-info": {
"enabled": true,
"note": "x402 payment is implemented for two networks (see /llms-full.txt) but each activates independently only once its own payout address is set server-side; this static document cannot reflect that live state. Check GET /api's response body -- fields payment.x402Enabled and payment.activeNetworks -- for the current, real-time status before assuming a 402 will occur, and for which network(s) are actually offered.",
"x402Version": 2,
"protocols": [
"x402"
],
"price": {
"mode": "fixed",
"currency": "USD",
"amount": "0.04"
},
"scheme": "exact",
"amount": "40000",
"resource": "/api/clean",
"mimeType": "application/json",
"facilitator": "https://facilitator.payai.network",
"networks": [
{
"network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
"asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
"payTo": null,
"envVar": "X402_PAY_TO"
},
{
"network": "eip155:8453",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"payTo": null,
"envVar": "X402_PAY_TO_BASE"
}
]
},
"parameters": [
{
"name": "envelope",
"in": "query",
"required": false,
"schema": {
"type": "string",
"enum": [
"1",
"true"
]
},
"description": "By default the response body IS the cleaned data, in the shape you posted - nothing to unwrap. Set envelope=1 to receive {\"data\": ..., \"summary\": ...} instead."
}
],
"requestBody": {
"description": "Either the scraper output posted directly as the whole body (a JSON array of row objects, a single object, a CSV body or plain text - the format is detected), or an envelope {\"rawJson\": ..., \"options\": {...}}.",
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CleanRequest"
},
"examples": {
"bare_scrape": {
"summary": "the scrape, posted directly",
"value": [
{
"title": "Café Lamp
",
"price": "US $59.99",
"stock": "N/A"
}
]
},
"with_options": {
"summary": "with opt-in rules",
"value": {
"rawJson": [
{
"title": "Lamp"
}
],
"options": {
"drop_exact_duplicates": true,
"placeholder_policy": "null_high_confidence"
}
}
}
}
},
"text/csv": {
"schema": {
"type": "string"
},
"example": "sku,title\nA1,Lamp\nA2,Café\nA3,Chair\n"
},
"text/plain": {
"schema": {
"type": "string"
}
}
}
},
"responses": {
"200": {
"description": "The repaired data. By default the body is the cleaned payload itself; with ?envelope=1 it is {\"data\", \"summary\"}.",
"headers": {
"X-DQG-Repairs": {
"schema": {
"type": "string"
},
"description": "Number of transformations applied."
},
"X-DQG-Values-Changed": {
"schema": {
"type": "string"
},
"description": "Number of distinct values changed."
},
"X-DQG-Flags": {
"schema": {
"type": "string"
},
"description": "Cases reported for your decision, never repaired."
},
"X-DQG-Rows-In": {
"schema": {
"type": "string"
}
},
"X-DQG-Rows-Out": {
"schema": {
"type": "string"
},
"description": "Differs from Rows-In only if drop_exact_duplicates was requested."
},
"X-DQG-Rules-Applied": {
"schema": {
"type": "string"
},
"description": "Comma-separated rule ids, most frequent first."
},
"X-DQG-Ruleset": {
"schema": {
"type": "string"
},
"description": "Ruleset version; part of the replay id."
},
"X-DQG-Deterministic": {
"schema": {
"type": "string"
},
"description": "Always 'true; no-llm'."
}
},
"content": {
"application/json": {
"schema": {
"oneOf": [
{
"$ref": "#/components/schemas/CleanedData"
},
{
"$ref": "#/components/schemas/CleanEnvelope"
}
]
},
"examples": {
"default_body_is_the_data": {
"summary": "default: the body IS the cleaned data",
"value": [
{
"title": "Café Lamp",
"price": "US $59.99",
"stock": "N/A"
}
]
}
}
}
}
},
"402": {
"description": "Payment required (x402). Body is the x402 PaymentRequired document; the PAYMENT-REQUIRED header carries the same, encoded."
},
"400": {
"description": "Malformed HTTP request.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"404": {
"description": "The cleaning tiers are not enabled on this deployment (DQG_CLEAN_ENABLED unset). POST /api is unaffected.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"413": {
"description": "Request body too large.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
}
}
},
"/api/clean/audit": {
"post": {
"operationId": "cleanScrapedDataAudited",
"tags": [
"clean"
],
"summary": "Repair scraped data and return it with a full audit trail",
"description": "Identical repair to POST /api/clean - a test asserts the two return byte-identical `data`, because tier 3 is the same engine with the trace switched on - plus an `audit` block:\n\n- `ledger`: every transformation with its path, path_steps (machine-usable, never parsed from a string), rule, action, before and after. A complete inverse patch: applying it in reverse reconstructs your input byte for byte.\n- `replay_id`: sha256(input_sha256 | options_sha256 | ruleset_version). Two runs sharing a replay id produced identical data.\n- `input_sha256` / `output_sha256`, and the ruleset manifest as it was at run time.\n\nUse it when you must be able to prove later what changed and why. Otherwise use /api/clean, which returns the same data with a compact summary.",
"x-payment-info": {
"enabled": true,
"note": "x402 payment is implemented for two networks (see /llms-full.txt) but each activates independently only once its own payout address is set server-side; this static document cannot reflect that live state. Check GET /api's response body -- fields payment.x402Enabled and payment.activeNetworks -- for the current, real-time status before assuming a 402 will occur, and for which network(s) are actually offered.",
"x402Version": 2,
"protocols": [
"x402"
],
"price": {
"mode": "fixed",
"currency": "USD",
"amount": "0.12"
},
"scheme": "exact",
"amount": "120000",
"resource": "/api/clean/audit",
"mimeType": "application/json",
"facilitator": "https://facilitator.payai.network",
"networks": [
{
"network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
"asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
"payTo": null,
"envVar": "X402_PAY_TO"
},
{
"network": "eip155:8453",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"payTo": null,
"envVar": "X402_PAY_TO_BASE"
}
]
},
"parameters": [],
"requestBody": {
"description": "Either the scraper output posted directly as the whole body (a JSON array of row objects, a single object, a CSV body or plain text - the format is detected), or an envelope {\"rawJson\": ..., \"options\": {...}}.",
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CleanRequest"
},
"examples": {
"bare_scrape": {
"summary": "the scrape, posted directly",
"value": [
{
"title": "Café Lamp
",
"price": "US $59.99",
"stock": "N/A"
}
]
},
"with_options": {
"summary": "with opt-in rules",
"value": {
"rawJson": [
{
"title": "Lamp"
}
],
"options": {
"drop_exact_duplicates": true,
"placeholder_policy": "null_high_confidence"
}
}
}
}
},
"text/csv": {
"schema": {
"type": "string"
},
"example": "sku,title\nA1,Lamp\nA2,Café\nA3,Chair\n"
},
"text/plain": {
"schema": {
"type": "string"
}
}
}
},
"responses": {
"200": {
"description": "The repaired data, a summary, and the full audit block.",
"headers": {
"X-DQG-Repairs": {
"schema": {
"type": "string"
},
"description": "Number of transformations applied."
},
"X-DQG-Values-Changed": {
"schema": {
"type": "string"
},
"description": "Number of distinct values changed."
},
"X-DQG-Flags": {
"schema": {
"type": "string"
},
"description": "Cases reported for your decision, never repaired."
},
"X-DQG-Rows-In": {
"schema": {
"type": "string"
}
},
"X-DQG-Rows-Out": {
"schema": {
"type": "string"
},
"description": "Differs from Rows-In only if drop_exact_duplicates was requested."
},
"X-DQG-Rules-Applied": {
"schema": {
"type": "string"
},
"description": "Comma-separated rule ids, most frequent first."
},
"X-DQG-Ruleset": {
"schema": {
"type": "string"
},
"description": "Ruleset version; part of the replay id."
},
"X-DQG-Deterministic": {
"schema": {
"type": "string"
},
"description": "Always 'true; no-llm'."
}
},
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CleanAuditEnvelope"
}
}
}
},
"402": {
"description": "Payment required (x402). Body is the x402 PaymentRequired document; the PAYMENT-REQUIRED header carries the same, encoded."
},
"400": {
"description": "Malformed HTTP request.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"404": {
"description": "The cleaning tiers are not enabled on this deployment (DQG_CLEAN_ENABLED unset). POST /api is unaffected.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"413": {
"description": "Request body too large.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
}
}
}
},
"components": {
"schemas": {
"QualityGateRequest": {
"description": "Either a bare JSON array of row objects, or an envelope object.",
"oneOf": [
{
"type": "array",
"items": {
"type": "object"
},
"description": "The dataset to analyze, posted directly."
},
{
"$ref": "#/components/schemas/QualityGateEnvelope"
}
]
},
"QualityGateEnvelope": {
"type": "object",
"properties": {
"rawJson": {
"description": "The dataset: a JSON array of row objects, or a single object (treated as a 1-row dataset).",
"oneOf": [
{
"type": "array",
"items": {
"type": "object"
}
},
{
"type": "object"
}
]
},
"datasetId": {
"type": "string",
"description": "An Apify dataset id. Only resolvable if the deployment has Apify fetching configured server-side; otherwise the response degrades to an UNRELIABLE verdict explaining that rawJson must be used instead. Ignored if rawJson is also set."
}
}
},
"ErrorResponse": {
"type": "object",
"required": [
"error"
],
"properties": {
"error": {
"type": "string"
}
}
},
"QualityGateResponse": {
"type": "object",
"required": [
"meta",
"facts",
"score",
"verdict"
],
"properties": {
"meta": {
"$ref": "#/components/schemas/Meta"
},
"facts": {
"$ref": "#/components/schemas/Facts"
},
"score": {
"$ref": "#/components/schemas/Score"
},
"verdict": {
"$ref": "#/components/schemas/Verdict"
},
"benchmark": {
"$ref": "#/components/schemas/Benchmark"
}
}
},
"Meta": {
"type": "object",
"properties": {
"tool": {
"type": "string",
"examples": [
"data-quality-gate"
]
},
"version": {
"type": "string",
"examples": [
"0.1.0"
]
},
"generated_at": {
"type": "string",
"format": "date-time",
"description": "Wall-clock timestamp; metadata only, not part of the deterministic contract."
},
"input_source": {
"type": "string",
"enum": [
"rawJson",
"datasetId",
"none"
]
},
"warnings": {
"type": "array",
"items": {
"type": "string"
}
},
"errors": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"Facts": {
"type": "object",
"description": "One block per check. See /llms-full.txt for the full method behind each. Six checks carry the weighted score; `structure` and `cardinality` are facts-only. `price_divergence` and `text_cleanliness` are optional and additive -- each present only when it has something to report -- and, like `outliers.modified_z_mad`, they never affect score or verdict.",
"properties": {
"structure": {
"$ref": "#/components/schemas/StructureFacts"
},
"completeness": {
"$ref": "#/components/schemas/CompletenessFacts"
},
"nulls": {
"$ref": "#/components/schemas/NullsFacts"
},
"types": {
"$ref": "#/components/schemas/TypesFacts"
},
"impossible_values": {
"$ref": "#/components/schemas/ImpossibleValuesFacts"
},
"duplicates": {
"$ref": "#/components/schemas/DuplicatesFacts"
},
"outliers": {
"$ref": "#/components/schemas/OutliersFacts"
},
"cardinality": {
"$ref": "#/components/schemas/CardinalityFacts"
},
"price_divergence": {
"$ref": "#/components/schemas/PriceDivergenceFacts"
},
"text_cleanliness": {
"$ref": "#/components/schemas/TextCleanlinessFacts"
}
}
},
"StructureFacts": {
"type": "object",
"properties": {
"row_count": {
"type": "integer"
},
"field_count": {
"type": "integer"
},
"homogeneous": {
"type": "boolean"
},
"method": {
"type": "string"
}
},
"additionalProperties": true
},
"CompletenessFacts": {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"overall_presence_pct": {
"type": [
"number",
"null"
]
},
"complete_rows_pct": {
"type": [
"number",
"null"
]
},
"fully_empty_fields": {
"type": "array",
"items": {
"type": "string"
}
},
"fields_with_missing_values": {
"type": "array",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"present_pct": {
"type": "number"
}
}
}
}
},
"additionalProperties": true
},
"NullsFacts": {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"global_null_pct": {
"type": [
"number",
"null"
]
},
"fields_with_nulls": {
"type": "array",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"null_pct": {
"type": "number"
},
"null_count": {
"type": "integer"
}
}
}
}
},
"additionalProperties": true
},
"TypesFacts": {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"type_coherence_pct": {
"type": "number"
},
"mixed_type_fields": {
"type": "array",
"items": {
"type": "object"
}
},
"numeric_stored_as_text": {
"type": "array",
"items": {
"type": "object"
}
}
},
"additionalProperties": true
},
"ImpossibleValuesFacts": {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"fields_checked": {
"type": "integer"
},
"total_violations": {
"type": "integer"
},
"by_field": {
"type": "array",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"rules_applied": {
"type": "array",
"items": {
"type": "string"
}
},
"checked": {
"type": "integer"
},
"violations": {
"type": "integer"
},
"violation_pct": {
"type": "number"
}
}
}
}
},
"additionalProperties": true
},
"DuplicatesFacts": {
"type": "object",
"properties": {
"exact": {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"count": {
"type": "integer"
},
"pct": {
"type": "number"
},
"sample_groups": {
"type": "array",
"items": {
"type": "object",
"properties": {
"row_indices": {
"type": "array",
"items": {
"type": "integer"
}
},
"occurrences": {
"type": "integer"
}
}
}
}
}
},
"near": {
"type": "object",
"description": "skipped=true when no field qualifies for fuzzy text comparison, under 2 rows, or over 5000 rows (see /llms-full.txt).",
"properties": {
"skipped": {
"type": "boolean"
},
"reason": {
"type": "string"
},
"method": {
"type": "string"
},
"fields_used": {
"type": "array",
"items": {
"type": "string"
}
},
"threshold": {
"type": "number"
},
"count": {
"type": "integer"
},
"pct": {
"type": "number"
},
"sample_pairs": {
"type": "array",
"items": {
"type": "object"
}
}
}
}
}
},
"OutliersFacts": {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"fields_checked": {
"type": "integer"
},
"total_outliers": {
"type": "integer"
},
"by_field": {
"type": "array",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"checked": {
"type": "integer"
},
"outlier_count": {
"type": "integer"
},
"lower_bound": {
"type": "number"
},
"upper_bound": {
"type": "number"
},
"sample_outlier_values": {
"type": "array",
"items": {
"type": "number"
}
}
}
}
},
"skipped_fields": {
"type": "array",
"items": {
"type": "object"
}
},
"modified_z_mad": {
"type": "object",
"description": "Facts-only robust cross-check (Iglewicz-Hoaglin modified z-score: M = 0.6745*(x - median)/MAD, flagged when |M| > 3.5). Present only when it flags something. Never affects score or verdict, which are driven by the Tukey fence above. by_field[].caught_only_by_mad lists values the fence could not see: a cluster of corrupted values in one tail widens its bounds until it stops flagging anything. Skipped for a field whose MAD is 0.",
"properties": {
"method": {
"type": "string"
},
"scoring": {
"type": "string"
},
"fields_analyzed": {
"type": "integer"
},
"total_caught_only_by_mad": {
"type": "integer"
},
"by_field": {
"type": "array",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"checked": {
"type": "integer"
},
"median": {
"type": "number"
},
"mad": {
"type": "number"
},
"threshold": {
"type": "number"
},
"flagged_distinct_values": {
"type": "integer"
},
"sample_flagged_values": {
"type": "array",
"items": {
"type": "number"
}
},
"caught_only_by_mad": {
"type": "array",
"items": {
"type": "number"
}
},
"caught_only_by_mad_count": {
"type": "integer"
},
"skipped": {
"type": "boolean"
},
"reason": {
"type": "string"
}
}
}
}
},
"additionalProperties": true
}
},
"additionalProperties": true
},
"TextCleanlinessFacts": {
"type": "object",
"description": "Facts-only: extraction artifacts inside string fields. Present only when at least one value is flagged. Never affects score or verdict. Complements completeness ('is a value there?') and types ('is it the right type?'), neither of which can see that a present, well-typed string is an artifact rather than data.",
"properties": {
"applicable": {
"type": "boolean"
},
"method": {
"type": "string"
},
"fields_checked": {
"type": "integer"
},
"values_checked": {
"type": "integer"
},
"total_flagged_values": {
"type": "integer"
},
"by_field": {
"type": "array",
"description": "Sparse: only fields with findings. A value failing several categories is counted once in flagged_values.",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"checked": {
"type": "integer"
},
"flagged_values": {
"type": "integer"
},
"flagged_pct": {
"type": "number"
},
"issues": {
"type": "object",
"description": "Keyed by category: html_markup, executable_markup, boilerplate (with phrases_matched), mojibake, placeholder, invisible_chars (with characters_found), non_normalized. Each carries count, pct and sample_values.",
"additionalProperties": true
}
}
}
}
},
"additionalProperties": true
},
"CardinalityFacts": {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"constant_fields": {
"type": "array",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string"
}
}
}
},
"quasi_unique_fields": {
"type": "array",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"unique_ratio": {
"type": "number"
},
"likely_identifier": {
"type": "boolean"
}
}
}
}
},
"additionalProperties": true
},
"PriceDivergenceFacts": {
"type": "object",
"description": "Optional, additive block: present only when the dataset has a group-key field (symbol/ticker/asset/instrument/pair) paired with a price-like field (price/cost/rate). Not one of the 8 core checks -- never affects score/verdict. When present, `applicable` is always true (the endpoint only adds this key when it found something to check). See /llms-full.txt for the full method and its documented small-sample-size limits.",
"properties": {
"applicable": {
"type": "boolean"
},
"method": {
"type": "string"
},
"pairs_checked": {
"type": "array",
"items": {
"type": "object",
"properties": {
"group_field": {
"type": "string"
},
"value_field": {
"type": "string"
},
"groups_checked": {
"type": "integer"
},
"groups_flagged": {
"type": "integer"
},
"flagged_groups": {
"type": "array",
"items": {
"type": "object",
"description": "method=tukey_iqr (>=5 points/group): lower_bound/upper_bound/outlier_values are set. method=ratio_heuristic (2-4 points/group): min/max/ratio/threshold/note are set instead -- flags the group as internally inconsistent but, unlike tukey_iqr, cannot identify which specific value is wrong.",
"properties": {
"group": {
"type": "string"
},
"n": {
"type": "integer"
},
"method": {
"type": "string",
"enum": [
"tukey_iqr",
"ratio_heuristic"
]
},
"lower_bound": {
"type": "number"
},
"upper_bound": {
"type": "number"
},
"outlier_values": {
"type": "array",
"items": {
"type": "number"
}
},
"min": {
"type": "number"
},
"max": {
"type": "number"
},
"ratio": {
"type": "number"
},
"threshold": {
"type": "number"
},
"note": {
"type": "string"
}
}
}
}
}
}
},
"total_groups_flagged": {
"type": "integer"
}
},
"additionalProperties": true
},
"Score": {
"type": "object",
"properties": {
"overall": {
"type": "number",
"minimum": 0,
"maximum": 100
},
"weights": {
"type": "object",
"properties": {
"completeness": {
"type": "number"
},
"nulls": {
"type": "number"
},
"types": {
"type": "number"
},
"impossible_values": {
"type": "number"
},
"duplicates": {
"type": "number"
},
"outliers": {
"type": "number"
}
}
},
"components": {
"type": "object",
"properties": {
"completeness": {
"type": "number"
},
"nulls": {
"type": "number"
},
"types": {
"type": "number"
},
"impossible_values": {
"type": "number"
},
"duplicates": {
"type": "number"
},
"outliers": {
"type": "number"
}
}
},
"formula": {
"type": "string",
"examples": [
"overall = sum(weights[k] * components[k] for k in weights)"
]
}
}
},
"Verdict": {
"type": "object",
"properties": {
"level": {
"type": "string",
"enum": [
"RELIABLE",
"USABLE_WITH_CLEANING",
"UNRELIABLE"
]
},
"thresholds": {
"type": "object",
"properties": {
"RELIABLE": {
"type": "string"
},
"USABLE_WITH_CLEANING": {
"type": "string"
},
"UNRELIABLE": {
"type": "string"
}
}
},
"reasons": {
"type": "array",
"items": {
"type": "object",
"properties": {
"check": {
"type": "string"
},
"severity": {
"type": "string",
"enum": [
"high",
"medium",
"low"
]
},
"message": {
"type": "string"
}
}
}
},
"recommendation": {
"type": "string"
},
"level_capped": {
"type": "boolean",
"description": "True when the impossible_values hard gate prevented a RELIABLE verdict."
},
"level_before_cap": {
"type": "string",
"description": "Only present when level_capped is true."
},
"cap_reason": {
"type": "string",
"description": "Only present when level_capped is true."
}
},
"required": [
"level",
"thresholds",
"reasons",
"recommendation",
"level_capped"
]
},
"Benchmark": {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"input_tokens": {
"type": "integer"
},
"output_tokens": {
"type": "integer"
},
"reduction_pct": {
"type": "number"
}
}
},
"CleanRequest": {
"oneOf": [
{
"type": "array",
"items": {
"type": "object"
},
"description": "The scrape, posted directly."
},
{
"type": "object",
"description": "A single row object, posted directly."
},
{
"type": "object",
"description": "Envelope form.",
"properties": {
"rawJson": {
"description": "Array of row objects, a single object, or a string (JSON/CSV/text)."
},
"options": {
"$ref": "#/components/schemas/CleanOptions"
}
},
"required": [
"rawJson"
]
}
]
},
"CleanOptions": {
"type": "object",
"description": "All optional. Every default is the safe one: with no options the row count, every value's JSON type and the schema are guaranteed unchanged, and only the 7 information-preserving AUTO rules run.",
"properties": {
"placeholder_policy": {
"type": "string",
"enum": [
"flag",
"null_high_confidence",
"null_all"
],
"default": "flag",
"description": "'flag' (default) reports masked-missing strings and changes nothing. 'null_high_confidence' nulls only tokens that cannot be real data in any field (n/a, #n/a, null, undefined, nan, not available, tbd...) and never the ambiguous ones. 'null_all' extends it to the ambiguous tokens ('None' is a surname, 'NA' is Namibia, '-' is a real value) - the one setting that can destroy legitimate data; fully reversible from the tier-3 ledger."
},
"drop_exact_duplicates": {
"type": "boolean",
"default": false,
"description": "Remove rows byte-identical to an earlier row, compared AFTER cleaning. Off by default because it changes the row count; duplicates are reported either way."
},
"coerce_numeric_text": {
"type": "boolean",
"default": false,
"description": "Parse 'US $5.59' into 5.59, per field, all-or-nothing. A lone ',' (1.5 in fr/de, malformed in en), a '1.234'-shaped group, or a mixed currency affix disqualifies the whole field rather than being guessed at."
},
"repair_keys": {
"type": "boolean",
"default": false,
"description": "Also repair dict KEYS (the BOM-prefixed first column of a CSV export). Never overwrites an existing key. Off by default: a key is a contract with everything downstream."
},
"trim_whitespace": {
"type": "boolean",
"default": true,
"description": "Strip leading/trailing whitespace. Interior whitespace is never touched."
},
"detect_duplicates": {
"type": "boolean",
"default": true,
"description": "Set false to skip duplicate detection on very large input."
}
}
},
"CleanedData": {
"description": "The repaired data, mirroring the shape of the input.",
"oneOf": [
{
"type": "array",
"items": {
"type": "object"
}
},
{
"type": "object"
},
{
"type": "string"
}
]
},
"CleanSummary": {
"type": "object",
"description": "What changed and what needs your decision. Never the primary payload.",
"properties": {
"tier": {
"type": "string",
"enum": [
"clean",
"clean_audit"
]
},
"engine": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"ruleset_version": {
"type": "string"
},
"llm_used": {
"type": "boolean",
"enum": [
false
],
"description": "Always false."
}
}
},
"input": {
"type": "object",
"properties": {
"detected_format": {
"type": "string",
"enum": [
"array",
"object",
"csv",
"text",
"invalid"
]
},
"rows_in": {
"type": "integer"
},
"rows_out": {
"type": "integer"
},
"warnings": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"repaired": {
"type": "object",
"properties": {
"values_changed": {
"type": "integer"
},
"transformations_applied": {
"type": "integer"
},
"by_rule": {
"type": "array",
"items": {
"type": "object"
}
}
}
},
"needs_your_decision": {
"type": "object",
"description": "Every ambiguous case, with the proposal that was refused and - for OPT_IN rules - the exact option that would apply it.",
"properties": {
"count": {
"type": "integer"
},
"by_rule": {
"type": "array",
"items": {
"type": "object"
}
},
"items": {
"type": "array",
"items": {
"type": "object"
}
},
"truncated": {
"type": "boolean"
}
}
},
"options_used": {
"$ref": "#/components/schemas/CleanOptions"
},
"guarantees": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"CleanEnvelope": {
"type": "object",
"required": [
"data",
"summary"
],
"properties": {
"data": {
"$ref": "#/components/schemas/CleanedData"
},
"summary": {
"$ref": "#/components/schemas/CleanSummary"
}
}
},
"CleanAuditEnvelope": {
"type": "object",
"required": [
"data",
"summary",
"audit"
],
"properties": {
"data": {
"$ref": "#/components/schemas/CleanedData"
},
"summary": {
"$ref": "#/components/schemas/CleanSummary"
},
"audit": {
"type": "object",
"properties": {
"replay_id": {
"type": "string",
"description": "sha256(input | options | ruleset_version), 32 hex chars."
},
"replay_inputs": {
"type": "object",
"properties": {
"input_sha256": {
"type": "string"
},
"options_sha256": {
"type": "string"
},
"ruleset_version": {
"type": "string"
}
}
},
"output_sha256": {
"type": "string"
},
"generated_at": {
"type": "string",
"format": "date-time",
"description": "The only non-deterministic field; outside the hashed region."
},
"reversible": {
"type": "boolean"
},
"how_to_verify": {
"type": "string"
},
"ledger": {
"type": "array",
"items": {
"$ref": "#/components/schemas/LedgerEntry"
}
},
"dropped_row_indices": {
"type": "array",
"items": {
"type": "integer"
}
},
"ruleset": {
"type": "object",
"description": "The full boundary manifest as it was at run time."
},
"duplicate_facts": {
"type": "object"
}
}
}
}
},
"LedgerEntry": {
"type": "object",
"description": "One applied transformation. The ledger is an inverse patch, not a log.",
"properties": {
"path": {
"type": "string",
"example": "[0].specs[1].value"
},
"path_steps": {
"type": "array",
"description": "Machine-usable path; never parsed from the string form.",
"items": {
"oneOf": [
{
"type": "string"
},
{
"type": "integer"
}
]
}
},
"rule": {
"type": "string",
"example": "mojibake.roundtrip"
},
"action": {
"type": "string",
"enum": [
"AUTO",
"OPT_IN"
]
},
"before": {},
"after": {},
"detail": {
"type": "object"
}
}
}
}
},
"tags": [
{
"name": "clean",
"description": "Deterministic post-scrape repair: returns the cleaned data. Covers data cleaning, scraping repair, post-scrape sanitization, deterministic data repair, mojibake correction and encoding repair (HTML stripping, invisible-character removal, non-breaking space normalization). Repairs how data was encoded, never what it says."
},
{
"name": "verdict",
"description": "Deterministic quality judgement: score, per-check facts and a RELIABLE / USABLE_WITH_CLEANING / UNRELIABLE verdict. Dataset validation, duplicate detection, outlier detection, pipeline quality gate, RAG pipeline guardrail."
}
]
}