164 lines
6.4 KiB
TypeScript
164 lines
6.4 KiB
TypeScript
/**
|
|
* src/connectors/connector.ts
|
|
* ============================
|
|
* P061: THE connector contract. Nine operations, one interface, every
|
|
* connector implements all nine:
|
|
*
|
|
* validate — is this config usable? Pure, no IO, no network. Fail-closed.
|
|
* connect — open whatever session the source needs. Must be idempotent.
|
|
* discover — what partitions exist, and what shape do their records have?
|
|
* health — is the source answering right now?
|
|
* read — produce records from a cursor forward, bounded by `limit`.
|
|
* checkpoint — the cursor meaning "everything before this is produced".
|
|
* resume — continue from a previously committed checkpoint.
|
|
* backfill — re-read a bounded historical range. Bounded, never "all".
|
|
* disconnect — release the session. Must be idempotent and safe after an error.
|
|
*
|
|
* WHY AN INTERFACE AND NOT A BASE CLASS: connectors are heterogeneous by
|
|
* nature (an HTTP API paginates by opaque token, a file store seeks by byte
|
|
* offset, a change feed advances an LSN). The contract fixes the SHAPE of
|
|
* the interaction and the meaning of the cursor; it does not pretend the
|
|
* mechanics are the same. `src/connectors/conformance.ts` is what proves two
|
|
* unrelated implementations really do honour the same shape.
|
|
*
|
|
* WHAT THIS REPLACES, HONESTLY: services/api/sources/connectors.py has
|
|
* per-source `async def fetch_x()` functions and nothing else — no lifecycle,
|
|
* no cursor, no resume, no health, no disconnect. Those functions remain the
|
|
* Python engine's fetch layer; they are NOT retrofitted here (that engine is
|
|
* not deployed — api.0pt.app answers 404 — and rewriting it blind would be
|
|
* the exact "build it again" failure this programme forbids). This contract
|
|
* is the seam a Python connector is wrapped into when that engine is revived.
|
|
*/
|
|
import type { IngestionCheckpoint, IngestionMode } from "../types/ingestion";
|
|
import type { RecordSchemaFields } from "../schema/record-schema";
|
|
|
|
/** Config is connector-specific; the contract only requires it be serializable. */
|
|
export type ConnectorConfig = Record<string, unknown>;
|
|
|
|
export interface ConnectorContext {
|
|
tenant_id: string;
|
|
project_id: string;
|
|
source_id: string;
|
|
/** Correlates every operation of one ingestion run. */
|
|
run_id: string;
|
|
}
|
|
|
|
/** Result of `validate` — never throws, always answers. */
|
|
export interface ValidationResult {
|
|
valid: boolean;
|
|
/** One entry per problem. Empty iff valid. */
|
|
errors: string[];
|
|
}
|
|
|
|
/** One readable partition (a table, a topic, a file, a page family). */
|
|
export interface PartitionDescriptor {
|
|
partition_key: string;
|
|
/** Human label; may equal partition_key. */
|
|
name: string;
|
|
/** Record shape as the connector currently sees it — fed straight into the schema registry (P063). */
|
|
schema: RecordSchemaFields;
|
|
}
|
|
|
|
export interface DiscoveryResult {
|
|
partitions: PartitionDescriptor[];
|
|
/** Modes this connector can actually serve for this config. */
|
|
modes: IngestionMode[];
|
|
}
|
|
|
|
export interface ConnectorHealth {
|
|
status: "up" | "degraded" | "down";
|
|
latency_ms?: number;
|
|
/** Short operator-facing text — never a raw upstream body. */
|
|
detail?: string;
|
|
}
|
|
|
|
export interface ReadRequest {
|
|
partition_key: string;
|
|
/** Start AFTER this cursor. Undefined = from the beginning of the partition. */
|
|
cursor?: string;
|
|
/** Hard upper bound on records returned. Connectors MUST honour it. */
|
|
limit: number;
|
|
}
|
|
|
|
/** A record as the connector sees it, before the runner wraps it in an envelope. */
|
|
export interface ConnectorRecord {
|
|
/** Stable business key within the partition. */
|
|
record_key: string;
|
|
/** Null only for a delete in a change feed. */
|
|
payload: Record<string, unknown> | null;
|
|
operation?: "upsert" | "insert" | "update" | "delete";
|
|
occurred_at?: string;
|
|
/** Cursor value AT this record — the runner records it per envelope. */
|
|
cursor?: string;
|
|
}
|
|
|
|
export interface ReadResult {
|
|
records: ConnectorRecord[];
|
|
/**
|
|
* Cursor meaning "everything up to and including `records` has been
|
|
* produced". The runner commits this only after the records are accepted.
|
|
*/
|
|
next_cursor?: string;
|
|
/** False when the connector knows more records are immediately available. */
|
|
exhausted: boolean;
|
|
}
|
|
|
|
/** A bounded historical range. Unbounded backfills are rejected by the contract. */
|
|
export interface BackfillRequest {
|
|
partition_key: string;
|
|
/** Inclusive lower bound (cursor or ISO timestamp — connector-defined, documented per connector). */
|
|
from: string;
|
|
/** Exclusive upper bound. Required: a backfill with no end is a full reload, not a backfill. */
|
|
to: string;
|
|
limit: number;
|
|
}
|
|
|
|
export interface Connector {
|
|
readonly connector_id: string;
|
|
readonly connector_version: string;
|
|
|
|
/** Pure config check. MUST NOT perform IO. MUST NOT throw. */
|
|
validate(config: ConnectorConfig): ValidationResult;
|
|
|
|
/** Open a session. Idempotent: calling it twice is a no-op, not an error. */
|
|
connect(config: ConnectorConfig, ctx: ConnectorContext): Promise<void>;
|
|
|
|
/** Requires a live session. */
|
|
discover(): Promise<DiscoveryResult>;
|
|
|
|
/** Requires a live session. Never throws for an unhealthy source — reports it. */
|
|
health(): Promise<ConnectorHealth>;
|
|
|
|
/** Requires a live session. Honours `limit` exactly. */
|
|
read(req: ReadRequest): Promise<ReadResult>;
|
|
|
|
/**
|
|
* The connector's own view of where it stands for a partition. The DURABLE
|
|
* checkpoint is owned by the runner (CheckpointStore); this is the
|
|
* in-session position, used to cross-check a resume.
|
|
*/
|
|
checkpoint(partitionKey: string): Promise<{ partition_key: string; cursor?: string }>;
|
|
|
|
/**
|
|
* Continue from a previously committed checkpoint. Equivalent to
|
|
* `read({cursor: checkpoint.cursor})` for cursor-based connectors, but
|
|
* explicit so a connector that must re-establish server-side state (a
|
|
* replication slot, a scroll context) has a place to do it.
|
|
*/
|
|
resume(checkpoint: IngestionCheckpoint, limit: number): Promise<ReadResult>;
|
|
|
|
/** Bounded historical re-read. MUST reject an unbounded range. */
|
|
backfill(req: BackfillRequest): Promise<ReadResult>;
|
|
|
|
/** Idempotent; safe to call without a session and after an error. */
|
|
disconnect(): Promise<void>;
|
|
}
|
|
|
|
/** Thrown by connectors when an operation needs a session that was never opened. */
|
|
export class ConnectorNotConnectedError extends Error {
|
|
constructor(connectorId: string, operation: string) {
|
|
super(`connector ${connectorId}: ${operation} requires connect() first`);
|
|
this.name = "ConnectorNotConnectedError";
|
|
}
|
|
}
|