Files
0pt/src/connectors/connector.ts
T

164 lines
6.4 KiB
TypeScript

/**
* src/connectors/connector.ts
* ============================
* P061: THE connector contract. Nine operations, one interface, every
* connector implements all nine:
*
* validate — is this config usable? Pure, no IO, no network. Fail-closed.
* connect — open whatever session the source needs. Must be idempotent.
* discover — what partitions exist, and what shape do their records have?
* health — is the source answering right now?
* read — produce records from a cursor forward, bounded by `limit`.
* checkpoint — the cursor meaning "everything before this is produced".
* resume — continue from a previously committed checkpoint.
* backfill — re-read a bounded historical range. Bounded, never "all".
* disconnect — release the session. Must be idempotent and safe after an error.
*
* WHY AN INTERFACE AND NOT A BASE CLASS: connectors are heterogeneous by
* nature (an HTTP API paginates by opaque token, a file store seeks by byte
* offset, a change feed advances an LSN). The contract fixes the SHAPE of
* the interaction and the meaning of the cursor; it does not pretend the
* mechanics are the same. `src/connectors/conformance.ts` is what proves two
* unrelated implementations really do honour the same shape.
*
* WHAT THIS REPLACES, HONESTLY: services/api/sources/connectors.py has
* per-source `async def fetch_x()` functions and nothing else — no lifecycle,
* no cursor, no resume, no health, no disconnect. Those functions remain the
* Python engine's fetch layer; they are NOT retrofitted here (that engine is
* not deployed — api.0pt.app answers 404 — and rewriting it blind would be
* the exact "build it again" failure this programme forbids). This contract
* is the seam a Python connector is wrapped into when that engine is revived.
*/
import type { IngestionCheckpoint, IngestionMode } from "../types/ingestion";
import type { RecordSchemaFields } from "../schema/record-schema";
/** Config is connector-specific; the contract only requires it be serializable. */
export type ConnectorConfig = Record<string, unknown>;
export interface ConnectorContext {
tenant_id: string;
project_id: string;
source_id: string;
/** Correlates every operation of one ingestion run. */
run_id: string;
}
/** Result of `validate` — never throws, always answers. */
export interface ValidationResult {
valid: boolean;
/** One entry per problem. Empty iff valid. */
errors: string[];
}
/** One readable partition (a table, a topic, a file, a page family). */
export interface PartitionDescriptor {
partition_key: string;
/** Human label; may equal partition_key. */
name: string;
/** Record shape as the connector currently sees it — fed straight into the schema registry (P063). */
schema: RecordSchemaFields;
}
export interface DiscoveryResult {
partitions: PartitionDescriptor[];
/** Modes this connector can actually serve for this config. */
modes: IngestionMode[];
}
export interface ConnectorHealth {
status: "up" | "degraded" | "down";
latency_ms?: number;
/** Short operator-facing text — never a raw upstream body. */
detail?: string;
}
export interface ReadRequest {
partition_key: string;
/** Start AFTER this cursor. Undefined = from the beginning of the partition. */
cursor?: string;
/** Hard upper bound on records returned. Connectors MUST honour it. */
limit: number;
}
/** A record as the connector sees it, before the runner wraps it in an envelope. */
export interface ConnectorRecord {
/** Stable business key within the partition. */
record_key: string;
/** Null only for a delete in a change feed. */
payload: Record<string, unknown> | null;
operation?: "upsert" | "insert" | "update" | "delete";
occurred_at?: string;
/** Cursor value AT this record — the runner records it per envelope. */
cursor?: string;
}
export interface ReadResult {
records: ConnectorRecord[];
/**
* Cursor meaning "everything up to and including `records` has been
* produced". The runner commits this only after the records are accepted.
*/
next_cursor?: string;
/** False when the connector knows more records are immediately available. */
exhausted: boolean;
}
/** A bounded historical range. Unbounded backfills are rejected by the contract. */
export interface BackfillRequest {
partition_key: string;
/** Inclusive lower bound (cursor or ISO timestamp — connector-defined, documented per connector). */
from: string;
/** Exclusive upper bound. Required: a backfill with no end is a full reload, not a backfill. */
to: string;
limit: number;
}
export interface Connector {
readonly connector_id: string;
readonly connector_version: string;
/** Pure config check. MUST NOT perform IO. MUST NOT throw. */
validate(config: ConnectorConfig): ValidationResult;
/** Open a session. Idempotent: calling it twice is a no-op, not an error. */
connect(config: ConnectorConfig, ctx: ConnectorContext): Promise<void>;
/** Requires a live session. */
discover(): Promise<DiscoveryResult>;
/** Requires a live session. Never throws for an unhealthy source — reports it. */
health(): Promise<ConnectorHealth>;
/** Requires a live session. Honours `limit` exactly. */
read(req: ReadRequest): Promise<ReadResult>;
/**
* The connector's own view of where it stands for a partition. The DURABLE
* checkpoint is owned by the runner (CheckpointStore); this is the
* in-session position, used to cross-check a resume.
*/
checkpoint(partitionKey: string): Promise<{ partition_key: string; cursor?: string }>;
/**
* Continue from a previously committed checkpoint. Equivalent to
* `read({cursor: checkpoint.cursor})` for cursor-based connectors, but
* explicit so a connector that must re-establish server-side state (a
* replication slot, a scroll context) has a place to do it.
*/
resume(checkpoint: IngestionCheckpoint, limit: number): Promise<ReadResult>;
/** Bounded historical re-read. MUST reject an unbounded range. */
backfill(req: BackfillRequest): Promise<ReadResult>;
/** Idempotent; safe to call without a session and after an error. */
disconnect(): Promise<void>;
}
/** Thrown by connectors when an operation needs a session that was never opened. */
export class ConnectorNotConnectedError extends Error {
constructor(connectorId: string, operation: string) {
super(`connector ${connectorId}: ${operation} requires connect() first`);
this.name = "ConnectorNotConnectedError";
}
}