Merges github/main (6 commits: 17-bug audit fix, CI quality gate,
.env.example + .gitignore fix, missing platform clients + registry
services, docs accuracy overhaul) into gitea/main (8 commits: Iceberg
write boundary + live ST0RE verification, lakehouse mapping routes,
Waves 1-4 foundational hardening).
16 conflicts. Both sides had independently written the four
src/clients/*.ts and the two registry services, so those are combined
rather than picked:
- st0re-client.ts: GitHub's endpoint wins on evidence. POST goes to
/v1/st0re/artifacts/register, not /v1/st0re/artifacts -- verified in
sax3l/st0re src/api/routes/artifacts.ts, where the bare path is a
GET-only list route. Gitea's URL normalisation, error-body text and
mock kept.
- n0d-client.ts: field is `status`, not `state`, with N0D's real 13-state
union -- verified in sax3l/n0d packages/schemas/n0d.job.schema.json,
which says explicitly not to add "succeeded". Gitea's mock echo,
URL normalisation and error-body text kept.
- cl0ud-client.ts: Gitea's fail-closed HTTP normalisation kept (a 200
with no `decision` reads as "deny", not as a silent allow). GitHub's
PolicyDecision/PolicyDecisionValue naming kept; PolicyResult retained
as an alias.
- aud0-client.ts: permissive union of both input types (optional
payload/metadata/correlation_id), GitHub's echo return, Gitea's URL
normalisation and error-body text, plus a JSON-parse fallback so a
successful POST with an empty body does not throw.
- dataset-service.ts / source-service.ts: GitHub's record/DTO split and
event naming (0pt.source_created / 0pt.create_source, which the merged
docs already reference) plus Gitea's name validation, description /
created_by fields and actor_id derivation. List filters are the union.
- lakehouse mapping routes: GitHub's shared lakehouse-mapping-registry
singletons (without which every write 400'd "mapping not found") AND
Gitea's Zod parseBody validation. Both, not either.
- src/types/source.ts, src/types/dataset.ts: unions of both vocabularies;
GitHub's `access` legal-sourcing field kept.
- .env.example: GitHub's audited inventory plus Gitea's Zitadel /
AUTH_ENFORCE block.
- CLAUDE.md, docs/ICEBERG_WRITE_BOUNDARY.md: GitHub's corrections kept,
Gitea's live-verification kept and marked as superseding the
"would 400 outright" finding (Gitea had already added the required
X-Tenant-Id/X-Actor-Id headers).
- pnpm-lock.yaml: regenerated with pnpm 9.15.9, not merged.
Also corrects CLAUDE.md's claim that the banned services/scrapers stack
was "since removed" -- the removal PR is open, not merged, and this
branch inherits that code from Gitea main. Nothing here resurrects,
extends or re-wires any of it; services/scrapers/ is byte-identical to
Gitea main.
Verified on the merged tree: typecheck (src/ + tests/) clean;
17 test files, 121 passed, 1 skipped (the realinfra test skipping
honestly on ST0RE HTTP 401), 0 failures. Gitea main baseline was the
same 121/1; GitHub main baseline could not even `pnpm install`
("packages field missing or empty") -- Gitea's pnpm-workspace.yaml fix
resolves that. Whole-repo `typecheck:app` reports 9 errors, an identical
set to Gitea main's baseline, so it is advisory in both CI workflows
rather than deleted. Lint: 16 problems, identical to Gitea baseline.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
145 lines
5.3 KiB
TypeScript
145 lines
5.3 KiB
TypeScript
/**
|
|
* src/services/source-service.ts
|
|
* ===============================
|
|
* Owns 0pt.source.schema.v1 — the per-tenant registry of upstream data
|
|
* sources (databases, crawl domains, APIs, file drops). This is the registry
|
|
* `SourceQueryService.querySource()` (src/services/source-query-service.ts)
|
|
* and `SourceRatePolicyService` (src/services/source-rate-policy-service.ts)
|
|
* both look a `source_id` up against — see docs/P0_QUERY_SOURCE_FLOW.md.
|
|
*
|
|
* Deliberately small and in-memory, same "pure core, thin registry" shape
|
|
* every other P0 service in this repo already uses; a real deployment backs
|
|
* this with `opt_sources`. In-memory scope: each `new SourceService()` owns
|
|
* its own registry — the pre-existing limitation docs/P0_QUERY_SOURCE_FLOW.md's
|
|
* "Known gaps" section documents (a Source registered via one API route module
|
|
* is not yet visible to another route module in the same running process).
|
|
* Callers that need a shared view pass an existing instance in via the
|
|
* `sourceService` constructor option, exactly like SourceQueryService and
|
|
* SourceRatePolicyService already do.
|
|
*
|
|
* Registering a source emits a best-effort AUD0 `0pt.source_created` event —
|
|
* the same "no mutation without an evented trail" doctrine every other P0
|
|
* registry service follows. With no AUD0_URL configured that emission is a
|
|
* no-op (see aud0-client.ts), so this stays hermetic in tests and local dev.
|
|
*/
|
|
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
|
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
|
|
|
|
/**
|
|
* Storage shape: nullable columns, exactly as the eventual `opt_sources` table
|
|
* would hold them. `toSource()` converts back to the `Source` DTO (undefined
|
|
* rather than null) so callers never have to handle both.
|
|
*/
|
|
interface SourceRecord {
|
|
source_id: string;
|
|
tenant_id: string;
|
|
project_id: string;
|
|
source_type: string;
|
|
name: string;
|
|
adapter_id: string | null;
|
|
access: string | null;
|
|
description: string | null;
|
|
status: string;
|
|
correlation_id: string | null;
|
|
created_at: string;
|
|
updated_at: string | null;
|
|
created_by: string | null;
|
|
}
|
|
|
|
export class SourceService {
|
|
private sources: Map<string, SourceRecord> = new Map();
|
|
private aud0: Aud0Client;
|
|
|
|
constructor(aud0BaseUrl?: string) {
|
|
this.aud0 = createAud0Client(aud0BaseUrl);
|
|
}
|
|
|
|
async createSource(input: CreateSourceInput): Promise<Source> {
|
|
if (this.sources.has(input.source_id)) {
|
|
throw new Error(`Source ${input.source_id} already exists`);
|
|
}
|
|
if (!input.name) {
|
|
throw new Error("name is required");
|
|
}
|
|
|
|
const now = new Date().toISOString();
|
|
const record: SourceRecord = {
|
|
source_id: input.source_id,
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
source_type: input.source_type,
|
|
name: input.name,
|
|
adapter_id: input.adapter_id ?? null,
|
|
access: input.access ?? null,
|
|
description: input.description ?? null,
|
|
status: input.status ?? "active",
|
|
correlation_id: input.correlation_id ?? null,
|
|
created_at: now,
|
|
updated_at: null,
|
|
created_by: input.created_by ?? null,
|
|
};
|
|
this.sources.set(record.source_id, record);
|
|
|
|
await this.aud0.ingestEvent({
|
|
event_id: `evt_0pt_source_created_${record.source_id}`,
|
|
event_type: "0pt.source_created",
|
|
tenant_id: record.tenant_id,
|
|
project_id: record.project_id,
|
|
actor_id: record.created_by ?? "system",
|
|
app_id: "0pt",
|
|
capability_id: "0pt.create_source",
|
|
resource_type: "source",
|
|
resource_id: record.source_id,
|
|
risk_level: "L2",
|
|
payload: {
|
|
source_type: record.source_type,
|
|
name: record.name,
|
|
adapter_id: record.adapter_id,
|
|
access: record.access,
|
|
},
|
|
created_at: now,
|
|
correlation_id: record.correlation_id ?? undefined,
|
|
});
|
|
|
|
return this.toSource(record);
|
|
}
|
|
|
|
async getSource(sourceId: string): Promise<Source> {
|
|
const record = this.sources.get(sourceId);
|
|
if (!record) throw new Error(`Source ${sourceId} not found`);
|
|
return this.toSource(record);
|
|
}
|
|
|
|
async listSources(filter?: {
|
|
tenant_id?: string;
|
|
project_id?: string;
|
|
source_type?: SourceType;
|
|
status?: SourceStatus;
|
|
}): Promise<Source[]> {
|
|
let results = Array.from(this.sources.values());
|
|
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
|
|
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
|
|
if (filter?.source_type) results = results.filter((r) => r.source_type === filter.source_type);
|
|
if (filter?.status) results = results.filter((r) => r.status === filter.status);
|
|
return results.map((r) => this.toSource(r));
|
|
}
|
|
|
|
private toSource(r: SourceRecord): Source {
|
|
return {
|
|
source_id: r.source_id,
|
|
tenant_id: r.tenant_id,
|
|
project_id: r.project_id,
|
|
source_type: r.source_type as SourceType,
|
|
name: r.name,
|
|
adapter_id: r.adapter_id ?? undefined,
|
|
access: (r.access as Source["access"]) ?? undefined,
|
|
description: r.description ?? undefined,
|
|
status: r.status as SourceStatus,
|
|
correlation_id: r.correlation_id ?? undefined,
|
|
created_at: r.created_at,
|
|
updated_at: r.updated_at ?? undefined,
|
|
created_by: r.created_by ?? undefined,
|
|
};
|
|
}
|
|
}
|