From 9961ac7e0365ba9529fda01315dd089106896b53 Mon Sep 17 00:00:00 2001 From: Simon Axelsson Date: Mon, 7 Sep 2026 18:09:05 +0200 Subject: [PATCH] refactor: make search definitions restart-safe --- src/search/search-service.ts | 192 ++++++++++++++++++++--------------- 1 file changed, 108 insertions(+), 84 deletions(-) diff --git a/src/search/search-service.ts b/src/search/search-service.ts index b551036..5a3450a 100644 --- a/src/search/search-service.ts +++ b/src/search/search-service.ts @@ -1,35 +1,15 @@ /** - * src/search/search-service.ts - * ============================= - * 0pt's Search capability (Master-DoD §10 Search): + * 0pt Search capability. * - * search.index -> registerIndexDefinition + indexDocuments (bulk) - * search.query -> query (tenant + purpose/rights filtering ENFORCED) - * search.rebuild -> rebuild (new generation, bulk load, validate) - * search.alias.switch -> switchGeneration (atomic alias cutover) - * - * Non-negotiables encoded here: - * - OpenSearch is NEVER canonical truth (§42): a rebuild takes a caller- - * supplied `DocumentSource` (an async iterator over canonical data) and - * can rebuild any index from an empty cluster. Deleting every index - * loses nothing canonical. - * - Tenant filter is applied by THIS service on every query — a caller - * cannot opt out; a missing tenant_id is INVALID_REQUEST (§31). - * - Purpose/rights filtering (§10): every document carries purpose_tags + - * rights_tags (mandatory on the envelope); every query names its purpose - * and gets only documents tagged for it. Documents with an EMPTY - * purpose_tags list are never returnable — fail closed. - * - Generation switch: physical indices are `opt-{name}--g{n}`; the alias - * `opt-{name}` points at exactly one generation; cutover is atomic - * (§42 generation isolation + cutover). - * - * Lineage: every completed rebuild emits an OpenLineage COMPLETE run to AUD0 - * (inputs: the document source name; outputs: the physical index). + * OpenSearch remains a disposable projection. Logical index definitions may be + * persisted through SearchDefinitionRepository so a process restart never + * forgets how to query or rebuild an otherwise valid projection. */ -import { SiaxError } from "../platform/errors"; -import type { OpenSearchAdapter } from "./opensearch-adapter"; -import { LineageEmitter } from "../lineage/lineage-emitter"; import type { Aud0Client } from "../clients/aud0-client"; +import { LineageEmitter } from "../lineage/lineage-emitter"; +import { SiaxError } from "../platform/errors"; +import type { SearchDefinitionRepository } from "../repositories/search-definition-repository"; +import type { OpenSearchAdapter } from "./opensearch-adapter"; import { MAX_SEARCH_PAGE_SIZE, type BulkIndexResult, @@ -41,9 +21,7 @@ import { type SearchResult, } from "./search-types"; -/** An async source of canonical documents for (re)builds. */ export interface DocumentSource { - /** Name for lineage, e.g. "postgres:opt_datasets" or "0pt:dataset:ds_x". */ name: string; iterate(): AsyncIterable; } @@ -59,24 +37,35 @@ const ENVELOPE_MAPPINGS: Record = { export class SearchService { private readonly definitions = new Map(); private readonly lineage?: LineageEmitter; + private readonly definitionRepository?: SearchDefinitionRepository; constructor( private readonly adapter: OpenSearchAdapter, - opts?: { aud0Client?: Aud0Client } + opts?: { aud0Client?: Aud0Client; definitionRepository?: SearchDefinitionRepository }, ) { this.lineage = opts?.aud0Client ? new LineageEmitter(opts.aud0Client) : undefined; + this.definitionRepository = opts?.definitionRepository; } // --------------------------------------------------------------------- // Index definitions // --------------------------------------------------------------------- - registerIndexDefinition(def: SearchIndexDefinition): SearchIndexDefinition { + private validateDefinition(def: SearchIndexDefinition): void { + if (!def.index_name?.trim() || !/^[a-z0-9][a-z0-9_-]{0,62}$/i.test(def.index_name)) { + throw new SiaxError("INVALID_REQUEST", "0pt.search", "index_name must be 1-63 letters, digits, _ or -"); + } + if (!Number.isSafeInteger(def.version) || def.version < 1) { + throw new SiaxError("INVALID_REQUEST", "0pt.search", "definition version must be a positive integer"); + } + if (Object.keys(def.fields).length === 0) { + throw new SiaxError("INVALID_REQUEST", "0pt.search", "an index definition needs at least one field"); + } for (const f of def.default_search_fields) { if (def.fields[f] !== "text") { throw new SiaxError( "INVALID_REQUEST", "0pt.search", - `default_search_field "${f}" must be a text field in index "${def.index_name}"` + `default_search_field "${f}" must be a text field in index "${def.index_name}"`, ); } } @@ -85,22 +74,59 @@ export class SearchService { throw new SiaxError( "INVALID_REQUEST", "0pt.search", - `field "${reserved}" is reserved by the 0pt search envelope` + `field "${reserved}" is reserved by the 0pt search envelope`, ); } } - this.definitions.set(def.index_name, def); - return def; + } + + /** In-memory registration retained for hermetic unit tests and local callers. */ + registerIndexDefinition(def: SearchIndexDefinition): SearchIndexDefinition { + this.validateDefinition(def); + const copy = { + ...def, + fields: { ...def.fields }, + default_search_fields: [...def.default_search_fields], + }; + this.definitions.set(def.index_name, copy); + return copy; + } + + /** Production registration: versioned durable write followed by cache refresh. */ + async registerIndexDefinitionDurable(def: SearchIndexDefinition, actorId?: string): Promise { + this.validateDefinition(def); + if (!this.definitionRepository) return this.registerIndexDefinition(def); + const stored = await this.definitionRepository.save(def, actorId); + return this.registerIndexDefinition(stored); } getIndexDefinition(indexName: string): SearchIndexDefinition { const def = this.definitions.get(indexName); if (!def) throw new SiaxError("NOT_FOUND", "0pt.search", `no index definition "${indexName}"`); - return def; + return { ...def, fields: { ...def.fields }, default_search_fields: [...def.default_search_fields] }; + } + + async resolveIndexDefinition(indexName: string): Promise { + const cached = this.definitions.get(indexName); + if (cached) return this.getIndexDefinition(indexName); + const stored = await this.definitionRepository?.latest(indexName); + if (!stored) throw new SiaxError("NOT_FOUND", "0pt.search", `no index definition "${indexName}"`); + return this.registerIndexDefinition(stored); } listIndexDefinitions(): SearchIndexDefinition[] { - return [...this.definitions.values()]; + return [...this.definitions.values()].map((def) => ({ + ...def, + fields: { ...def.fields }, + default_search_fields: [...def.default_search_fields], + })); + } + + async listIndexDefinitionsDurable(): Promise { + if (!this.definitionRepository) return this.listIndexDefinitions(); + const definitions = await this.definitionRepository.listLatest(); + for (const def of definitions) this.registerIndexDefinition(def); + return definitions; } aliasFor(indexName: string): string { @@ -113,51 +139,49 @@ export class SearchService { private mappingsFor(def: SearchIndexDefinition): Record { const properties: Record = { ...ENVELOPE_MAPPINGS }; - for (const [field, type] of Object.entries(def.fields)) { - properties[field] = { type }; - } + for (const [field, type] of Object.entries(def.fields)) properties[field] = { type }; return { properties, _meta: { opt_definition_version: def.version } }; } - // --------------------------------------------------------------------- - // search.index — bulk write into the ACTIVE generation - // --------------------------------------------------------------------- async indexDocuments(indexName: string, docs: SearchDocument[]): Promise { - const def = this.getIndexDefinition(indexName); + const def = await this.resolveIndexDefinition(indexName); const state = await this.generationState(indexName); if (state.active_generation === undefined) { throw new SiaxError( "CONFLICT", "0pt.search", - `index "${indexName}" has no active generation — run a rebuild first` + `index "${indexName}" has no active generation — run a rebuild first`, ); } this.validateDocuments(def, docs); const physical = this.physicalIndexFor(indexName, state.active_generation); const result = await this.adapter.bulkIndex( physical, - docs.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) })) + docs.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) })), ); return { indexed: docs.length - result.errors.length, errors: result.errors }; } - // --------------------------------------------------------------------- - // search.query — tenant + purpose/rights filtering ENFORCED here - // --------------------------------------------------------------------- async query(indexName: string, q: SearchQuery): Promise { - const def = this.getIndexDefinition(indexName); + const def = await this.resolveIndexDefinition(indexName); if (!q.tenant_id?.trim()) { throw new SiaxError("INVALID_REQUEST", "0pt.search", "tenant_id is required on every search query"); } if (!q.purpose?.trim()) { throw new SiaxError("INVALID_REQUEST", "0pt.search", "purpose is required on every search query (§10)"); } - const size = Math.min(q.size ?? 25, MAX_SEARCH_PAGE_SIZE); + const from = q.from ?? 0; + const requestedSize = q.size ?? 25; + if (!Number.isSafeInteger(from) || from < 0) { + throw new SiaxError("INVALID_REQUEST", "0pt.search", "from must be a non-negative integer"); + } + if (!Number.isSafeInteger(requestedSize) || requestedSize < 1) { + throw new SiaxError("INVALID_REQUEST", "0pt.search", "size must be a positive integer"); + } + const size = Math.min(requestedSize, MAX_SEARCH_PAGE_SIZE); const filter: unknown[] = [ { term: { tenant_id: q.tenant_id } }, - // Purpose fit: the document must be tagged for THIS query's purpose. - // Docs with empty purpose_tags match nothing — fail closed by shape. { term: { purpose_tags: q.purpose } }, { term: { rights_tags: q.rights_tag ?? "internal" } }, ]; @@ -171,10 +195,9 @@ export class SearchService { const must = q.text?.trim() ? [{ multi_match: { query: q.text, fields: def.default_search_fields } }] : [{ match_all: {} }]; - const res = await this.adapter.search(this.aliasFor(indexName), { query: { bool: { must, filter } }, - from: q.from ?? 0, + from, size, }); @@ -196,30 +219,27 @@ export class SearchService { }; } - // --------------------------------------------------------------------- - // search.rebuild — new generation from canonical truth (§42) - // --------------------------------------------------------------------- async rebuild( indexName: string, source: DocumentSource, opts?: { - /** Switch the alias to the new generation when done (default true). */ switch?: boolean; - /** Delete superseded generations after a successful switch (default false — keep for rollback). */ prune?: boolean; batchSize?: number; tenant_id?: string; correlation_id?: string; - } + }, ): Promise { - const def = this.getIndexDefinition(indexName); + const def = await this.resolveIndexDefinition(indexName); const state = await this.generationState(indexName); const newGeneration = (state.active_generation ?? 0) + 1; const physical = this.physicalIndexFor(indexName, newGeneration); + const batchSize = opts?.batchSize ?? 500; + if (!Number.isSafeInteger(batchSize) || batchSize < 1 || batchSize > 5_000) { + throw new SiaxError("INVALID_REQUEST", "0pt.search", "batchSize must be between 1 and 5000"); + } await this.adapter.createIndex(physical, this.mappingsFor(def)); - - const batchSize = opts?.batchSize ?? 500; let batch: SearchDocument[] = []; let docsIndexed = 0; const flush = async () => { @@ -227,28 +247,33 @@ export class SearchService { this.validateDocuments(def, batch); const res = await this.adapter.bulkIndex( physical, - batch.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) })) + batch.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) })), ); if (res.errors.length > 0) { - // A rebuild must be all-or-nothing: destroy the failed generation and - // leave the active alias untouched (generation isolation, §42). await this.adapter.deleteIndex(physical); throw new SiaxError( "RETRYABLE_FAILURE", "0pt.search", - `rebuild of "${indexName}" g${newGeneration} failed on ${res.errors.length} document(s); generation discarded` + `rebuild of "${indexName}" g${newGeneration} failed on ${res.errors.length} document(s); generation discarded`, ); } docsIndexed += batch.length; batch = []; }; - for await (const doc of source.iterate()) { - batch.push(doc); - if (batch.length >= batchSize) await flush(); + try { + for await (const doc of source.iterate()) { + batch.push(doc); + if (batch.length >= batchSize) await flush(); + } + await flush(); + await this.adapter.refresh(physical); + } catch (error) { + // Best-effort cleanup of an incomplete generation. Preserve the original + // failure even if cleanup itself fails. + await this.adapter.deleteIndex(physical).catch(() => undefined); + throw error; } - await flush(); - await this.adapter.refresh(physical); const shouldSwitch = opts?.switch !== false; if (shouldSwitch) { @@ -280,10 +305,11 @@ export class SearchService { }; } - // --------------------------------------------------------------------- - // search.alias.switch — explicit generation cutover / rollback - // --------------------------------------------------------------------- async switchGeneration(indexName: string, generation: number): Promise { + if (!Number.isSafeInteger(generation) || generation < 1) { + throw new SiaxError("INVALID_REQUEST", "0pt.search", "generation must be a positive integer"); + } + await this.resolveIndexDefinition(indexName); const physical = this.physicalIndexFor(indexName, generation); if (!(await this.adapter.indexExists(physical))) { throw new SiaxError("NOT_FOUND", "0pt.search", `generation g${generation} of "${indexName}" does not exist`); @@ -308,12 +334,7 @@ export class SearchService { }; } - // --------------------------------------------------------------------- - // internals - // --------------------------------------------------------------------- private docKey(d: SearchDocument): string { - // Physical _id is tenant-prefixed so two tenants can use the same doc_id - // without collision — defense in depth under the tenant term filter. return `${d.tenant_id}:${d.doc_id}`; } @@ -330,10 +351,13 @@ export class SearchService { private validateDocuments(def: SearchIndexDefinition, docs: SearchDocument[]): void { for (const d of docs) { + if (!d.doc_id?.trim()) { + throw new SiaxError("INVALID_REQUEST", "0pt.search", "every document requires doc_id"); + } if (!d.tenant_id?.trim()) { throw new SiaxError("INVALID_REQUEST", "0pt.search", `document ${d.doc_id} is missing tenant_id`); } - if (!Array.isArray(d.purpose_tags)) { + if (!Array.isArray(d.purpose_tags) || d.purpose_tags.length === 0) { throw new SiaxError("INVALID_REQUEST", "0pt.search", `document ${d.doc_id} is missing purpose_tags`); } if (!Array.isArray(d.rights_tags) || d.rights_tags.length === 0) { @@ -344,7 +368,7 @@ export class SearchService { throw new SiaxError( "INVALID_REQUEST", "0pt.search", - `document ${d.doc_id} has field "${field}" not in index definition "${def.index_name}" v${def.version}` + `document ${d.doc_id} has field "${field}" not in index definition "${def.index_name}" v${def.version}`, ); } }