refactor: make search definitions restart-safe

This commit is contained in:
2026-09-07 18:09:05 +02:00
parent 7e2b5304b7
commit 9961ac7e03
+108 -84
View File
@@ -1,35 +1,15 @@
/**
* src/search/search-service.ts
* =============================
* 0pt's Search capability (Master-DoD §10 Search):
* 0pt Search capability.
*
* search.index -> registerIndexDefinition + indexDocuments (bulk)
* search.query -> query (tenant + purpose/rights filtering ENFORCED)
* search.rebuild -> rebuild (new generation, bulk load, validate)
* search.alias.switch -> switchGeneration (atomic alias cutover)
*
* Non-negotiables encoded here:
* - OpenSearch is NEVER canonical truth (§42): a rebuild takes a caller-
* supplied `DocumentSource` (an async iterator over canonical data) and
* can rebuild any index from an empty cluster. Deleting every index
* loses nothing canonical.
* - Tenant filter is applied by THIS service on every query — a caller
* cannot opt out; a missing tenant_id is INVALID_REQUEST (§31).
* - Purpose/rights filtering (§10): every document carries purpose_tags +
* rights_tags (mandatory on the envelope); every query names its purpose
* and gets only documents tagged for it. Documents with an EMPTY
* purpose_tags list are never returnable — fail closed.
* - Generation switch: physical indices are `opt-{name}--g{n}`; the alias
* `opt-{name}` points at exactly one generation; cutover is atomic
* (§42 generation isolation + cutover).
*
* Lineage: every completed rebuild emits an OpenLineage COMPLETE run to AUD0
* (inputs: the document source name; outputs: the physical index).
* OpenSearch remains a disposable projection. Logical index definitions may be
* persisted through SearchDefinitionRepository so a process restart never
* forgets how to query or rebuild an otherwise valid projection.
*/
import { SiaxError } from "../platform/errors";
import type { OpenSearchAdapter } from "./opensearch-adapter";
import { LineageEmitter } from "../lineage/lineage-emitter";
import type { Aud0Client } from "../clients/aud0-client";
import { LineageEmitter } from "../lineage/lineage-emitter";
import { SiaxError } from "../platform/errors";
import type { SearchDefinitionRepository } from "../repositories/search-definition-repository";
import type { OpenSearchAdapter } from "./opensearch-adapter";
import {
MAX_SEARCH_PAGE_SIZE,
type BulkIndexResult,
@@ -41,9 +21,7 @@ import {
type SearchResult,
} from "./search-types";
/** An async source of canonical documents for (re)builds. */
export interface DocumentSource {
/** Name for lineage, e.g. "postgres:opt_datasets" or "0pt:dataset:ds_x". */
name: string;
iterate(): AsyncIterable<SearchDocument>;
}
@@ -59,24 +37,35 @@ const ENVELOPE_MAPPINGS: Record<string, unknown> = {
export class SearchService {
private readonly definitions = new Map<string, SearchIndexDefinition>();
private readonly lineage?: LineageEmitter;
private readonly definitionRepository?: SearchDefinitionRepository;
constructor(
private readonly adapter: OpenSearchAdapter,
opts?: { aud0Client?: Aud0Client }
opts?: { aud0Client?: Aud0Client; definitionRepository?: SearchDefinitionRepository },
) {
this.lineage = opts?.aud0Client ? new LineageEmitter(opts.aud0Client) : undefined;
this.definitionRepository = opts?.definitionRepository;
}
// ---------------------------------------------------------------------
// Index definitions
// ---------------------------------------------------------------------
registerIndexDefinition(def: SearchIndexDefinition): SearchIndexDefinition {
private validateDefinition(def: SearchIndexDefinition): void {
if (!def.index_name?.trim() || !/^[a-z0-9][a-z0-9_-]{0,62}$/i.test(def.index_name)) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "index_name must be 1-63 letters, digits, _ or -");
}
if (!Number.isSafeInteger(def.version) || def.version < 1) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "definition version must be a positive integer");
}
if (Object.keys(def.fields).length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "an index definition needs at least one field");
}
for (const f of def.default_search_fields) {
if (def.fields[f] !== "text") {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.search",
`default_search_field "${f}" must be a text field in index "${def.index_name}"`
`default_search_field "${f}" must be a text field in index "${def.index_name}"`,
);
}
}
@@ -85,22 +74,59 @@ export class SearchService {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.search",
`field "${reserved}" is reserved by the 0pt search envelope`
`field "${reserved}" is reserved by the 0pt search envelope`,
);
}
}
this.definitions.set(def.index_name, def);
return def;
}
/** In-memory registration retained for hermetic unit tests and local callers. */
registerIndexDefinition(def: SearchIndexDefinition): SearchIndexDefinition {
this.validateDefinition(def);
const copy = {
...def,
fields: { ...def.fields },
default_search_fields: [...def.default_search_fields],
};
this.definitions.set(def.index_name, copy);
return copy;
}
/** Production registration: versioned durable write followed by cache refresh. */
async registerIndexDefinitionDurable(def: SearchIndexDefinition, actorId?: string): Promise<SearchIndexDefinition> {
this.validateDefinition(def);
if (!this.definitionRepository) return this.registerIndexDefinition(def);
const stored = await this.definitionRepository.save(def, actorId);
return this.registerIndexDefinition(stored);
}
getIndexDefinition(indexName: string): SearchIndexDefinition {
const def = this.definitions.get(indexName);
if (!def) throw new SiaxError("NOT_FOUND", "0pt.search", `no index definition "${indexName}"`);
return def;
return { ...def, fields: { ...def.fields }, default_search_fields: [...def.default_search_fields] };
}
async resolveIndexDefinition(indexName: string): Promise<SearchIndexDefinition> {
const cached = this.definitions.get(indexName);
if (cached) return this.getIndexDefinition(indexName);
const stored = await this.definitionRepository?.latest(indexName);
if (!stored) throw new SiaxError("NOT_FOUND", "0pt.search", `no index definition "${indexName}"`);
return this.registerIndexDefinition(stored);
}
listIndexDefinitions(): SearchIndexDefinition[] {
return [...this.definitions.values()];
return [...this.definitions.values()].map((def) => ({
...def,
fields: { ...def.fields },
default_search_fields: [...def.default_search_fields],
}));
}
async listIndexDefinitionsDurable(): Promise<SearchIndexDefinition[]> {
if (!this.definitionRepository) return this.listIndexDefinitions();
const definitions = await this.definitionRepository.listLatest();
for (const def of definitions) this.registerIndexDefinition(def);
return definitions;
}
aliasFor(indexName: string): string {
@@ -113,51 +139,49 @@ export class SearchService {
private mappingsFor(def: SearchIndexDefinition): Record<string, unknown> {
const properties: Record<string, unknown> = { ...ENVELOPE_MAPPINGS };
for (const [field, type] of Object.entries(def.fields)) {
properties[field] = { type };
}
for (const [field, type] of Object.entries(def.fields)) properties[field] = { type };
return { properties, _meta: { opt_definition_version: def.version } };
}
// ---------------------------------------------------------------------
// search.index — bulk write into the ACTIVE generation
// ---------------------------------------------------------------------
async indexDocuments(indexName: string, docs: SearchDocument[]): Promise<BulkIndexResult> {
const def = this.getIndexDefinition(indexName);
const def = await this.resolveIndexDefinition(indexName);
const state = await this.generationState(indexName);
if (state.active_generation === undefined) {
throw new SiaxError(
"CONFLICT",
"0pt.search",
`index "${indexName}" has no active generation — run a rebuild first`
`index "${indexName}" has no active generation — run a rebuild first`,
);
}
this.validateDocuments(def, docs);
const physical = this.physicalIndexFor(indexName, state.active_generation);
const result = await this.adapter.bulkIndex(
physical,
docs.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) }))
docs.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) })),
);
return { indexed: docs.length - result.errors.length, errors: result.errors };
}
// ---------------------------------------------------------------------
// search.query — tenant + purpose/rights filtering ENFORCED here
// ---------------------------------------------------------------------
async query(indexName: string, q: SearchQuery): Promise<SearchResult> {
const def = this.getIndexDefinition(indexName);
const def = await this.resolveIndexDefinition(indexName);
if (!q.tenant_id?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "tenant_id is required on every search query");
}
if (!q.purpose?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "purpose is required on every search query (§10)");
}
const size = Math.min(q.size ?? 25, MAX_SEARCH_PAGE_SIZE);
const from = q.from ?? 0;
const requestedSize = q.size ?? 25;
if (!Number.isSafeInteger(from) || from < 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "from must be a non-negative integer");
}
if (!Number.isSafeInteger(requestedSize) || requestedSize < 1) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "size must be a positive integer");
}
const size = Math.min(requestedSize, MAX_SEARCH_PAGE_SIZE);
const filter: unknown[] = [
{ term: { tenant_id: q.tenant_id } },
// Purpose fit: the document must be tagged for THIS query's purpose.
// Docs with empty purpose_tags match nothing — fail closed by shape.
{ term: { purpose_tags: q.purpose } },
{ term: { rights_tags: q.rights_tag ?? "internal" } },
];
@@ -171,10 +195,9 @@ export class SearchService {
const must = q.text?.trim()
? [{ multi_match: { query: q.text, fields: def.default_search_fields } }]
: [{ match_all: {} }];
const res = await this.adapter.search(this.aliasFor(indexName), {
query: { bool: { must, filter } },
from: q.from ?? 0,
from,
size,
});
@@ -196,30 +219,27 @@ export class SearchService {
};
}
// ---------------------------------------------------------------------
// search.rebuild — new generation from canonical truth (§42)
// ---------------------------------------------------------------------
async rebuild(
indexName: string,
source: DocumentSource,
opts?: {
/** Switch the alias to the new generation when done (default true). */
switch?: boolean;
/** Delete superseded generations after a successful switch (default false — keep for rollback). */
prune?: boolean;
batchSize?: number;
tenant_id?: string;
correlation_id?: string;
}
},
): Promise<RebuildResult> {
const def = this.getIndexDefinition(indexName);
const def = await this.resolveIndexDefinition(indexName);
const state = await this.generationState(indexName);
const newGeneration = (state.active_generation ?? 0) + 1;
const physical = this.physicalIndexFor(indexName, newGeneration);
const batchSize = opts?.batchSize ?? 500;
if (!Number.isSafeInteger(batchSize) || batchSize < 1 || batchSize > 5_000) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "batchSize must be between 1 and 5000");
}
await this.adapter.createIndex(physical, this.mappingsFor(def));
const batchSize = opts?.batchSize ?? 500;
let batch: SearchDocument[] = [];
let docsIndexed = 0;
const flush = async () => {
@@ -227,28 +247,33 @@ export class SearchService {
this.validateDocuments(def, batch);
const res = await this.adapter.bulkIndex(
physical,
batch.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) }))
batch.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) })),
);
if (res.errors.length > 0) {
// A rebuild must be all-or-nothing: destroy the failed generation and
// leave the active alias untouched (generation isolation, §42).
await this.adapter.deleteIndex(physical);
throw new SiaxError(
"RETRYABLE_FAILURE",
"0pt.search",
`rebuild of "${indexName}" g${newGeneration} failed on ${res.errors.length} document(s); generation discarded`
`rebuild of "${indexName}" g${newGeneration} failed on ${res.errors.length} document(s); generation discarded`,
);
}
docsIndexed += batch.length;
batch = [];
};
for await (const doc of source.iterate()) {
batch.push(doc);
if (batch.length >= batchSize) await flush();
try {
for await (const doc of source.iterate()) {
batch.push(doc);
if (batch.length >= batchSize) await flush();
}
await flush();
await this.adapter.refresh(physical);
} catch (error) {
// Best-effort cleanup of an incomplete generation. Preserve the original
// failure even if cleanup itself fails.
await this.adapter.deleteIndex(physical).catch(() => undefined);
throw error;
}
await flush();
await this.adapter.refresh(physical);
const shouldSwitch = opts?.switch !== false;
if (shouldSwitch) {
@@ -280,10 +305,11 @@ export class SearchService {
};
}
// ---------------------------------------------------------------------
// search.alias.switch — explicit generation cutover / rollback
// ---------------------------------------------------------------------
async switchGeneration(indexName: string, generation: number): Promise<IndexGenerationState> {
if (!Number.isSafeInteger(generation) || generation < 1) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "generation must be a positive integer");
}
await this.resolveIndexDefinition(indexName);
const physical = this.physicalIndexFor(indexName, generation);
if (!(await this.adapter.indexExists(physical))) {
throw new SiaxError("NOT_FOUND", "0pt.search", `generation g${generation} of "${indexName}" does not exist`);
@@ -308,12 +334,7 @@ export class SearchService {
};
}
// ---------------------------------------------------------------------
// internals
// ---------------------------------------------------------------------
private docKey(d: SearchDocument): string {
// Physical _id is tenant-prefixed so two tenants can use the same doc_id
// without collision — defense in depth under the tenant term filter.
return `${d.tenant_id}:${d.doc_id}`;
}
@@ -330,10 +351,13 @@ export class SearchService {
private validateDocuments(def: SearchIndexDefinition, docs: SearchDocument[]): void {
for (const d of docs) {
if (!d.doc_id?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", "every document requires doc_id");
}
if (!d.tenant_id?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", `document ${d.doc_id} is missing tenant_id`);
}
if (!Array.isArray(d.purpose_tags)) {
if (!Array.isArray(d.purpose_tags) || d.purpose_tags.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.search", `document ${d.doc_id} is missing purpose_tags`);
}
if (!Array.isArray(d.rights_tags) || d.rights_tags.length === 0) {
@@ -344,7 +368,7 @@ export class SearchService {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.search",
`document ${d.doc_id} has field "${field}" not in index definition "${def.index_name}" v${def.version}`
`document ${d.doc_id} has field "${field}" not in index definition "${def.index_name}" v${def.version}`,
);
}
}