refactor: make search definitions restart-safe
This commit is contained in:
+108
-84
@@ -1,35 +1,15 @@
|
||||
/**
|
||||
* src/search/search-service.ts
|
||||
* =============================
|
||||
* 0pt's Search capability (Master-DoD §10 Search):
|
||||
* 0pt Search capability.
|
||||
*
|
||||
* search.index -> registerIndexDefinition + indexDocuments (bulk)
|
||||
* search.query -> query (tenant + purpose/rights filtering ENFORCED)
|
||||
* search.rebuild -> rebuild (new generation, bulk load, validate)
|
||||
* search.alias.switch -> switchGeneration (atomic alias cutover)
|
||||
*
|
||||
* Non-negotiables encoded here:
|
||||
* - OpenSearch is NEVER canonical truth (§42): a rebuild takes a caller-
|
||||
* supplied `DocumentSource` (an async iterator over canonical data) and
|
||||
* can rebuild any index from an empty cluster. Deleting every index
|
||||
* loses nothing canonical.
|
||||
* - Tenant filter is applied by THIS service on every query — a caller
|
||||
* cannot opt out; a missing tenant_id is INVALID_REQUEST (§31).
|
||||
* - Purpose/rights filtering (§10): every document carries purpose_tags +
|
||||
* rights_tags (mandatory on the envelope); every query names its purpose
|
||||
* and gets only documents tagged for it. Documents with an EMPTY
|
||||
* purpose_tags list are never returnable — fail closed.
|
||||
* - Generation switch: physical indices are `opt-{name}--g{n}`; the alias
|
||||
* `opt-{name}` points at exactly one generation; cutover is atomic
|
||||
* (§42 generation isolation + cutover).
|
||||
*
|
||||
* Lineage: every completed rebuild emits an OpenLineage COMPLETE run to AUD0
|
||||
* (inputs: the document source name; outputs: the physical index).
|
||||
* OpenSearch remains a disposable projection. Logical index definitions may be
|
||||
* persisted through SearchDefinitionRepository so a process restart never
|
||||
* forgets how to query or rebuild an otherwise valid projection.
|
||||
*/
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import type { OpenSearchAdapter } from "./opensearch-adapter";
|
||||
import { LineageEmitter } from "../lineage/lineage-emitter";
|
||||
import type { Aud0Client } from "../clients/aud0-client";
|
||||
import { LineageEmitter } from "../lineage/lineage-emitter";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import type { SearchDefinitionRepository } from "../repositories/search-definition-repository";
|
||||
import type { OpenSearchAdapter } from "./opensearch-adapter";
|
||||
import {
|
||||
MAX_SEARCH_PAGE_SIZE,
|
||||
type BulkIndexResult,
|
||||
@@ -41,9 +21,7 @@ import {
|
||||
type SearchResult,
|
||||
} from "./search-types";
|
||||
|
||||
/** An async source of canonical documents for (re)builds. */
|
||||
export interface DocumentSource {
|
||||
/** Name for lineage, e.g. "postgres:opt_datasets" or "0pt:dataset:ds_x". */
|
||||
name: string;
|
||||
iterate(): AsyncIterable<SearchDocument>;
|
||||
}
|
||||
@@ -59,24 +37,35 @@ const ENVELOPE_MAPPINGS: Record<string, unknown> = {
|
||||
export class SearchService {
|
||||
private readonly definitions = new Map<string, SearchIndexDefinition>();
|
||||
private readonly lineage?: LineageEmitter;
|
||||
private readonly definitionRepository?: SearchDefinitionRepository;
|
||||
|
||||
constructor(
|
||||
private readonly adapter: OpenSearchAdapter,
|
||||
opts?: { aud0Client?: Aud0Client }
|
||||
opts?: { aud0Client?: Aud0Client; definitionRepository?: SearchDefinitionRepository },
|
||||
) {
|
||||
this.lineage = opts?.aud0Client ? new LineageEmitter(opts.aud0Client) : undefined;
|
||||
this.definitionRepository = opts?.definitionRepository;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Index definitions
|
||||
// ---------------------------------------------------------------------
|
||||
registerIndexDefinition(def: SearchIndexDefinition): SearchIndexDefinition {
|
||||
private validateDefinition(def: SearchIndexDefinition): void {
|
||||
if (!def.index_name?.trim() || !/^[a-z0-9][a-z0-9_-]{0,62}$/i.test(def.index_name)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "index_name must be 1-63 letters, digits, _ or -");
|
||||
}
|
||||
if (!Number.isSafeInteger(def.version) || def.version < 1) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "definition version must be a positive integer");
|
||||
}
|
||||
if (Object.keys(def.fields).length === 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "an index definition needs at least one field");
|
||||
}
|
||||
for (const f of def.default_search_fields) {
|
||||
if (def.fields[f] !== "text") {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.search",
|
||||
`default_search_field "${f}" must be a text field in index "${def.index_name}"`
|
||||
`default_search_field "${f}" must be a text field in index "${def.index_name}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -85,22 +74,59 @@ export class SearchService {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.search",
|
||||
`field "${reserved}" is reserved by the 0pt search envelope`
|
||||
`field "${reserved}" is reserved by the 0pt search envelope`,
|
||||
);
|
||||
}
|
||||
}
|
||||
this.definitions.set(def.index_name, def);
|
||||
return def;
|
||||
}
|
||||
|
||||
/** In-memory registration retained for hermetic unit tests and local callers. */
|
||||
registerIndexDefinition(def: SearchIndexDefinition): SearchIndexDefinition {
|
||||
this.validateDefinition(def);
|
||||
const copy = {
|
||||
...def,
|
||||
fields: { ...def.fields },
|
||||
default_search_fields: [...def.default_search_fields],
|
||||
};
|
||||
this.definitions.set(def.index_name, copy);
|
||||
return copy;
|
||||
}
|
||||
|
||||
/** Production registration: versioned durable write followed by cache refresh. */
|
||||
async registerIndexDefinitionDurable(def: SearchIndexDefinition, actorId?: string): Promise<SearchIndexDefinition> {
|
||||
this.validateDefinition(def);
|
||||
if (!this.definitionRepository) return this.registerIndexDefinition(def);
|
||||
const stored = await this.definitionRepository.save(def, actorId);
|
||||
return this.registerIndexDefinition(stored);
|
||||
}
|
||||
|
||||
getIndexDefinition(indexName: string): SearchIndexDefinition {
|
||||
const def = this.definitions.get(indexName);
|
||||
if (!def) throw new SiaxError("NOT_FOUND", "0pt.search", `no index definition "${indexName}"`);
|
||||
return def;
|
||||
return { ...def, fields: { ...def.fields }, default_search_fields: [...def.default_search_fields] };
|
||||
}
|
||||
|
||||
async resolveIndexDefinition(indexName: string): Promise<SearchIndexDefinition> {
|
||||
const cached = this.definitions.get(indexName);
|
||||
if (cached) return this.getIndexDefinition(indexName);
|
||||
const stored = await this.definitionRepository?.latest(indexName);
|
||||
if (!stored) throw new SiaxError("NOT_FOUND", "0pt.search", `no index definition "${indexName}"`);
|
||||
return this.registerIndexDefinition(stored);
|
||||
}
|
||||
|
||||
listIndexDefinitions(): SearchIndexDefinition[] {
|
||||
return [...this.definitions.values()];
|
||||
return [...this.definitions.values()].map((def) => ({
|
||||
...def,
|
||||
fields: { ...def.fields },
|
||||
default_search_fields: [...def.default_search_fields],
|
||||
}));
|
||||
}
|
||||
|
||||
async listIndexDefinitionsDurable(): Promise<SearchIndexDefinition[]> {
|
||||
if (!this.definitionRepository) return this.listIndexDefinitions();
|
||||
const definitions = await this.definitionRepository.listLatest();
|
||||
for (const def of definitions) this.registerIndexDefinition(def);
|
||||
return definitions;
|
||||
}
|
||||
|
||||
aliasFor(indexName: string): string {
|
||||
@@ -113,51 +139,49 @@ export class SearchService {
|
||||
|
||||
private mappingsFor(def: SearchIndexDefinition): Record<string, unknown> {
|
||||
const properties: Record<string, unknown> = { ...ENVELOPE_MAPPINGS };
|
||||
for (const [field, type] of Object.entries(def.fields)) {
|
||||
properties[field] = { type };
|
||||
}
|
||||
for (const [field, type] of Object.entries(def.fields)) properties[field] = { type };
|
||||
return { properties, _meta: { opt_definition_version: def.version } };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// search.index — bulk write into the ACTIVE generation
|
||||
// ---------------------------------------------------------------------
|
||||
async indexDocuments(indexName: string, docs: SearchDocument[]): Promise<BulkIndexResult> {
|
||||
const def = this.getIndexDefinition(indexName);
|
||||
const def = await this.resolveIndexDefinition(indexName);
|
||||
const state = await this.generationState(indexName);
|
||||
if (state.active_generation === undefined) {
|
||||
throw new SiaxError(
|
||||
"CONFLICT",
|
||||
"0pt.search",
|
||||
`index "${indexName}" has no active generation — run a rebuild first`
|
||||
`index "${indexName}" has no active generation — run a rebuild first`,
|
||||
);
|
||||
}
|
||||
this.validateDocuments(def, docs);
|
||||
const physical = this.physicalIndexFor(indexName, state.active_generation);
|
||||
const result = await this.adapter.bulkIndex(
|
||||
physical,
|
||||
docs.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) }))
|
||||
docs.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) })),
|
||||
);
|
||||
return { indexed: docs.length - result.errors.length, errors: result.errors };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// search.query — tenant + purpose/rights filtering ENFORCED here
|
||||
// ---------------------------------------------------------------------
|
||||
async query(indexName: string, q: SearchQuery): Promise<SearchResult> {
|
||||
const def = this.getIndexDefinition(indexName);
|
||||
const def = await this.resolveIndexDefinition(indexName);
|
||||
if (!q.tenant_id?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "tenant_id is required on every search query");
|
||||
}
|
||||
if (!q.purpose?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "purpose is required on every search query (§10)");
|
||||
}
|
||||
const size = Math.min(q.size ?? 25, MAX_SEARCH_PAGE_SIZE);
|
||||
const from = q.from ?? 0;
|
||||
const requestedSize = q.size ?? 25;
|
||||
if (!Number.isSafeInteger(from) || from < 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "from must be a non-negative integer");
|
||||
}
|
||||
if (!Number.isSafeInteger(requestedSize) || requestedSize < 1) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "size must be a positive integer");
|
||||
}
|
||||
const size = Math.min(requestedSize, MAX_SEARCH_PAGE_SIZE);
|
||||
|
||||
const filter: unknown[] = [
|
||||
{ term: { tenant_id: q.tenant_id } },
|
||||
// Purpose fit: the document must be tagged for THIS query's purpose.
|
||||
// Docs with empty purpose_tags match nothing — fail closed by shape.
|
||||
{ term: { purpose_tags: q.purpose } },
|
||||
{ term: { rights_tags: q.rights_tag ?? "internal" } },
|
||||
];
|
||||
@@ -171,10 +195,9 @@ export class SearchService {
|
||||
const must = q.text?.trim()
|
||||
? [{ multi_match: { query: q.text, fields: def.default_search_fields } }]
|
||||
: [{ match_all: {} }];
|
||||
|
||||
const res = await this.adapter.search(this.aliasFor(indexName), {
|
||||
query: { bool: { must, filter } },
|
||||
from: q.from ?? 0,
|
||||
from,
|
||||
size,
|
||||
});
|
||||
|
||||
@@ -196,30 +219,27 @@ export class SearchService {
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// search.rebuild — new generation from canonical truth (§42)
|
||||
// ---------------------------------------------------------------------
|
||||
async rebuild(
|
||||
indexName: string,
|
||||
source: DocumentSource,
|
||||
opts?: {
|
||||
/** Switch the alias to the new generation when done (default true). */
|
||||
switch?: boolean;
|
||||
/** Delete superseded generations after a successful switch (default false — keep for rollback). */
|
||||
prune?: boolean;
|
||||
batchSize?: number;
|
||||
tenant_id?: string;
|
||||
correlation_id?: string;
|
||||
}
|
||||
},
|
||||
): Promise<RebuildResult> {
|
||||
const def = this.getIndexDefinition(indexName);
|
||||
const def = await this.resolveIndexDefinition(indexName);
|
||||
const state = await this.generationState(indexName);
|
||||
const newGeneration = (state.active_generation ?? 0) + 1;
|
||||
const physical = this.physicalIndexFor(indexName, newGeneration);
|
||||
const batchSize = opts?.batchSize ?? 500;
|
||||
if (!Number.isSafeInteger(batchSize) || batchSize < 1 || batchSize > 5_000) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "batchSize must be between 1 and 5000");
|
||||
}
|
||||
|
||||
await this.adapter.createIndex(physical, this.mappingsFor(def));
|
||||
|
||||
const batchSize = opts?.batchSize ?? 500;
|
||||
let batch: SearchDocument[] = [];
|
||||
let docsIndexed = 0;
|
||||
const flush = async () => {
|
||||
@@ -227,28 +247,33 @@ export class SearchService {
|
||||
this.validateDocuments(def, batch);
|
||||
const res = await this.adapter.bulkIndex(
|
||||
physical,
|
||||
batch.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) }))
|
||||
batch.map((d) => ({ id: this.docKey(d), body: this.toIndexBody(d) })),
|
||||
);
|
||||
if (res.errors.length > 0) {
|
||||
// A rebuild must be all-or-nothing: destroy the failed generation and
|
||||
// leave the active alias untouched (generation isolation, §42).
|
||||
await this.adapter.deleteIndex(physical);
|
||||
throw new SiaxError(
|
||||
"RETRYABLE_FAILURE",
|
||||
"0pt.search",
|
||||
`rebuild of "${indexName}" g${newGeneration} failed on ${res.errors.length} document(s); generation discarded`
|
||||
`rebuild of "${indexName}" g${newGeneration} failed on ${res.errors.length} document(s); generation discarded`,
|
||||
);
|
||||
}
|
||||
docsIndexed += batch.length;
|
||||
batch = [];
|
||||
};
|
||||
|
||||
for await (const doc of source.iterate()) {
|
||||
batch.push(doc);
|
||||
if (batch.length >= batchSize) await flush();
|
||||
try {
|
||||
for await (const doc of source.iterate()) {
|
||||
batch.push(doc);
|
||||
if (batch.length >= batchSize) await flush();
|
||||
}
|
||||
await flush();
|
||||
await this.adapter.refresh(physical);
|
||||
} catch (error) {
|
||||
// Best-effort cleanup of an incomplete generation. Preserve the original
|
||||
// failure even if cleanup itself fails.
|
||||
await this.adapter.deleteIndex(physical).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
await flush();
|
||||
await this.adapter.refresh(physical);
|
||||
|
||||
const shouldSwitch = opts?.switch !== false;
|
||||
if (shouldSwitch) {
|
||||
@@ -280,10 +305,11 @@ export class SearchService {
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// search.alias.switch — explicit generation cutover / rollback
|
||||
// ---------------------------------------------------------------------
|
||||
async switchGeneration(indexName: string, generation: number): Promise<IndexGenerationState> {
|
||||
if (!Number.isSafeInteger(generation) || generation < 1) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "generation must be a positive integer");
|
||||
}
|
||||
await this.resolveIndexDefinition(indexName);
|
||||
const physical = this.physicalIndexFor(indexName, generation);
|
||||
if (!(await this.adapter.indexExists(physical))) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.search", `generation g${generation} of "${indexName}" does not exist`);
|
||||
@@ -308,12 +334,7 @@ export class SearchService {
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// internals
|
||||
// ---------------------------------------------------------------------
|
||||
private docKey(d: SearchDocument): string {
|
||||
// Physical _id is tenant-prefixed so two tenants can use the same doc_id
|
||||
// without collision — defense in depth under the tenant term filter.
|
||||
return `${d.tenant_id}:${d.doc_id}`;
|
||||
}
|
||||
|
||||
@@ -330,10 +351,13 @@ export class SearchService {
|
||||
|
||||
private validateDocuments(def: SearchIndexDefinition, docs: SearchDocument[]): void {
|
||||
for (const d of docs) {
|
||||
if (!d.doc_id?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", "every document requires doc_id");
|
||||
}
|
||||
if (!d.tenant_id?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", `document ${d.doc_id} is missing tenant_id`);
|
||||
}
|
||||
if (!Array.isArray(d.purpose_tags)) {
|
||||
if (!Array.isArray(d.purpose_tags) || d.purpose_tags.length === 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.search", `document ${d.doc_id} is missing purpose_tags`);
|
||||
}
|
||||
if (!Array.isArray(d.rights_tags) || d.rights_tags.length === 0) {
|
||||
@@ -344,7 +368,7 @@ export class SearchService {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.search",
|
||||
`document ${d.doc_id} has field "${field}" not in index definition "${def.index_name}" v${def.version}`
|
||||
`document ${d.doc_id} has field "${field}" not in index definition "${def.index_name}" v${def.version}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user