0pt/tests/p0/source-registry-postgres.realinfra.test.ts

/**
 * REAL Postgres integration test (Master-DoD §48/§50) for the Source
 * Registry's new persistence layer (src/repositories/*,
 * sql/002_source_registry.sql) -- the P010 task's central deliverable: this
 * is what proves the Map -> Postgres migration actually works, not just
 * that it typechecks.
 *
 * Same honest-degrade convention as tests/p0/iceberg-write-service.realinfra.test.ts
 * and tests/integration/analytics-clickhouse.integration.test.ts: gated on
 * an env var (SOURCE_REGISTRY_TEST_DATABASE_URL), SKIPS HONESTLY when unset,
 * never fabricates success. Point it at a disposable Postgres, e.g.:
 *
 *   docker run -d --name opt-source-registry-pgtest \
 *     -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \
 *     -p 15433:5432 postgres:16-alpine
 *   SOURCE_REGISTRY_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15433/opt_test \
 *     pnpm vitest run tests/p0/source-registry-postgres.realinfra.test.ts
 *
 * This is NOT a mock-repository test -- every assertion below is driven
 * through SourceRegistryService with `databaseUrl` set, i.e. the real
 * PostgresSourceRepository/PostgresSourceRegistryRepository HTTP-free SQL
 * path (postgres.js against the live container), with a handful of
 * assertions cross-checked with direct SQL against the same database to
 * prove the rows are really there, not an artifact of the service's own
 * in-process bookkeeping.
 *
 * beforeAll applies every sql/NNN_*.sql file in order (same restore-drill
 * pattern as tests/analytics/analytics-domain-postgres.realinfra.test.ts),
 * not just sql/002_source_registry.sql -- this test does not require the
 * operator to have pre-run migrations.
 *
 * CORRECTED (this pass, 2026-09-14): it used to apply ONLY
 * sql/002_source_registry.sql. That was enough the day this file was
 * written, but sql/007_ingestion_plane.sql later added `opt_sources.owner_id`
 * (`ALTER TABLE opt_sources ADD COLUMN IF NOT EXISTS owner_id TEXT`), and
 * src/repositories/postgres-source-repository.ts's INSERT has named that
 * column ever since. Because this env var is never set in CI (confirmed by
 * grep over .gitea/workflows/ci.yml — no job references
 * SOURCE_REGISTRY_TEST_DATABASE_URL), the drift was never caught: every
 * assertion below failed with `column "owner_id" of relation "opt_sources"
 * does not exist` the moment it was actually run against a real, freshly
 * migrated-to-002-only Postgres. The fix is to stop hand-picking one
 * migration file and replay the whole chain, exactly like the analytics
 * suite already does.
 */
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import fs from "node:fs";
import path from "node:path";
import postgres from "postgres";
import { SourceRegistryService } from "@/src/services/source-registry-service";
import { closeRepositorySqlForTests } from "@/src/repositories/db";

const TEST_DATABASE_URL = process.env.SOURCE_REGISTRY_TEST_DATABASE_URL;
const run = TEST_DATABASE_URL ? describe : describe.skip;

let adminSql: ReturnType<typeof postgres> | undefined;
let reachable = false;
let unreachableReason = "";

beforeAll(async () => {
  if (!TEST_DATABASE_URL) {
    console.warn(
      "[realinfra] SOURCE_REGISTRY_TEST_DATABASE_URL not set this run -- skipping honestly, not fabricating success.",
    );
    return;
  }
  try {
    adminSql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 2 });
    await Promise.race([
      adminSql`SELECT 1`,
      new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)),
    ]);
    // Replay every migration in order, not just 002 -- see the file header
    // "CORRECTED" note. Tolerate an individual file failing (mirrors
    // analytics-domain-postgres.realinfra.test.ts: sql/001 needs the
    // `vector` extension, which a stock postgres:16-alpine does not carry)
    // as long as the files this test actually depends on (002, 007) apply.
    const dir = path.join(process.cwd(), "sql");
    for (const file of fs
      .readdirSync(dir)
      .filter((f) => f.endsWith(".sql"))
      .sort()) {
      const body = fs.readFileSync(path.join(dir, file), "utf8");
      try {
        await adminSql.unsafe(body);
      } catch (err) {
        console.warn(`[realinfra] ${file} did not apply cleanly: ${String(err).slice(0, 200)}`);
      }
    }
    reachable = true;
  } catch (err) {
    reachable = false;
    unreachableReason = err instanceof Error ? err.message : String(err);
  }
}, 30000);

afterAll(async () => {
  await adminSql?.end({ timeout: 5 });
  await closeRepositorySqlForTests();
});

function svc(): SourceRegistryService {
  return new SourceRegistryService({ databaseUrl: TEST_DATABASE_URL });
}

run("SourceRegistryService against a LIVE Postgres (sql/002_source_registry.sql)", () => {
  it("persists across independent service instances -- the exact bug this migration fixes (in-memory Maps were per-instance, not even per-process)", async (ctx) => {
    if (!reachable) {
      console.warn(
        `[realinfra] Postgres not reachable this run (${unreachableReason}) -- skipping honestly.`,
      );
      ctx.skip();
      return;
    }
    const stamp = Date.now();
    const sourceId = `src_realpg_${stamp}`;
    const tenantId = `t_realpg_${stamp}`;

    const writer = svc();
    await writer.registerSource({
      source_id: sourceId,
      tenant_id: tenantId,
      project_id: "p_1",
      source_type: "api",
      name: "Live Postgres pilot source",
      access: "api",
      created_by: "realinfra-test",
      rights: {
        license_id: "official-public-data",
        allowed_purposes: ["company_enrichment"],
        pii_allowed: false,
        redistribution: "customer_deliverable",
        verified_at: "2026-09-01",
        verified_by: "realinfra-test",
      },
      provenance: {
        method: "official_api",
        authority: "Bolagsverket",
        cadence: "daily",
        freshness_sla_hours: 24,
      },
    });

    // A BRAND NEW SourceRegistryService, own SourceService/repository
    // construction under the hood, same DATABASE_URL -- pre-migration this
    // would see NOTHING (each `new SourceService()` owned a private Map).
    const reader = svc();
    const descriptor = await reader.describeSource(sourceId, tenantId);
    expect(descriptor.source.source_id).toBe(sourceId);
    expect(descriptor.source.tenant_id).toBe(tenantId);
    expect(descriptor.rights?.license_id).toBe("official-public-data");
    expect(descriptor.rights?.allowed_purposes).toContain("company_enrichment");
    expect(descriptor.provenance?.method).toBe("official_api");
    expect(descriptor.provenance?.authority).toBe("Bolagsverket");

    // Cross-checked with direct SQL -- real rows, not in-process bookkeeping.
    const sourceRows = await adminSql!<{ source_id: string; tenant_id: string }[]>`
      SELECT source_id, tenant_id FROM opt_sources WHERE source_id = ${sourceId}`;
    expect(sourceRows).toHaveLength(1);
    expect(sourceRows[0].tenant_id).toBe(tenantId);

    const rightsRows = await adminSql!<{ license_id: string; allowed_purposes: string[] }[]>`
      SELECT license_id, allowed_purposes FROM opt_source_rights WHERE source_id = ${sourceId}`;
    expect(rightsRows).toHaveLength(1);
    expect(rightsRows[0].license_id).toBe("official-public-data");
    expect(rightsRows[0].allowed_purposes).toEqual(["company_enrichment"]);
  });

  it("registering the same source_id twice -> 'already exists', enforced by real Postgres (not app-side memory)", async (ctx) => {
    if (!reachable) return ctx.skip();
    const stamp = Date.now();
    const sourceId = `src_realpg_dup_${stamp}`;
    const tenantId = `t_realpg_dup_${stamp}`;
    await svc().registerSource({
      source_id: sourceId,
      tenant_id: tenantId,
      project_id: "p_1",
      source_type: "web",
      name: "First",
    });
    await expect(
      svc().registerSource({
        source_id: sourceId,
        tenant_id: tenantId,
        project_id: "p_1",
        source_type: "web",
        name: "Second",
      }),
    ).rejects.toThrow(/already exists/i);
  });

  it("cross-tenant describe -> NOT_FOUND against real Postgres (§50, no existence oracle)", async (ctx) => {
    if (!reachable) return ctx.skip();
    const stamp = Date.now();
    const sourceId = `src_realpg_tenant_${stamp}`;
    await svc().registerSource({
      source_id: sourceId,
      tenant_id: `t_a_realpg_${stamp}`,
      project_id: "p_1",
      source_type: "web",
      name: "Tenant A source",
    });
    await expect(svc().describeSource(sourceId, `t_b_realpg_${stamp}`)).rejects.toMatchObject({
      code: "NOT_FOUND",
    });
    // And a foreign-tenant query never sees it either.
    const foreign = await svc().querySources({ tenant_id: `t_b_realpg_${stamp}` });
    expect(foreign.find((s) => s.source_id === sourceId)).toBeUndefined();
  });

  it("purpose gate fail-closed against real Postgres: no rights row denies every purpose", async (ctx) => {
    if (!reachable) return ctx.skip();
    const stamp = Date.now();
    const sourceId = `src_realpg_norights_${stamp}`;
    const tenantId = `t_realpg_${stamp}`;
    await svc().registerSource({
      source_id: sourceId,
      tenant_id: tenantId,
      project_id: "p_1",
      source_type: "web",
      name: "No rights yet",
    });
    await expect(svc().assertPurposeAllowed(sourceId, tenantId, "anything")).rejects.toMatchObject({
      code: "AUTHORIZATION_DENIED",
    });
  });

  it("health checks: bounded to the latest 100 rows in Postgres, consecutive-failure count survives across service instances", async (ctx) => {
    if (!reachable) return ctx.skip();
    const stamp = Date.now();
    const sourceId = `src_realpg_health_${stamp}`;
    const tenantId = `t_realpg_${stamp}`;
    await svc().registerSource({
      source_id: sourceId,
      tenant_id: tenantId,
      project_id: "p_1",
      source_type: "api",
      name: "Health test source",
    });

    // 103 "down" checks (oldest -> newest), each via a FRESH service
    // instance, simulating separate route-handler invocations.
    for (let i = 0; i < 103; i++) {
      await svc().recordHealthCheck({
        source_id: sourceId,
        tenant_id: tenantId,
        status: "down",
        checked_at: new Date(Date.now() - (300 - i) * 1000).toISOString(),
      });
    }
    // Then 2 "up" checks, strictly more recent than every "down" above.
    for (let i = 0; i < 2; i++) {
      await svc().recordHealthCheck({
        source_id: sourceId,
        tenant_id: tenantId,
        status: "up",
        checked_at: new Date(Date.now() - (2 - i) * 500).toISOString(),
        latency_ms: 42,
      });
    }

    // Bounded storage: never more than 100 rows for this source_id in Postgres.
    const countRows = await adminSql!<{ n: number }[]>`
      SELECT count(*)::int AS n FROM opt_source_health_checks WHERE source_id = ${sourceId}`;
    expect(countRows[0].n).toBeLessThanOrEqual(100);
    // All 103 "down" writes plus 2 "up" writes were made; some of the
    // oldest "down" rows must have been pruned.
    expect(countRows[0].n).toBeLessThan(105);

    const health = await svc().getHealth(sourceId, tenantId);
    expect(health.status).toBe("up");
    expect(health.consecutive_failures).toBe(0);

    const freshness = await svc().getFreshness(sourceId, tenantId);
    expect(freshness.fresh).toBe(true);
    expect(freshness.last_success_at).toBeDefined();
  });

  it("source.query filters by purpose and redistribution against real rows in opt_source_rights", async (ctx) => {
    if (!reachable) return ctx.skip();
    const stamp = Date.now();
    const tenantId = `t_realpg_query_${stamp}`;
    const sourceA = `src_realpg_a_${stamp}`;
    const sourceB = `src_realpg_b_${stamp}`;
    await svc().registerSource({
      source_id: sourceA,
      tenant_id: tenantId,
      project_id: "p_1",
      source_type: "api",
      name: "A",
      rights: {
        license_id: "l1",
        allowed_purposes: ["company_enrichment"],
        pii_allowed: false,
        redistribution: "customer_deliverable",
        verified_at: "2026-09-01",
        verified_by: "t",
      },
    });
    await svc().registerSource({
      source_id: sourceB,
      tenant_id: tenantId,
      project_id: "p_1",
      source_type: "web",
      name: "B",
      rights: {
        license_id: "l2",
        allowed_purposes: ["compliance"],
        pii_allowed: false,
        redistribution: "internal",
        verified_at: "2026-09-01",
        verified_by: "t",
      },
    });

    const enrichment = await svc().querySources({ tenant_id: tenantId, purpose: "company_enrichment" });
    expect(enrichment.map((s) => s.source_id)).toEqual([sourceA]);

    const internalOnly = await svc().querySources({ tenant_id: tenantId, redistribution: "internal" });
    expect(internalOnly.map((s) => s.source_id)).toEqual([sourceB]);
  });

  it("setRights/setProvenance upsert in place (matching the Map.set replace semantics they replace)", async (ctx) => {
    if (!reachable) return ctx.skip();
    const stamp = Date.now();
    const sourceId = `src_realpg_upsert_${stamp}`;
    const tenantId = `t_realpg_${stamp}`;
    const registry = svc();
    await registry.registerSource({
      source_id: sourceId,
      tenant_id: tenantId,
      project_id: "p_1",
      source_type: "api",
      name: "Upsert test",
    });

    await registry.setRights({
      source_id: sourceId,
      tenant_id: tenantId,
      license_id: "v1",
      allowed_purposes: ["compliance"],
      pii_allowed: false,
      redistribution: "internal",
      verified_at: "2026-09-01",
      verified_by: "t",
    });
    await registry.setRights({
      source_id: sourceId,
      tenant_id: tenantId,
      license_id: "v2",
      allowed_purposes: ["compliance", "company_enrichment"],
      pii_allowed: false,
      redistribution: "customer_deliverable",
      verified_at: "2026-09-02",
      verified_by: "t2",
    });

    const rows = await adminSql!<
      { license_id: string }[]
    >`SELECT license_id FROM opt_source_rights WHERE source_id = ${sourceId}`;
    expect(rows).toHaveLength(1); // one row, updated in place -- not two
    expect(rows[0].license_id).toBe("v2");

    const rights = await registry.getRights(sourceId, tenantId);
    expect(rights?.license_id).toBe("v2");
    expect(rights?.allowed_purposes).toEqual(["compliance", "company_enrichment"]);
  });
});