feat(agent): telemetry + CI-gate quick wins from the "AI systems that ship" audit (#677)
* feat(agent): telemetry completeness + durability, CI gates, commit_method provenance Quick wins from the "Building AI systems that ship" audit: - mcp.tool_called gains errorMessage (message_sv, truncated 500 chars) on all failure exits; new mcp.skill_loaded event on every gnubok_load_skill (all tiers) so atom usage is finally measurable - event_log: (event_type, created_at) index; cleanup cron keeps mcp.*/agent.* telemetry 180 days (delivery events stay 30) - CI: lint ratchet (npm run check:lint — 60 legacy errors baselined, fails only on NEW errors) and a pg-real coverage gate (migrations touching trigger/RPC/RLS/DEFERRABLE require a *.pg.test.ts change; escape hatch: -- pg-test: covered-by/skip) - journal_entries.commit_method CHECK widened with 'api_key'/'agent'; the MCP approve path records 'api_key' truthfully instead of 'user_accept' (agent_first_vision §8 P0-1). 'agent' is reserved — ALL MCP traffic (incl. claude.ai OAuth, whose access_token is a minted API key) authenticates as api_key today Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(import): derive opening balances from prior-year #UB when SIE lacks #IB (#675) SIE files exported without #IB 0 rows (only #UB -1) previously imported with zero opening balances. getEffectiveOpeningBalances() now derives IB from prior-year UB for balance-sheet accounts when explicit #IB is absent, surfaces the derivation as an info issue in the import preview, and excludes share-capital vouchers from opening-balance detection. Detection regexes are shared between parser and importer so the two checks cannot drift. 507 lib/import tests pass. (Authored in a parallel session in this checkout; included per request.) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(review): address PR #677 bot findings — RoPA entry, execFileSync, gate scope note Triage of the compliance-swarm + Greptile findings: Applied: - .compliance/ropa.yaml: new mcp.telemetry processing activity declaring the 180-day mcp.*/agent.* retention, lawful basis, data categories, and the no-args/no-results minimisation (ISO A.8.10, GDPR Art.5(1)(c) — the retention split is now formally documented, referenced from the cron) - check-pg-test-coverage.mjs: execFileSync with argv array — no shell, so a hostile base-ref can't inject (ASVS V13.2.1); verified an injection attempt exits 2 without executing - check-pg-test-coverage.mjs: documented the PR-level (not per-migration) scope of the gate so reviewers know to check coverage per migration when a PR carries several risky migrations (Greptile P2) Acknowledged, no change: - errorMessage PII risk: messages are domain-mapped strings; event_log already persists far richer delivery payloads under the same RLS; now declared in ropa.yaml - cron error envelope: errorResponse maps to the canonical safe envelope and the endpoint is CRON_SECRET-gated - two-pass delete "partial state": TTL deletes are idempotent — the next daily run sweeps whatever a failed pass left behind - skill_loaded actorLabel/sessionId: mirrors the pre-existing mcp.tool_called payload; sessionId is the join key the analytics exist for Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
076bb169f8
commit
bc61862e76
@@ -1678,6 +1678,12 @@ export const tools: McpTool[] = [
|
||||
const available = all.map((s) => s.slug).join(', ')
|
||||
throw new Error(`Skill not found: "${slug}". Available skills: ${available}`)
|
||||
}
|
||||
// Every load, every tier — records which skill/atom bodies agents
|
||||
// actually pull (mcp.skill_loaded). Without this, "which atom was
|
||||
// loaded" is unanswerable and atom effectiveness can't be measured.
|
||||
if (actor) {
|
||||
emitSkillLoaded({ slug: skill.slug, tier: skill.tier, actor, userId, companyId })
|
||||
}
|
||||
// Workflow-tier skills are the closed-form processes (month-end-close,
|
||||
// year-end-close, payroll-monthly). Loading one is a strong signal the
|
||||
// agent is starting that workflow — emit so we can track completion
|
||||
@@ -7516,7 +7522,7 @@ export const tools: McpTool[] = [
|
||||
// entries, what balances) and a broken/unbalanced file is rejected HERE,
|
||||
// not after they approve a blind byte count. commitImportSie re-parses on
|
||||
// commit (defense-in-depth — the staged string could be tampered).
|
||||
const { parseSIEFile, validateSIEFile } = await import('@/lib/import/sie-parser')
|
||||
const { parseSIEFile, validateSIEFile, getEffectiveOpeningBalances } = await import('@/lib/import/sie-parser')
|
||||
let parsed
|
||||
try {
|
||||
parsed = parseSIEFile(fileContent)
|
||||
@@ -7528,7 +7534,10 @@ export const tools: McpTool[] = [
|
||||
throw new Error(`SIE-filen är ogiltig och importeras inte: ${validation.errors.join('; ')}`)
|
||||
}
|
||||
|
||||
const ibCurrent = parsed.openingBalances.filter((b) => b.yearIndex === 0)
|
||||
// Effective set: explicit #IB 0, or IB derived from #UB -1 when the
|
||||
// source system exports none (issue #675) — so the approver sees the
|
||||
// real IB total and UB-1-only files pass the coverage check below.
|
||||
const ibCurrent = getEffectiveOpeningBalances(parsed).balances
|
||||
const ibTotal = Math.round(ibCurrent.reduce((s, b) => s + b.amount, 0) * 100) / 100
|
||||
|
||||
// Mapping-coverage check. The executor's per-voucher loop silently
|
||||
@@ -8591,12 +8600,34 @@ export const tools: McpTool[] = [
|
||||
log.warn('Failed to resolve user email for MCP approval', { userId, err })
|
||||
}
|
||||
|
||||
// commit_method provenance (agent_first_vision.md §8 P0-1): MCP
|
||||
// approvals are relayed through an agent credential — record that in
|
||||
// the immutable layer instead of claiming 'user_accept'. The positive
|
||||
// acknowledgment (confirmed=true for high risk) is agent-attested, not
|
||||
// a first-party human session; an auditor reading the GL can now tell
|
||||
// the difference (BFNAR 2013:2 kap 8 behandlingshistorik).
|
||||
//
|
||||
// ALL MCP traffic authenticates as an api_key actor — the claude.ai
|
||||
// OAuth connector's access_token is itself a minted gnubok_sk_ key
|
||||
// (app/api/mcp-oauth/token/route.ts), indistinguishable from the
|
||||
// bridge at this layer — so 'api_key' is the truthful value for every
|
||||
// path through this handler. 'agent' (also in the CHECK) is reserved
|
||||
// for first-party agent surfaces (e.g. in-app agent chat) once they
|
||||
// commit through this layer with a distinguishable actor type.
|
||||
//
|
||||
// Note: commitPendingOperation currently threads commitMethod into the
|
||||
// journal only for create_voucher ops (pre-existing); other operation
|
||||
// types keep their per-handler defaults, with this approval's actor
|
||||
// recorded in processing_history below either way.
|
||||
const commitMethod =
|
||||
actor?.type === 'api_key' ? ('api_key' as const) : ('user_accept' as const)
|
||||
|
||||
const result = await commitPendingOperation(
|
||||
supabase,
|
||||
userId,
|
||||
companyId,
|
||||
operation,
|
||||
{ commitMethod: 'user_accept', ...(userEmail ? { userEmail } : {}) }
|
||||
{ commitMethod, ...(userEmail ? { userEmail } : {}) }
|
||||
)
|
||||
|
||||
// Audit the MCP-initiated approval. Failure must not break the user
|
||||
@@ -8613,7 +8644,7 @@ export const tools: McpTool[] = [
|
||||
operation_type: operation.operation_type,
|
||||
risk_level: operation.risk_level,
|
||||
outcome: result.status,
|
||||
commit_method: 'user_accept',
|
||||
commit_method: commitMethod,
|
||||
channel: 'mcp',
|
||||
confirmed: args.confirmed === true,
|
||||
},
|
||||
@@ -8902,6 +8933,7 @@ function emitToolCallTelemetry(payload: {
|
||||
isError: boolean
|
||||
errorCode: string | null
|
||||
errorKind: 'execution' | 'scope_denied' | 'unknown_tool' | null
|
||||
errorMessage: string | null
|
||||
requestId: string | number | null
|
||||
userId: string
|
||||
companyId: string
|
||||
@@ -8920,6 +8952,10 @@ function emitToolCallTelemetry(payload: {
|
||||
isError: payload.isError,
|
||||
errorCode: payload.errorCode,
|
||||
errorKind: payload.errorKind,
|
||||
// Truncated: domain error messages are short, but unknown-tool /
|
||||
// validation messages can embed long lists. 500 chars is plenty for
|
||||
// clustering failures into gotchas without bloating event_log rows.
|
||||
errorMessage: payload.errorMessage ? payload.errorMessage.slice(0, 500) : null,
|
||||
requestId: payload.requestId,
|
||||
userId: payload.userId,
|
||||
companyId: payload.companyId,
|
||||
@@ -9059,6 +9095,36 @@ function checkAndEmitNextHintFollowed(
|
||||
.catch((err) => console.error('[mcp] next_hint_followed emit failed:', err))
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire-and-forget telemetry for every successful gnubok_load_skill, all tiers.
|
||||
* Unlike mcp.workflow_started (workflow tier only), this records WHICH skill
|
||||
* or atom body the agent pulled — the denominator for correlating a loaded
|
||||
* atom with downstream tool-error rates.
|
||||
*/
|
||||
function emitSkillLoaded(payload: {
|
||||
slug: string
|
||||
tier: 'workflow' | 'horizontal' | 'vertical' | 'modifier'
|
||||
actor: ActorContext
|
||||
userId: string
|
||||
companyId: string
|
||||
}): void {
|
||||
void eventBus
|
||||
.emit({
|
||||
type: 'mcp.skill_loaded',
|
||||
payload: {
|
||||
slug: payload.slug,
|
||||
tier: payload.tier,
|
||||
sessionId: payload.actor.sessionId ?? null,
|
||||
actorType: payload.actor.type,
|
||||
actorId: payload.actor.id ?? null,
|
||||
actorLabel: payload.actor.label ?? null,
|
||||
userId: payload.userId,
|
||||
companyId: payload.companyId,
|
||||
},
|
||||
})
|
||||
.catch((err) => console.error('[mcp] skill_loaded emit failed:', err))
|
||||
}
|
||||
|
||||
/** Fire-and-forget telemetry for workflow lifecycle. */
|
||||
function emitWorkflowStarted(payload: {
|
||||
slug: string
|
||||
@@ -9259,6 +9325,10 @@ export async function handleMcpRequest(request: Request): Promise<Response> {
|
||||
isError: true,
|
||||
errorCode: 'UNKNOWN_TOOL',
|
||||
errorKind: 'unknown_tool',
|
||||
// Just the requested name — the full available-tools list returned
|
||||
// to the client would blow the truncation budget without adding
|
||||
// analytical signal.
|
||||
errorMessage: `Unknown tool: "${toolName}"`,
|
||||
requestId: id ?? null,
|
||||
userId,
|
||||
companyId,
|
||||
@@ -9285,6 +9355,7 @@ export async function handleMcpRequest(request: Request): Promise<Response> {
|
||||
isError: true,
|
||||
errorCode: scopeError.error.code,
|
||||
errorKind: 'scope_denied',
|
||||
errorMessage: scopeError.error.message_sv,
|
||||
requestId: id ?? null,
|
||||
userId,
|
||||
companyId,
|
||||
@@ -9347,6 +9418,7 @@ export async function handleMcpRequest(request: Request): Promise<Response> {
|
||||
isError: false,
|
||||
errorCode: null,
|
||||
errorKind: null,
|
||||
errorMessage: null,
|
||||
requestId: id ?? null,
|
||||
userId,
|
||||
companyId,
|
||||
@@ -9364,6 +9436,10 @@ export async function handleMcpRequest(request: Request): Promise<Response> {
|
||||
isError: true,
|
||||
errorCode: structured.error.code,
|
||||
errorKind: 'execution',
|
||||
// message_sv is the canonical domain message ("Verifikationen
|
||||
// balanserar inte", "Perioden är låst", …) — the text worth
|
||||
// clustering when mining failures for gotchas.
|
||||
errorMessage: structured.error.message_sv,
|
||||
requestId: id ?? null,
|
||||
userId,
|
||||
companyId,
|
||||
|
||||
Reference in New Issue
Block a user