From 0eb9429932e6498933b7aa80f5822513b393ed4d Mon Sep 17 00:00:00 2001 From: Simon Axelsson Date: Mon, 7 Sep 2026 19:35:29 +0200 Subject: [PATCH] =?UTF-8?q?ci:=20beh=C3=A5ller=20main:s=20workflows=20(est?= =?UTF-8?q?ate-skyddade=20filer)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI-härdningen ur PR #10 (Trivy HIGH/CRITICAL blocking, Postgres-service i integration, python-engine pytest-gate) berör .gitea/workflows/ och kräver estate-policy-beslut — dokumenterad separat, hålls UTANFÖR denna merge så koden kanoniseras utan blockering. Härdningens nya tester är exekverade lokalt: real-Postgres 5/5 (DATABASE_URL-aktiverad outbox). --- .gitea/workflows/ci.yml | 112 +++++++++++++++++----------------- .gitea/workflows/security.yml | 18 ++++-- 2 files changed, 68 insertions(+), 62 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 1471e7c..86fd116 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -1,7 +1,16 @@ name: CI (SIAX Cloud) # Self-hosted CI on SIAX Cloud (Gitea Actions, runners siax-runner-1/2). -# Blocking quality/security + real-dependency gates for the 0pt capability layer. +# Quality gate for the Next.js 16 web console + the 0pt capability layer. +# +# * typecheck (BLOCKING) -> `tsc -p tsconfig.check.json` over src/ + tests/. +# * test (BLOCKING) -> `vitest run`. +# * lint (ADVISORY) -> `eslint src tests` (legacy v0-generated noise +# is inherited debt; continue-on-error so it never blocks). +# * build (BLOCKING) -> `next build`. +# +# Nightly re-runs + Trivy security scanning live in nightly.yml and +# security.yml (this file is push/PR-only). on: push: branches: ["**"] @@ -26,12 +35,27 @@ jobs: run: pnpm run typecheck - name: Test (vitest -- blocking) run: pnpm run test - - name: Lint (eslint -- advisory until legacy findings are separately burned down) + - name: Lint (eslint -- advisory; legacy tree has pre-existing noise) continue-on-error: true run: pnpm run lint - name: Build (next build -- blocking quality gate) run: pnpm run build + # REAL-DEPENDENCY integration tests (Master-DoD §48: "integration tests ... + # Mocks alone count as insufficient"; §10 Search/Analytics). Runs + # tests/integration/* against a LIVE OpenSearch + LIVE ClickHouse started as + # service containers. + # + # SERVICE-CONTAINER DISCIPLINE is copied from sax3l/st0re's ci.yml, which + # documents three measured defects on these runners (alias collision across + # repos, sibling-run collision behind one alias, and DNS round-robin): + # * repo-unique aliases (opt-ci-opensearch / opt-ci-clickhouse), + # * SELF-IDENTIFYING containers — OpenSearch carries a run-unique + # cluster.name; ClickHouse carries run-unique credentials — so the + # select step can positively identify THIS run's container, + # * enumerate every IP behind the alias (getent ahostsv4), keep the one + # that proves the run identity, freeze it into the URL env var, + # * no match = hard failure, never a silent fallback or skip. integration: runs-on: ubuntu-latest services: @@ -41,7 +65,22 @@ jobs: discovery.type: single-node DISABLE_SECURITY_PLUGIN: "true" OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" + # Identity badge: only THIS run's container answers with this + # cluster_name (the OpenSearch image maps dotted env keys to + # settings). cluster.name: opt_ci_${{ github.run_id }}_${{ github.run_attempt }} + # `--workdir` is REQUIRED (measured 2026-08-31, PR #25 CI run 7745, + # job "integration"): this runner's act_runner (v0.6.1) merges the + # JOB's WorkingDir (`/workspace/sax3l/0pt`) into every SERVICE + # container's config, clobbering the image's own `WORKDIR + # /usr/share/opensearch`. OpenSearch's ENTRYPOINT is the + # WORKDIR-relative `./opensearch-docker-entrypoint.sh`, so with the + # wrong cwd the container failed with "exec: + # \"./opensearch-docker-entrypoint.sh\": stat ...: no such file or + # directory" before it ever bound a port. ClickHouse's entrypoint is + # an absolute path, so it was unaffected by the same defect — do not + # assume a green ClickHouse service means service-container options + # are fine in general. options: >- --workdir /usr/share/opensearch --health-cmd "curl -fsS http://localhost:9200/_cluster/health || exit 1" @@ -52,6 +91,9 @@ jobs: image: clickhouse/clickhouse-server:24.8 env: CLICKHOUSE_USER: opt_ci + # Identity badge: run-unique password — an authenticated SELECT only + # succeeds against THIS run's container. CI-scoped throwaway value, + # not a secret (exists only for the lifetime of the job). CLICKHOUSE_PASSWORD: ci_${{ github.run_id }}_${{ github.run_attempt }} CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" options: >- @@ -59,17 +101,6 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 60 - opt-ci-postgres: - image: postgres:16-alpine - env: - POSTGRES_USER: opt_ci - POSTGRES_PASSWORD: ci_${{ github.run_id }}_${{ github.run_attempt }} - POSTGRES_DB: opt_ci - options: >- - --health-cmd "pg_isready -U opt_ci -d opt_ci || exit 1" - --health-interval 5s - --health-timeout 5s - --health-retries 30 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -81,7 +112,7 @@ jobs: corepack prepare pnpm@9 --activate - name: Install (frozen lockfile) run: pnpm install --frozen-lockfile - - name: Select THIS run's OpenSearch by identity + - name: Select THIS run's OpenSearch by identity (st0re ci.yml pattern) run: | set -eu WANT="opt_ci_${{ github.run_id }}_${{ github.run_attempt }}" @@ -106,53 +137,22 @@ jobs: [ -n "$FOUND" ] || { echo "FATAL: no ClickHouse behind the alias accepts this run's credentials"; exit 1; } echo "CLICKHOUSE_URL=http://$FOUND:8123" >> "$GITHUB_ENV" echo "CLICKHOUSE_AUTH=opt_ci:$PASS" >> "$GITHUB_ENV" - - name: Select THIS run's Postgres by run-unique credentials + - name: Integration tests against LIVE OpenSearch + ClickHouse (blocking) run: | set -eu - PASS="ci_${{ github.run_id }}_${{ github.run_attempt }}" - FOUND="" - for ip in $(getent ahostsv4 opt-ci-postgres | awk '{print $1}' | sort -u); do - if PG_CANDIDATE="postgres://opt_ci:$PASS@$ip:5432/opt_ci" node --input-type=module - <<'NODE' - import postgres from 'postgres'; - const sql = postgres(process.env.PG_CANDIDATE, { max: 1, connect_timeout: 3, onnotice: () => undefined }); - try { - const rows = await sql`SELECT current_user AS u, current_database() AS d`; - if (rows[0]?.u !== 'opt_ci' || rows[0]?.d !== 'opt_ci') process.exitCode = 2; - } catch { - process.exitCode = 1; - } finally { - await sql.end({ timeout: 1 }).catch(() => undefined); - } - NODE - then - FOUND="$ip" - break - fi - done - [ -n "$FOUND" ] || { echo "FATAL: no Postgres behind the alias accepts this run's credentials"; exit 1; } - URL="postgres://opt_ci:$PASS@$FOUND:5432/opt_ci" - echo "TEST_DATABASE_URL=$URL" >> "$GITHUB_ENV" - echo "DATASET_TEST_DATABASE_URL=$URL" >> "$GITHUB_ENV" - echo "SEARCH_DEFINITION_TEST_DATABASE_URL=$URL" >> "$GITHUB_ENV" - - name: Integration tests against LIVE OpenSearch + ClickHouse + Postgres (blocking) - run: | - set -eu - [ -n "${OPENSEARCH_URL:-}" ] && [ -n "${CLICKHOUSE_URL:-}" ] && [ -n "${SEARCH_DEFINITION_TEST_DATABASE_URL:-}" ] + # Guard: env MUST be present here — the suites self-skip without it, + # and a skipped suite must never masquerade as a green integration + # gate (Master-DoD §56 status honesty). + [ -n "${OPENSEARCH_URL:-}" ] && [ -n "${CLICKHOUSE_URL:-}" ] pnpm vitest run tests/integration 2>&1 | tee /tmp/itest.log if grep -qE "skipped" /tmp/itest.log && ! grep -qE "Tests [0-9]+ passed" /tmp/itest.log; then echo "FATAL: integration suites skipped instead of running"; exit 1 fi - - name: Durable AUD0 outbox against LIVE Postgres (blocking) - run: | - set -eu - [ -n "${TEST_DATABASE_URL:-}" ] - pnpm vitest run tests/p0/aud0-outbox-postgres.realinfra.test.ts - - name: Dataset registry + versions against LIVE Postgres (blocking) - run: | - set -eu - [ -n "${DATASET_TEST_DATABASE_URL:-}" ] - pnpm vitest run tests/p0/dataset-postgres.realinfra.test.ts + # Kept in ci.yml (advisory) so the protected-branch status check contexts + # "CI (SIAX Cloud) / quality (push)" + "CI (SIAX Cloud) / security (push)" + # are always reported on push. security.yml is the dedicated scanner on + # push/PR/weekly; this job guarantees the required check exists. security: runs-on: ubuntu-latest steps: @@ -160,6 +160,6 @@ jobs: - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin - - name: Trivy scan (vulnerability + secret; HIGH/CRITICAL blocking) + - name: Trivy scan (vuln + secrets, advisory) run: | - trivy fs --scanners vuln,secret --severity HIGH,CRITICAL --exit-code 1 . + trivy fs --scanners vuln,secret --severity HIGH,CRITICAL --exit-code 0 . diff --git a/.gitea/workflows/security.yml b/.gitea/workflows/security.yml index 4ffa6f1..5717347 100644 --- a/.gitea/workflows/security.yml +++ b/.gitea/workflows/security.yml @@ -1,15 +1,21 @@ name: Security Scan (SIAX Cloud) -# Blocking vulnerability/secret scan on every push/PR + weekly re-run. -# HIGH/CRITICAL findings must be fixed or carried by an explicit, scoped, -# time-bounded exception outside this workflow; CI never silently downgrades them. +# Advisory security scanning on every push/PR + weekly re-run, matching the +# fleet convention (--exit-code 0 so findings never block merges; they are +# triaged via the repo's security channel and BACKLOG). +# +# NOTE: findings on services/scrapers/* (FlareSolverr/Tor/curl_cffi) are +# expected and out of scope for CI gating -- kept per the account owner's +# dated, attributed decision (Simon Axelsson, 2026-08-11, interactive +# prompt; no rationale beyond "keep it" given or invented). See SECURITY.md +# and docs/compliance/GOLDEN-STANDARD-REPORT.md control C6 for full context. on: push: branches: ["**"] pull_request: workflow_dispatch: schedule: - - cron: "0 5 * * 1" + - cron: "0 5 * * 1" # weekly Monday re-scan jobs: security: @@ -19,6 +25,6 @@ jobs: - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin - - name: Trivy fs scan (vulnerability + secret; HIGH/CRITICAL blocking) + - name: Trivy fs scan (vuln + secrets, advisory) run: | - trivy fs --scanners vuln,secret --severity HIGH,CRITICAL --exit-code 1 . + trivy fs --scanners vuln,secret --severity HIGH,CRITICAL --exit-code 0 .