Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sharang
|
5992e4b8e0 | ||
|
sharang
|
cfb94d3259 | ||
|
sharang
|
1fd03592c0 | ||
|
sharang
|
a8593091c2 | ||
|
sharang
|
03a5b4846e | ||
|
sharang
|
1ed2dcee57 |
+37
-10
@@ -1,6 +1,7 @@
|
||||
# CI skeleton (TypeScript shape; no app code yet).
|
||||
# Lights up to commitlint + gitleaks + trivy fs scan. Add lint/test/build jobs
|
||||
# when this repo grows real package code.
|
||||
# CI skeleton — applies to every repo until per-stack code lands.
|
||||
# The `shared` job runs on every push + PR; per-stack `test`/`image`/`e2e`
|
||||
# jobs light up automatically when the repo gains go.sum / package.json /
|
||||
# Cargo.toml etc. — see the conditional gates below.
|
||||
name: ci
|
||||
|
||||
on:
|
||||
@@ -18,14 +19,40 @@ jobs:
|
||||
|
||||
- name: commitlint (PR only)
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: wagoid/commitlint-github-action@v6
|
||||
shell: bash
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pattern='^(feat|fix|docs|chore|refactor|test|perf|build|ci|revert)(\([a-z0-9_/.-]+\))?!?: .{1,72}$'
|
||||
bad=0
|
||||
while IFS= read -r subject; do
|
||||
if ! [[ "$subject" =~ $pattern ]]; then
|
||||
echo "::error::Commit subject does not match Conventional Commits: $subject"
|
||||
bad=$((bad+1))
|
||||
fi
|
||||
done < <(git log --format=%s "${BASE_SHA}..${HEAD_SHA}")
|
||||
if [ "$bad" -gt 0 ]; then
|
||||
echo "::error::$bad commit(s) failed commitlint. See https://www.conventionalcommits.org/"
|
||||
exit 1
|
||||
fi
|
||||
echo "commitlint: all commit subjects OK"
|
||||
|
||||
- name: gitleaks
|
||||
uses: gitleaks/gitleaks-action@v2
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
GL_VERSION=8.18.4
|
||||
curl -fsSL "https://github.com/gitleaks/gitleaks/releases/download/v${GL_VERSION}/gitleaks_${GL_VERSION}_linux_x64.tar.gz" \
|
||||
| tar -xz -C /tmp gitleaks
|
||||
/tmp/gitleaks detect --source . --no-banner --redact --verbose --exit-code 1
|
||||
|
||||
- name: trivy fs scan
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
scan-type: fs
|
||||
severity: HIGH,CRITICAL
|
||||
exit-code: 1
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TRIVY_VERSION=0.70.0
|
||||
curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \
|
||||
| tar -xz -C /tmp trivy
|
||||
/tmp/trivy fs --severity HIGH,CRITICAL --exit-code 1 --no-progress --skip-dirs node_modules,target,dist .
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
runs-on: docker
|
||||
environment:
|
||||
name: production # Gitea Environments — requires sign-off per branch protection
|
||||
url: https://yourplatform.com
|
||||
url: https://breakpilot.com
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with: { fetch-depth: 0 }
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
|
||||
- name: verify stage soak (>= 24h on this image)
|
||||
run: |
|
||||
IMG=registry.yourplatform.com/${{ github.event.repository.name }}:env-stage
|
||||
IMG=repo.breakpilot.com/breakpilot/${{ github.event.repository.name }}:env-stage
|
||||
SOAK_SECONDS=$(orca image-age --env=stage --image $IMG)
|
||||
if [ "$SOAK_SECONDS" -lt 86400 ]; then
|
||||
echo "Stage soak only $SOAK_SECONDS s, < 24h. Aborting."
|
||||
@@ -34,12 +34,12 @@ jobs:
|
||||
- name: re-tag image as semver + env-prod
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: registry.yourplatform.com
|
||||
registry: repo.breakpilot.com
|
||||
username: ${{ secrets.REGISTRY_USER }}
|
||||
password: ${{ secrets.REGISTRY_PASS }}
|
||||
|
||||
- run: |
|
||||
IMG=registry.yourplatform.com/${{ github.event.repository.name }}
|
||||
IMG=repo.breakpilot.com/breakpilot/${{ github.event.repository.name }}
|
||||
docker pull $IMG:env-stage
|
||||
docker tag $IMG:env-stage $IMG:v${{ steps.v.outputs.version }}
|
||||
docker tag $IMG:env-stage $IMG:env-prod
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
curl -X POST -H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$(jq -Rs '{tag_name:"v${{ steps.v.outputs.version }}", name:"v${{ steps.v.outputs.version }}", body:.}' < RELEASE_NOTES.md)" \
|
||||
https://gitea.meghsakha.com/api/v1/repos/${{ github.repository }}/releases
|
||||
https://git.breakpilot.com/api/v1/repos/${{ github.repository }}/releases
|
||||
|
||||
rollback-on-failure:
|
||||
needs: promote
|
||||
|
||||
+3
-1
@@ -6,12 +6,14 @@ Generated section is appended on release tag via `git-cliff` (see `.gitea/workfl
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- chore: smoke PR for M0.2 CI verification
|
||||
-
|
||||
|
||||
### Changed
|
||||
- chore(domain): yourplatform.com → breakpilot.com
|
||||
-
|
||||
|
||||
### Fixed
|
||||
- ci: rework workflow for Gitea Actions (bash commitlint, inline gitleaks binary, per-stack jobs gated on real code)
|
||||
-
|
||||
|
||||
### Removed
|
||||
|
||||
+1
-1
@@ -86,4 +86,4 @@ When reviewing, check in this order:
|
||||
|
||||
## Questions
|
||||
|
||||
`#engineering` channel · `oncall@yourplatform.com` · or open a PR with a `[WIP]` prefix and ask in the description.
|
||||
`#engineering` channel · `oncall@breakpilot.com` · or open a PR with a `[WIP]` prefix and ask in the description.
|
||||
|
||||
+109
-75
@@ -19,13 +19,21 @@ This is the build plan for an AI coding agent (Claude Code, executing PRs agains
|
||||
## 1. Cross-cutting conventions (apply to every PR in every repo)
|
||||
|
||||
### 1.1 Repo strategy
|
||||
Polyrepo under a new Gitea org `gitea.meghsakha.com/platform/`. One repo per deployable unit. Existing product repos stay where they are.
|
||||
Polyrepo under the Gitea org `gitea.meghsakha.com/platform/`. One repo per deployable unit. Existing product repos stay where they are.
|
||||
|
||||
**Repos to create:**
|
||||
**Infrastructure repos.** The IaC is split by Orca cluster — one cluster per plane, one repo per cluster — per the 2026-06-30 cluster-split decision (see [Plane model memory](../../../../home/sharang/.claude/projects/-home-sharang-workspace/memory/project_breakpilot_plane_model.md) and `INFRASTRUCTURE.md §6`). Until the legal entity is established and SysEleven / Hetzner business contracts can be signed, the cluster manifests live as design-only staging directories under `platform/orca-platform/clusters/`. At migration time each `clusters/<name>/` subdir becomes its own repo:
|
||||
|
||||
| Repo | Purpose | Created in |
|
||||
|---|---|---|
|
||||
| `platform/orca-platform` | IaC staging — cluster.toml templates, per-cluster service manifests under `clusters/breakpilot-{edge,control,app}/`, overlays, DNS zones, dev-compose | M1.1 |
|
||||
| `platform/breakpilot-edge` | Identity + Infra cluster (vm-edge): Keycloak, Gitea, Infisical, PowerDNS, Orca-Proxy | M1.2 (split from `orca-platform/clusters/breakpilot-edge/`) |
|
||||
| `platform/breakpilot-control` | Control cluster (vm-control): portal, tenant-registry, ERPNext, MariaDB, Stalwart, Frappe HD | M1.2 (split from `orca-platform/clusters/breakpilot-control/`) |
|
||||
| `platform/breakpilot-app` | App cluster (vm-app-prod + vm-app-stage): CERTifAI, compliance-*, Mongo, MinIO, Qdrant, LiteLLM, pg-app | M1.2 (split from `orca-platform/clusters/breakpilot-app/`) |
|
||||
|
||||
**Product / business repos:**
|
||||
|
||||
| Repo | Purpose | Created in |
|
||||
|---|---|---|
|
||||
| `platform/orca-platform` | IaC for VMs, Orca manifests, DNS, TLS, backups | M1.1 |
|
||||
| `platform/tenant-registry` | Go service: tenant glue, audit, API keys | M4.1 |
|
||||
| `platform/portal` | Next.js 15: customer area + backstage | M5.1 |
|
||||
| `platform/docs` | Architecture, integration spec, this plan, runbooks | M0.1 |
|
||||
@@ -110,18 +118,26 @@ Per language defaults:
|
||||
- **Database migrations** are forward-only and run as an init container before the service starts. Migrations that delete columns require two releases (1: stop writing, 2: drop).
|
||||
|
||||
### 1.8 Environments
|
||||
Three Orca clusters, all on the same hardware until volume justifies separation:
|
||||
Three Orca clusters, **one per plane** (not one per env). Stage and prod live in the same `breakpilot-app` cluster on different VMs:
|
||||
|
||||
| Env | Cluster name | Purpose | Data | Auto-deploy? |
|
||||
|---|---|---|---|---|
|
||||
| dev | local | Developer machine, docker-compose | fixtures | n/a |
|
||||
| stage | `orca-stage` | Pre-prod validation | seeded demo + synthetic customers | yes (on merge to main) |
|
||||
| prod | `orca-prod` | Live customer traffic | real | tag + gate |
|
||||
| Cluster | VMs | Plane | Envs hosted |
|
||||
|---|---|---|---|
|
||||
| `breakpilot-edge` | `vm-edge` (1) | Identity + Infra | shared by stage + prod (single Keycloak realm, `tenant.kind` differentiates) |
|
||||
| `breakpilot-control` | `vm-control` (1) | Control | shared by stage + prod (single tenant-registry, `tenant.kind` differentiates) |
|
||||
| `breakpilot-app` | `vm-app-prod`, `vm-app-stage` (2) | App | prod on `vm-app-prod`, stage on `vm-app-stage` |
|
||||
|
||||
| Env | Where it runs | Data | Auto-deploy? |
|
||||
|---|---|---|---|
|
||||
| dev | Developer machine, docker-compose (`platform/orca-platform/dev/`) | fixtures | n/a |
|
||||
| stage | `vm-app-stage` in `breakpilot-app`; calls prod KC + prod tenant-registry under `tenant.kind = "stage"` | seeded demo + synthetic customers | yes (on merge to main) |
|
||||
| prod | `vm-app-prod` in `breakpilot-app` + `vm-control` + `vm-edge` | real | tag + gate |
|
||||
|
||||
Identity and Control are NOT duplicated for stage. Stage workloads authenticate against the prod Keycloak realm with `tenant.kind = "stage"` and read tenant config from the prod `tenant-registry` (read-only for stage tenants) per `INFRASTRUCTURE.md §5`.
|
||||
|
||||
Domain pattern:
|
||||
- dev: `*.localhost` (mkcert)
|
||||
- stage: `*.stage.yourplatform.com`
|
||||
- prod: `*.yourplatform.com`
|
||||
- stage: `*.stage.breakpilot.com`
|
||||
- prod: `*.breakpilot.com`
|
||||
|
||||
### 1.9 Observability + audit
|
||||
- **SigNoz** (already running at `signoz.meghsakha.com`) for traces, logs, metrics. Every service ships OTel SDK from day one.
|
||||
@@ -157,6 +173,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
- **Tests:** n/a
|
||||
- **Gate:** standard
|
||||
- **Effort:** S
|
||||
- **Note:** the three cluster repos (`platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app`) are NOT created at M0.1 — their service manifests live as design-only directories under `platform/orca-platform/clusters/` until M1.2 splits them.
|
||||
|
||||
### M0.2 — CI templates + branch protection
|
||||
- **Depends on:** M0.1
|
||||
@@ -169,66 +186,81 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
|
||||
### M0.3 — Self-hosted DNS + wildcard TLS
|
||||
- **Depends on:** M1.2 (vm-edge must exist before PowerDNS lands)
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Repos:** `platform/breakpilot-edge`
|
||||
- **Deliverables:**
|
||||
- **PowerDNS Authoritative** on `vm-edge` (Orca-managed). PostgreSQL backend on same VM (small; ~100 records).
|
||||
- At the registrar (Benjamin's account): set `ns1.yourplatform.com` and `ns2.yourplatform.com` glue records pointing at vm-edge public IP; delegate the domain to those NS.
|
||||
- Zone file committed in `orca-platform/dns/yourplatform.com.zone`; Orca syncs into PowerDNS on apply.
|
||||
- Records: apex `yourplatform.com`, wildcards `*.yourplatform.com` + `*.stage.yourplatform.com`, plus `auth.`, `erp.`, `mcp.`, `cdn.`, `mail.`, `ns1.`, `ns2.`, SPF/DKIM/DMARC TXT records (for M3.2).
|
||||
- At the registrar (Benjamin's account): set `ns1.breakpilot.com` and `ns2.breakpilot.com` glue records pointing at vm-edge public IP; delegate the domain to those NS.
|
||||
- Zone file committed in `orca-platform/dns/breakpilot.com.zone`; Orca syncs into PowerDNS on apply.
|
||||
- Records: apex `breakpilot.com`, wildcards `*.breakpilot.com` + `*.stage.breakpilot.com`, plus `auth.`, `erp.`, `mcp.`, `cdn.`, `mail.`, `ns1.`, `ns2.`, SPF/DKIM/DMARC TXT records (for M3.2).
|
||||
- Wildcard TLS via Let's Encrypt **DNS-01 against PowerDNS** (Lego's `--dns=pdns` provider); ACME credentials in Infisical at `/prod/orca-proxy/PDNS_API_KEY`.
|
||||
- Orca-Proxy reloads the cert via watch on the secret file; renewal cron runs at 02:00 daily.
|
||||
- **Acceptance:** `dig @1.1.1.1 anything.yourplatform.com` returns an answer; `curl https://anything.yourplatform.com` returns 404 from Orca-Proxy (no TLS error).
|
||||
- **Acceptance:** `dig @1.1.1.1 anything.breakpilot.com` returns an answer; `curl https://anything.breakpilot.com` returns 404 from Orca-Proxy (no TLS error).
|
||||
- **Tests:** ACME renewal dry-run; PowerDNS zone-diff check in CI; reach via stage and prod subdomains; cert expiry page wired to SigNoz alert.
|
||||
- **Gate:** standard + manual DNS-delegation check by both founders (irreversible from registrar side without 24–48h propagation)
|
||||
- **Effort:** M (was S — registrar delegation + PowerDNS adds setup time vs. Cloudflare)
|
||||
|
||||
### M1.1 — `orca-platform` repo (IaC)
|
||||
### M1.1 — `orca-platform` repo (IaC staging)
|
||||
- **Depends on:** M0.1, M0.2
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Deliverables:** directory layout per `INFRASTRUCTURE.md`; one Orca manifest per VM × service; per-env overlays (`overlays/dev`, `overlays/stage`, `overlays/prod`); a `Makefile` with `make plan` / `make apply` per env.
|
||||
- **Acceptance:** `make plan ENV=stage` produces a no-op diff once applied.
|
||||
- **Tests:** `orca validate` runs in CI; PRs that break a manifest fail.
|
||||
- **Deliverables:** per-cluster manifest layout under `clusters/breakpilot-{edge,control,app}/services/`; per-env overlays (`overlays/dev`, `overlays/stage`, `overlays/prod`); `cluster.toml.tmpl` template parameterized by cluster; a `Makefile` with `make plan` / `make apply` per env; `scripts/validate.sh` enforces a per-cluster `placement.node` whitelist (`breakpilot-edge → vm-edge`, `breakpilot-control → vm-control`, `breakpilot-app → {vm-app-prod, vm-app-stage}`).
|
||||
- **Acceptance:** `make plan ENV=stage` and `make plan ENV=prod` both resolve cleanly; `make validate` passes the per-cluster node check on all manifests.
|
||||
- **Tests:** `make validate` runs in CI; PRs that break a manifest or cross-place a service into the wrong cluster's node fail.
|
||||
- **Gate:** standard
|
||||
- **Effort:** M
|
||||
- **Status (2026-06-30):** layout landed; cluster split refactor merged via PR #6 (`6be727d4`). Manifests under `clusters/breakpilot-*/services/` are stubs awaiting per-milestone fill-in. `make apply` is a no-op until M1.2.
|
||||
|
||||
### M1.2 — Provision VMs (locked topology)
|
||||
- **Depends on:** M1.1 (Orca manifest layout)
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Deliverables:** the **4 VMs** from `INFRASTRUCTURE.md §1` provisioned on SysEleven (DUS2):
|
||||
- **stage** (m2.small, public IP) — runs app-plane code only, calls prod KC + Stalwart
|
||||
- **vm-edge** (m2.small, public IP) — Identity + Infra planes (orca-proxy, PowerDNS, Keycloak, pg-keycloak, Infisical, pg-infisical, Gitea)
|
||||
- **vm-control** (m2.medium) — Control plane (portal, tenant-registry, ERPNext, Frappe HD, MariaDB, Stalwart)
|
||||
- **vm-data** (m2.medium) — Data plane (CERTifAI, MongoDB, LiteLLM, compliance ×3, pg-app, Qdrant, MinIO)
|
||||
- Private network 10.0.0.0/16 between all four. Public ingress only via vm-edge (and stage's own IP for tester access).
|
||||
- SSH disabled; only `orca exec` for shell access.
|
||||
- **Acceptance:** every VM reachable from Orca control plane; private-network connectivity verified; resource limits per service set in manifest per `INFRASTRUCTURE.md §6` co-tenant notes.
|
||||
- **Tests:** cold-start sequence from `INFRASTRUCTURE.md §10 Scenario F` runs successfully on stage VMs.
|
||||
- **Gate:** standard + manual sign-off (touches infra spend and 36M commitment decision)
|
||||
- **Effort:** M
|
||||
- **Cost impact:** see COST_PLAN.md §3. Initial run: ~€552/mo On-Demand, dropping to ~€310/mo after 36M-upfront commit in Month 4.
|
||||
### M1.2 — Provision VMs + split cluster repos (locked topology)
|
||||
- **Depends on:** M1.1 (manifest layout exists); **legal entity established** (gating SysEleven / Hetzner business contracts; no commercial activity before this).
|
||||
- **Repos:** `platform/orca-platform` (source of split), `platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app` (created in this milestone).
|
||||
- **Deliverables:** **4 VMs across 3 clusters**, one cluster per plane (per the 2026-06-30 cluster-split decision):
|
||||
|
||||
| Cluster | VM | Flavor | Plane | Services |
|
||||
|---|---|---|---|---|
|
||||
| `breakpilot-edge` | `vm-edge` | m2.small, public IP | Identity + Infra | orca-proxy, PowerDNS, Keycloak (repurposed from CERTifAI), pg-keycloak, Infisical, pg-infisical, redis-infisical, Gitea |
|
||||
| `breakpilot-control` | `vm-control` | m2.medium | Control | portal, tenant-registry, ERPNext, Frappe HD, MariaDB, redis-erpnext, Stalwart |
|
||||
| `breakpilot-app` | `vm-app-prod` | m2.medium | App (prod) | CERTifAI, compliance ×3, MongoDB, LiteLLM, pg-app, Qdrant, MinIO |
|
||||
| `breakpilot-app` | `vm-app-stage` | m2.small, public IP | App (stage) | slim parallel set for staging; calls prod KC + prod tenant-registry under `tenant.kind = "stage"` |
|
||||
|
||||
- **Split the cluster repos:** for each cluster, lift `platform/orca-platform/clusters/breakpilot-<name>/` into its own Gitea repo `platform/breakpilot-<name>` with the §1.2 scaffolding + its own `cluster.toml` + its own `services/` tree. Empty out (or stub) the original `clusters/breakpilot-<name>/` in `orca-platform` and leave a pointer README.
|
||||
- **Repurpose the existing Keycloak.** The CERTifAI Keycloak realm export + user database moves to `vm-edge` rather than standing up a fresh instance — a hostname swap and realm export/import, not a re-bootstrap.
|
||||
- **Private network** 10.0.0.0/16 between all four VMs.
|
||||
- **Public ingress** per Orca's per-node-ingress model: every VM binds 80/443 and runs its own Let's Encrypt ACME; A records for a service point at the agent IP that runs the container, not at a master.
|
||||
- **SSH disabled**; only `orca exec` for shell access.
|
||||
- **Acceptance:**
|
||||
- Every VM is reachable from its cluster's Orca master; private-network connectivity verified.
|
||||
- Resource limits per service set in manifest per `INFRASTRUCTURE.md §6` co-tenant notes.
|
||||
- `make validate` (in `orca-platform`) and the new per-cluster repo CIs all pass.
|
||||
- Cold-start sequence from `INFRASTRUCTURE.md §10 Scenario F` runs successfully on `vm-app-stage`.
|
||||
- **Tests:** cold-start drill on `vm-app-stage`; cross-cluster reachability matrix; KC realm-import smoke (stage tenant logs in).
|
||||
- **Gate:** standard + manual sign-off (touches infra spend and 36M commitment decision) + legal-entity confirmation before any contract signing.
|
||||
- **Effort:** L (was M — adds the 3-repo split + KC realm migration on top of provisioning).
|
||||
- **Cost impact:** see COST_PLAN.md §3. Initial run: ~€552/mo On-Demand, dropping to ~€310/mo after 36M-upfront commit in Month 4. `vm-app-stage` is the cheapest VM in the topology so adding it doesn't materially shift the bill.
|
||||
|
||||
**Pre-M1.2 reality (today, single-VM era):** until this milestone lands, everything runs on the existing single VM (`46.225.100.82`) via `~/workspace/orca-infra`. M7.x work (tenant-registry-aware compliance-agent, per-tenant MCP) and other product work is meaningful on the single-VM setup and ports cleanly into `breakpilot-app/services/prod/` at migration time.
|
||||
|
||||
### M1.3 — Backups, monitoring, on-call
|
||||
- **Depends on:** M1.2
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Deliverables:** backup cron per VM per `INFRASTRUCTURE.md §3` (Postgres pg_dump, MinIO bucket replication); SigNoz OTel collector running on every VM; alert routing to `oncall@yourplatform.com`; restore runbook in `platform/docs/runbooks/restore.md`.
|
||||
- **Acceptance:** restore drill on stage succeeds (script in `platform/orca-platform/scripts/restore-drill.sh`); SigNoz shows traces from a synthetic request.
|
||||
- **Repos:** `platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app` (each cluster owns its own `[backup]` block + restore script per `INFRASTRUCTURE.md §6`)
|
||||
- **Deliverables:** backup cron per VM per `INFRASTRUCTURE.md §3` (Postgres pg_dump on `pg-keycloak` / `pg-app` / `mariadb`, MinIO bucket replication); SigNoz OTel collector running on every VM; alert routing to `oncall@breakpilot.com`; restore runbook in `platform/docs/runbooks/restore.md`. Each cluster repo has its own S3 bucket per `INFRASTRUCTURE.md §6` so a runaway backup on `breakpilot-app` cannot fill the bucket `breakpilot-edge` writes to.
|
||||
- **Acceptance:** restore drill on `vm-app-stage` succeeds (script in `breakpilot-app/scripts/restore-drill.sh`); SigNoz shows traces from a synthetic request through `vm-app-stage` → `vm-control` → `vm-edge`.
|
||||
- **Tests:** disaster-recovery exercise per failure scenario in `INFRASTRUCTURE.md §10` — at least Scenarios A, B, F validated on stage.
|
||||
- **Gate:** standard + manual sign-off
|
||||
- **Effort:** L
|
||||
|
||||
### M2.1 — Keycloak deployment
|
||||
### M2.1 — Keycloak deployment (repurpose from CERTifAI)
|
||||
- **Depends on:** M1.2, M1.3
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Deliverables:** Keycloak 26 on `vm-identity`, Postgres backing store on `vm-control`, exposed at `auth.yourplatform.com` and `auth.stage.yourplatform.com`. Realm import file in `orca-platform/keycloak/realm-export.json` (committed, source-of-truth).
|
||||
- **Acceptance:** master admin login works; realm `breakpilot-prod` exists in both envs.
|
||||
- **Tests:** automated realm-state diff in CI (`kcadm` against checked-in export).
|
||||
- **Gate:** standard + security checklist
|
||||
- **Repos:** `platform/breakpilot-edge`
|
||||
- **Deliverables:** Keycloak 26 on **`vm-edge`** (co-tenant with the rest of infra; JVM heap pinned `-Xmx1500m` so it cannot starve PowerDNS / Infisical, per `INFRASTRUCTURE.md §6`). Postgres backing store (`pg-keycloak`) also on `vm-edge`. Exposed at `auth.breakpilot.com`; stage tenants authenticate against the same instance under `tenant.kind = "stage"` (no separate stage realm). **The CERTifAI Keycloak instance is repurposed here** — its realm export + user database moves to `vm-edge` rather than standing up a fresh instance. Realm import file in `breakpilot-edge/services/keycloak/realm-export.json` (committed, source-of-truth).
|
||||
- **Acceptance:** master admin login works; realm `breakpilot-prod` exists; the realm carries the migrated CERTifAI users; a stage tenant test login succeeds with `tenant.kind = "stage"` in the JWT.
|
||||
- **Tests:** automated realm-state diff in CI (`kcadm` against checked-in export); side-by-side comparison of pre-migration and post-migration realm exports flags any silent drift.
|
||||
- **Gate:** standard + security checklist + manual sign-off on the realm-import diff (irreversible without restoring from KC backup).
|
||||
- **Effort:** M
|
||||
- **Note:** the escape hatch — if the JVM/PowerDNS fight shows up in alerts after launch — is to flip Keycloak's `placement.node` to a new `vm-edge-identity` VM inside the same `breakpilot-edge` cluster. No schema migration, just a container move.
|
||||
|
||||
### M2.2 — Realm configuration: roles + protocol mappers + Organizations
|
||||
- **Depends on:** M2.1
|
||||
- **Repos:** `platform/orca-platform` (realm config)
|
||||
- **Deliverables:** Organizations feature enabled; realm roles `BREAKPILOT_ADMIN`, `SUPPORT_ENGINEER`, `SALES_REP`; org roles `IT_ADMIN`, `CXO`, `FINANCE`, `LEGAL`, `USER`; protocol mapper that calls Tenant Registry at token issuance for `products`, `plan`, `tenant_status` claims; SALES_REP guardrail policy (token only issuable with `org_id = demo`).
|
||||
- **Repos:** `platform/breakpilot-edge` (realm config)
|
||||
- **Deliverables:** Organizations feature enabled; realm roles `BREAKPILOT_ADMIN`, `SUPPORT_ENGINEER`, `SALES_REP`; org roles `IT_ADMIN`, `CXO`, `FINANCE`, `LEGAL`, `USER`; protocol mapper that calls Tenant Registry at token issuance for `products`, `plan`, `tenant_status`, `tenant_kind` claims; SALES_REP guardrail policy (token only issuable with `org_id = demo`).
|
||||
- **Acceptance:** a test user gets the expected JWT claims; a SALES_REP user cannot get a JWT for a non-demo org (verified by integration test).
|
||||
- **Tests:** Keycloak integration suite in `platform/tenant-registry/test/keycloak_test.go`.
|
||||
- **Gate:** standard + security checklist
|
||||
@@ -236,32 +268,34 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
|
||||
### M3.1 — Infisical
|
||||
- **Depends on:** M1.2
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Deliverables:** Infisical on `vm-secrets`, machine identity per service, secret paths laid out per `PRODUCT_INTEGRATION_SPEC.md §9.4`.
|
||||
- **Repos:** `platform/breakpilot-edge`
|
||||
- **Deliverables:** Infisical on **`vm-edge`** (co-tenant with Keycloak, PowerDNS, Gitea), machine identity per service, secret paths laid out per `PRODUCT_INTEGRATION_SPEC.md §9.4`.
|
||||
- **Acceptance:** a stub service can read its secrets at startup; rotating a secret in Infisical UI is picked up on next pod start.
|
||||
- **Tests:** smoke test container reads secrets.
|
||||
- **Gate:** standard + security checklist
|
||||
- **Effort:** S
|
||||
- **Bootstrap exception (per `INFRASTRUCTURE.md §8 rule 3`):** Keycloak's `KC_DB_URL` lives in Orca env (not Infisical) because both run on `vm-edge` and we'd otherwise have a circular bootstrap dependency.
|
||||
|
||||
### M3.2 — Stalwart transactional email
|
||||
- **Depends on:** M0.3 (needs DNS records under our control), M3.1
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Repos:** `platform/breakpilot-control`
|
||||
- **Deliverables:**
|
||||
- **Stalwart** on `vm-control` (Orca-managed); reachable at `mail.yourplatform.com`.
|
||||
- DNS records added to the zone in M0.3: `mail` A record, MX → mail, SPF (`v=spf1 mx -all`), DKIM (Stalwart-generated public key), DMARC (`p=quarantine; rua=mailto:dmarc@yourplatform.com`), reverse DNS (PTR) configured at the cloud provider for the vm-control public IP — coordinate with vm-edge since outbound mail must egress from a host with a clean PTR.
|
||||
- **Stalwart** on `vm-control` (Orca-managed); reachable at `mail.breakpilot.com`.
|
||||
- DNS records added to the zone in M0.3: `mail` A record, MX → mail, SPF (`v=spf1 mx -all`), DKIM (Stalwart-generated public key), DMARC (`p=quarantine; rua=mailto:dmarc@breakpilot.com`), reverse DNS (PTR) configured at the cloud provider for the vm-control public IP — coordinate with vm-edge since outbound mail must egress from a host with a clean PTR.
|
||||
- SMTP submission service account per platform sender: `noreply@`, `oncall@`, `support@`, `billing@`, `dmarc@`.
|
||||
- Outbound queue and bounce handler; failed deliveries surface as audit events.
|
||||
- Webhook receiver at `/inbound/postmaster` for bounce/complaint feedback loops (Gmail FBL, MS SNDS).
|
||||
- **IP warming plan**: write a `platform/docs/runbooks/email-warming.md` documenting the 4–8 week ramp from low daily volumes; first 2 weeks of trial nudges (M12.2) explicitly throttled.
|
||||
- **Acceptance:** test email from `noreply@yourplatform.com` to `parnerkarsharang@gmail.com` lands in inbox (not spam) on day 1; SPF/DKIM/DMARC all "pass" in Gmail's "show original" view; mail-tester.com score ≥ 9/10.
|
||||
- **Acceptance:** test email from `noreply@breakpilot.com` to `parnerkarsharang@gmail.com` lands in inbox (not spam) on day 1; SPF/DKIM/DMARC all "pass" in Gmail's "show original" view; mail-tester.com score ≥ 9/10.
|
||||
- **Tests:** automated daily mail-tester check (failure pages on-call); bounce-handling integration test.
|
||||
- **Gate:** standard + security checklist + manual deliverability sign-off (DKIM keys are load-bearing)
|
||||
- **Effort:** L (deliverability tuning is the long tail)
|
||||
|
||||
**Phase 0 exit criteria:**
|
||||
- Stage cluster boots cold from cron-driven nightly stop/start using only `INFRASTRUCTURE.md §5` ordering.
|
||||
- A synthetic HTTPS request to `https://hello.stage.yourplatform.com` reaches a stub container.
|
||||
- Restore drill on stage Postgres succeeds end-to-end.
|
||||
- `vm-app-stage` boots cold from cron-driven nightly stop/start using only `INFRASTRUCTURE.md §5` ordering; `breakpilot-edge` and `breakpilot-control` clusters remain warm throughout (no nightly cycling for identity / control).
|
||||
- A synthetic HTTPS request to `https://hello.stage.breakpilot.com` reaches a stub container on `vm-app-stage`.
|
||||
- Restore drill on `vm-app-prod` Postgres (`pg-app`) and `vm-control` Postgres (`pg-keycloak`, `pg-mariadb`) both succeed end-to-end.
|
||||
- All three cluster repos (`platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app`) have green CI on main and a successful `make validate`.
|
||||
|
||||
---
|
||||
|
||||
@@ -300,7 +334,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
- **Depends on:** M2.2, M4.3, M0.3
|
||||
- **Repos:** `platform/portal`, `platform/design-tokens`
|
||||
- **Deliverables:** Next.js 15 app on `vm-control`; middleware reads `Host` → extracts slug → calls Tenant Registry `GET /tenants?slug=` → injects tenant context; Keycloak OIDC login; logout; `design-tokens` package consumed by portal.
|
||||
- **Acceptance:** visiting `https://acme.stage.yourplatform.com` redirects to Keycloak; after login, user lands on `/acme/dashboard` (empty page) with valid session.
|
||||
- **Acceptance:** visiting `https://acme.stage.breakpilot.com` redirects to Keycloak; after login, user lands on `/acme/dashboard` (empty page) with valid session.
|
||||
- **Tests:** Playwright e2e: login + logout for an existing test tenant.
|
||||
- **Gate:** standard
|
||||
- **Effort:** M
|
||||
@@ -317,14 +351,14 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
### M5.3 — Playwright e2e harness
|
||||
- **Depends on:** M5.2
|
||||
- **Repos:** `platform/portal`
|
||||
- **Deliverables:** Playwright config that runs against `stage.yourplatform.com` post-deploy; CI job `e2e-stage` triggered after stage deploy; failure pages on-call.
|
||||
- **Deliverables:** Playwright config that runs against `stage.breakpilot.com` post-deploy; CI job `e2e-stage` triggered after stage deploy; failure pages on-call.
|
||||
- **Acceptance:** breaking change to login is caught in CI within 10 min of merge.
|
||||
- **Tests:** the suite itself.
|
||||
- **Gate:** standard
|
||||
- **Effort:** S
|
||||
|
||||
**Phase 1 exit criteria:**
|
||||
- A tenant created via `POST /tenants` results in a working login flow at `<slug>.stage.yourplatform.com`.
|
||||
- A tenant created via `POST /tenants` results in a working login flow at `<slug>.stage.breakpilot.com`.
|
||||
- All Phase 1 routes have a passing Playwright spec running on every stage deploy.
|
||||
|
||||
---
|
||||
@@ -354,7 +388,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
### M6.3 — CERTifAI: manifest + integration assets
|
||||
- **Depends on:** M6.2
|
||||
- **Repos:** `benjamin_boenisch/certifai`
|
||||
- **Deliverables:** `product.manifest.yaml` per `PRODUCT_INTEGRATION_SPEC.md §10` published to `cdn.yourplatform.com`; OpenAPI 3.1 spec; `/v1/health`, `/v1/usage`, `/v1/tenants/:id/export`, `DELETE /v1/tenants/:id/data`, `POST /v1/tenants/demo/reset`; web component `certifai-dashboard` per §5.A.
|
||||
- **Deliverables:** `product.manifest.yaml` per `PRODUCT_INTEGRATION_SPEC.md §10` published to `cdn.breakpilot.com`; OpenAPI 3.1 spec; `/v1/health`, `/v1/usage`, `/v1/tenants/:id/export`, `DELETE /v1/tenants/:id/data`, `POST /v1/tenants/demo/reset`; web component `certifai-dashboard` per §5.A.
|
||||
- **Acceptance:** CERTifAI appears in the portal catalog; subscribed tenants can open it from the dashboard.
|
||||
- **Tests:** contract test that manifest validates against schema; web component renders inside portal shadow-DOM host.
|
||||
- **Gate:** standard
|
||||
@@ -390,8 +424,8 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
|
||||
### M8.1 — ERPNext deployment
|
||||
- **Depends on:** M1.2, M2.1
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Deliverables:** Frappe + ERPNext on `vm-control` (separate Postgres database from tenant_registry — see `INFRASTRUCTURE.md` RISK-1); reached at `erp.yourplatform.com`; Keycloak OIDC; IP-restricted at Orca-Proxy.
|
||||
- **Repos:** `platform/breakpilot-control`
|
||||
- **Deliverables:** Frappe + ERPNext on `vm-control` (separate Postgres database from tenant_registry — see `INFRASTRUCTURE.md` RISK-1); reached at `erp.breakpilot.com`; Keycloak OIDC; IP-restricted at Orca-Proxy.
|
||||
- **Acceptance:** us login works; a Customer record can be created manually.
|
||||
- **Tests:** smoke test for OIDC; backup of Frappe filestore validated.
|
||||
- **Gate:** standard + manual sign-off (touches `vm-control` resources)
|
||||
@@ -399,7 +433,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
|
||||
### M8.2 — ERPNext customization
|
||||
- **Depends on:** M8.1
|
||||
- **Repos:** `platform/orca-platform/erpnext-app/`
|
||||
- **Repos:** `platform/breakpilot-control` (`erpnext-app/` subdir under the cluster repo)
|
||||
- **Deliverables:** custom Frappe app with: `tenant_id` field on `Customer`; `sales_owner` field on `Lead`; server scripts for the Sales Order → Tenant Registry webhook; `Cancel` workflow that calls Tenant Registry `/cancel`.
|
||||
- **Acceptance:** submitting a Sales Order in ERPNext triggers a tenant activation in stage Tenant Registry.
|
||||
- **Tests:** server-script unit tests (Frappe test harness); integration test exercises the full webhook.
|
||||
@@ -424,7 +458,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
|
||||
### M9.1 — Frappe Helpdesk
|
||||
- **Depends on:** M8.1
|
||||
- **Repos:** `platform/orca-platform`
|
||||
- **Repos:** `platform/breakpilot-control`
|
||||
- **Deliverables:** Frappe HD on the same Frappe bench; customer portal embedded at `/[slug]/support/`.
|
||||
- **Acceptance:** a customer user can submit a ticket; we receive it.
|
||||
- **Tests:** Playwright spec for ticket submission.
|
||||
@@ -433,7 +467,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
|
||||
### M9.2 — HD → Gitea escalation
|
||||
- **Depends on:** M9.1
|
||||
- **Repos:** `platform/orca-platform/erpnext-app/`
|
||||
- **Repos:** `platform/breakpilot-control` (`erpnext-app/` subdir)
|
||||
- **Deliverables:** server script that on a `Ticket: Escalate to Engineering` action creates a Gitea issue in the matching repo via Gitea REST API; reverse webhook from Gitea on issue close marks ticket resolved.
|
||||
- **Acceptance:** the round-trip works for a test ticket on stage.
|
||||
- **Tests:** integration test against stage Gitea.
|
||||
@@ -489,7 +523,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
### M12.2 — Trial lifecycle cron + emails
|
||||
- **Depends on:** M12.1, M3.2 (Stalwart must be deliverability-clean)
|
||||
- **Repos:** `platform/tenant-registry`
|
||||
- **Deliverables:** scheduler in tenant-registry that runs day-7/12/14 emails; status transitions trial → active (on payment) or trial → frozen → archived; SMTP via Stalwart at `mail.yourplatform.com:587`; sender `noreply@yourplatform.com`; HTML + plaintext templates committed under `tenant-registry/templates/email/`; List-Unsubscribe headers per RFC 8058.
|
||||
- **Deliverables:** scheduler in tenant-registry that runs day-7/12/14 emails; status transitions trial → active (on payment) or trial → frozen → archived; SMTP via Stalwart at `mail.breakpilot.com:587`; sender `noreply@breakpilot.com`; HTML + plaintext templates committed under `tenant-registry/templates/email/`; List-Unsubscribe headers per RFC 8058.
|
||||
- **Acceptance:** in a time-warped stage test (script that advances `trial_ends_at`), all transitions fire in order and all three emails land in Gmail inbox.
|
||||
- **Tests:** integration test with time injection; deliverability spot-check at each release.
|
||||
- **Gate:** standard
|
||||
@@ -498,7 +532,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
### M13.1 — Demo tenant seeding
|
||||
- **Depends on:** M6.3, M7.2
|
||||
- **Repos:** `platform/seed-data`
|
||||
- **Deliverables:** per-product fixture archives (`certifai/seed-v1.tar.gz`, `compliance/seed-v1.tar.gz`); publishing pipeline to `cdn.yourplatform.com`; `catalog.demo.seed_data_url` populated in product manifests.
|
||||
- **Deliverables:** per-product fixture archives (`certifai/seed-v1.tar.gz`, `compliance/seed-v1.tar.gz`); publishing pipeline to `cdn.breakpilot.com`; `catalog.demo.seed_data_url` populated in product manifests.
|
||||
- **Acceptance:** calling `POST /v1/tenants/demo/reset` on either product restores fixtures.
|
||||
- **Tests:** integration test asserts fixture state after reset.
|
||||
- **Gate:** standard
|
||||
@@ -508,7 +542,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
- **Depends on:** M2.2, M13.1
|
||||
- **Repos:** `platform/portal`, `platform/tenant-registry`
|
||||
- **Deliverables:** demo tenant created in stage and prod with `kind=demo, status=demo`; SALES_REP role usable; backstage routes restricted to `/backstage/leads` and `/backstage/demo`; demo tenant audit events tagged `{"demo": true}` and hidden from real-tenant audit views.
|
||||
- **Acceptance:** sales rep logs in at `demo.yourplatform.com`, walks both products live, [Request Trial] modal creates a CRM Lead with `sales_owner = the rep`.
|
||||
- **Acceptance:** sales rep logs in at `demo.breakpilot.com`, walks both products live, [Request Trial] modal creates a CRM Lead with `sales_owner = the rep`.
|
||||
- **Tests:** Playwright e2e for the sales walk-through.
|
||||
- **Gate:** standard + security checklist (SALES_REP guardrail enforcement is the load-bearing piece)
|
||||
- **Effort:** M
|
||||
@@ -543,7 +577,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
**Phase 4 exit criteria:**
|
||||
- Every flow P1–P16 from `PLATFORM_ARCHITECTURE.md` has a passing Playwright spec.
|
||||
- Stage runs a full lifecycle: sign-up trial → convert → use → cancel → offboard, in an automated nightly job.
|
||||
- We can hand a prospect a real demo using `demo.yourplatform.com`.
|
||||
- We can hand a prospect a real demo using `demo.breakpilot.com`.
|
||||
|
||||
---
|
||||
|
||||
@@ -581,8 +615,8 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
||||
### M17.1 — MCP servers (Enterprise)
|
||||
- **Depends on:** M6.3, M7.2
|
||||
- **Repos:** `benjamin_boenisch/certifai`, `benjamin_boenisch/breakpilot-compliance`
|
||||
- **Deliverables:** MCP endpoints per `PRODUCT_INTEGRATION_SPEC.md §10` `mcp:` block; gated on `plan == enterprise`; routed via `mcp.yourplatform.com`.
|
||||
- **Acceptance:** Claude Code can connect to `mcp.yourplatform.com/certifai` with a service token and call `list_ai_agents`.
|
||||
- **Deliverables:** MCP endpoints per `PRODUCT_INTEGRATION_SPEC.md §10` `mcp:` block; gated on `plan == enterprise`; routed via `mcp.breakpilot.com`.
|
||||
- **Acceptance:** Claude Code can connect to `mcp.breakpilot.com/certifai` with a service token and call `list_ai_agents`.
|
||||
- **Tests:** MCP contract test using `mcp-cli`.
|
||||
- **Gate:** standard + security checklist
|
||||
- **Effort:** L
|
||||
@@ -598,7 +632,7 @@ These ship only when a paying customer requires them.
|
||||
|
||||
### M18.1 — Custom domains
|
||||
- **Depends on:** M0.3, M10.1
|
||||
- **Repos:** `platform/orca-platform`, `platform/portal`
|
||||
- **Repos:** `platform/breakpilot-edge` (Orca-Proxy ACME), `platform/portal` (UI)
|
||||
- **Deliverables:** ACME on-demand TLS in Orca-Proxy; portal UI for customer to add domain; CNAME verification.
|
||||
- **Acceptance:** `compliance.acme.com` resolves and renders the Acme portal.
|
||||
- **Tests:** integration test with a synthetic domain.
|
||||
@@ -607,7 +641,7 @@ These ship only when a paying customer requires them.
|
||||
|
||||
### M18.2 — Physical data isolation
|
||||
- **Depends on:** M4.1, M6.1, M7.1
|
||||
- **Repos:** all data-plane products + `tenant-registry`
|
||||
- **Repos:** all app-plane products + `platform/tenant-registry`
|
||||
- **Deliverables:** option per tenant for a dedicated Postgres / Mongo schema or database; provisioning automation; migration path from logical → physical.
|
||||
- **Acceptance:** an enterprise tenant runs on a dedicated schema; cross-tenant queries are physically impossible.
|
||||
- **Tests:** isolation enforcement test.
|
||||
@@ -645,7 +679,7 @@ When starting work, the first sequence of PRs should be:
|
||||
2. **PR-2 to PR-7** (M0.1 continued): Bootstrap each of the other five repos with §1.2 scaffolding. Land in parallel.
|
||||
3. **PR-8** (M0.2): CI templates + branch protection per repo.
|
||||
4. **PR-9** (M1.1): `orca-platform` directory layout + first stub manifest.
|
||||
5. **PR-10** (M1.2): VM provisioning (vm-edge, vm-identity, vm-secrets, vm-control first — DNS and Keycloak depend on these).
|
||||
5. **PR-10** (M1.2): VM provisioning + cluster-repo split. Per the 2026-06-30 decision: 4 VMs across 3 clusters (`vm-edge` in `breakpilot-edge`, `vm-control` in `breakpilot-control`, `vm-app-prod` + `vm-app-stage` in `breakpilot-app`). Spin up `vm-edge` first since DNS, Keycloak, and Infisical all live there. **Gated on legal entity** (cannot sign SysEleven / Hetzner business contracts before).
|
||||
6. **PR-11** (M0.3): PowerDNS on vm-edge + zone file + registrar NS delegation + wildcard TLS via Let's Encrypt DNS-01.
|
||||
|
||||
After PR-11, the dependency graph fans out and parallel work begins.
|
||||
@@ -690,7 +724,7 @@ That's 18 milestones. With one full-time agent and standard human review pacing,
|
||||
|
||||
**Parallelism opportunities:**
|
||||
- M6.x and M7.x can run fully in parallel (different repos, different stacks).
|
||||
- M8.x is independent of all data-plane work once M2.2 is done.
|
||||
- M8.x is independent of all app-plane work once M2.2 is done.
|
||||
- M15.x can begin as soon as M10.1 lands.
|
||||
|
||||
---
|
||||
@@ -703,7 +737,7 @@ That's 18 milestones. With one full-time agent and standard human review pacing,
|
||||
- ~~Cloudflare account ownership~~ → not used; DNS is self-hosted via PowerDNS on vm-edge (M0.3). Registrar account (Benjamin's) still needs documented 2FA recovery — see new DR item below.
|
||||
|
||||
**Still open:**
|
||||
- **CDN host** for `cdn.yourplatform.com`: self-hosted MinIO + Caddy on vm-edge is the OSS-aligned default; alternative is BunnyCDN (cheap, EU). Decide before M6.3 (manifest bundles + hero images).
|
||||
- **CDN host** for `cdn.breakpilot.com`: self-hosted MinIO + Caddy on vm-edge is the OSS-aligned default; alternative is BunnyCDN (cheap, EU). Decide before M6.3 (manifest bundles + hero images).
|
||||
- **Cloud provider for port 25 outbound.** Stalwart needs unblocked port 25 to send mail. Hetzner blocks by default and requires a request to unblock with proof of intent + abuse contact; OVH and Scaleway unblock on request faster. Confirm with Benjamin which provider vm-control runs on. Block on M3.2 if port 25 is unblockable — fallback is sending via a different provider's IP with reverse DNS.
|
||||
- **Test data privacy.** The demo tenant must contain ONLY synthetic data — confirm seed pipeline strips real PII even if our test orgs accidentally seed from prod.
|
||||
- **Registrar + DNS bus-factor.** Document who owns the registrar account, who has 2FA recovery codes, and the procedure to update NS records without that person available. Goes in `platform/docs/runbooks/dr.md` before M0.3 ships.
|
||||
|
||||
+96
-75
@@ -1,55 +1,59 @@
|
||||
# Infrastructure Specification
|
||||
**Status:** Locked Topology
|
||||
**Authors:** Sharang, Benjamin
|
||||
**Date:** 2026-05-11 (topology lock: 2026-05-18)
|
||||
**Date:** 2026-05-11 (topology lock: 2026-05-18; cluster split: 2026-06-30)
|
||||
**Companion docs:** PLATFORM_ARCHITECTURE.md, IMPLEMENTATION_PLAN.md, COST_PLAN.md
|
||||
**Cloud provider:** SysEleven Cloud Services (DUS2, OpenStack)
|
||||
**Cloud provider:** SysEleven Cloud Services (DUS2, OpenStack) — multi-VM rollout gated on legal entity
|
||||
|
||||
---
|
||||
|
||||
> **2026-06-30 cluster-split decision.** The 4 VMs from the May 18 lock are unchanged in count and flavor. What changes is how they're organized: **one Orca cluster per plane**, three cluster repos (`platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app`). Identity and Infra are co-tenant on `vm-edge` inside `breakpilot-edge` (single-VM core; KC JVM heap pinned so it cannot starve PowerDNS / Infisical). The "Data plane" is renamed to **App plane** to reflect that it carries product workloads, not just data stores. `vm-data` is renamed `vm-app-prod`, and the standalone `stage` VM is renamed `vm-app-stage` — both live in the same `breakpilot-app` cluster. Multi-VM rollout is gated on the legal entity being established so we can sign SysEleven business contracts; until then, single-VM ops continues via `~/workspace/orca-infra`. See `IMPLEMENTATION_PLAN.md §1.1 / M1.2` and `platform/orca-platform/clusters/README.md` for the migration mechanics. **The SLA targets in §6 / §7 below are unaffected** — they apply at the plane level regardless of which cluster a plane lives in.
|
||||
|
||||
## 1. VM Inventory
|
||||
|
||||
**Four billable VMs total.** Three in production (one per plane after collapsing Identity+Infra), one in stage. Dev runs entirely on developer laptops via docker-compose.
|
||||
**Four billable VMs across three Orca clusters.** Dev runs entirely on developer laptops via docker-compose.
|
||||
|
||||
```
|
||||
┌──────────────┬─────────────────┬────────────────────────┬───────────┬─────────────────┐
|
||||
│ Name │ Env │ SysEleven flavor │ Public IP │ Planes owned │
|
||||
├──────────────┼─────────────────┼────────────────────────┼───────────┼─────────────────┤
|
||||
│ vm-edge │ prod │ m2.small (2v / 8 GB) │ YES (1) │ Identity + Infra│
|
||||
│ vm-control │ prod │ m2.medium (4v / 16 GB) │ No │ Control │
|
||||
│ vm-data │ prod │ m2.medium (4v / 16 GB) │ No │ Data │
|
||||
│ stage │ stage │ m2.small (2v / 8 GB) │ YES (1) │ App plane only │
|
||||
│ (dev) │ dev │ local docker-compose │ n/a │ all (in-memory) │
|
||||
└──────────────┴─────────────────┴────────────────────────┴───────────┴─────────────────┘
|
||||
┌──────────────────┬────────────────────┬──────────┬────────────────────────┬───────────┬─────────────────┐
|
||||
│ Name │ Cluster │ Env │ SysEleven flavor │ Public IP │ Planes owned │
|
||||
├──────────────────┼────────────────────┼──────────┼────────────────────────┼───────────┼─────────────────┤
|
||||
│ vm-edge │ breakpilot-edge │ prod │ m2.small (2v / 8 GB) │ YES (1) │ Identity + Infra│
|
||||
│ vm-control │ breakpilot-control │ prod │ m2.medium (4v / 16 GB) │ No │ Control │
|
||||
│ vm-app-prod │ breakpilot-app │ prod │ m2.medium (4v / 16 GB) │ No │ App (prod) │
|
||||
│ vm-app-stage │ breakpilot-app │ stage │ m2.small (2v / 8 GB) │ YES (1) │ App (stage) │
|
||||
│ (dev) │ — │ dev │ local docker-compose │ n/a │ all (in-memory) │
|
||||
└──────────────────┴────────────────────┴──────────┴────────────────────────┴───────────┴─────────────────┘
|
||||
```
|
||||
|
||||
**Total compute:** 48 GiB-RAM, 12 vCPU. **Monthly compute net: €192 (36M upfront) / €295 (12M) / €435 (On-Demand).** See COST_PLAN.md for the full three-mode table.
|
||||
**Total compute:** 48 GiB-RAM, 12 vCPU. **Monthly compute net: €192 (36M upfront) / €295 (12M) / €435 (On-Demand).** See COST_PLAN.md for the full three-mode table. The cluster split does not change the per-VM bill (same flavors); the additional cost is operational (3 cluster repos to keep CI green in) and is offset by the independent failure-domain wins.
|
||||
|
||||
### Why this topology and not the previous 7-VM layout
|
||||
|
||||
The earlier draft proposed one VM per service group (vm-gateway, vm-identity, vm-secrets, vm-ops, vm-control, vm-certifai, vm-compliance). That gave maximum failure isolation but cost 132 GiB-RAM stage+prod. At 5 customers the isolation is unused — every VM ran at <10% utilisation. The locked topology buys back failure isolation incrementally as load grows (see §13 Growth Trajectory).
|
||||
|
||||
Critical isolations preserved even at 4 VMs:
|
||||
- **vm-edge isolates identity from app workloads.** Keycloak JVM has its own page cache; ERPNext background jobs cannot starve token issuance.
|
||||
- **vm-data isolates databases from stateless services.** All data-plane DBs share one host, but they're walled off from the portal + ERPNext + Stalwart competing on vm-control.
|
||||
- **stage runs the app plane only.** It calls prod Keycloak + prod Tenant Registry under `tenant.kind = stage` rather than mirroring those services.
|
||||
- **vm-edge isolates identity from app workloads.** Keycloak JVM has its own page cache + heap pin (`-Xmx1500m`) so ERPNext or product background jobs on a different VM cannot starve token issuance — and PowerDNS / Infisical on the same VM cannot be starved by KC either.
|
||||
- **vm-app-prod isolates product databases from stateless services.** All app-plane DBs share one host, but they're walled off from the portal + ERPNext + Stalwart competing on vm-control.
|
||||
- **vm-app-stage runs the app plane only.** It calls prod Keycloak (`breakpilot-edge`) + prod Tenant Registry (`breakpilot-control`) under `tenant.kind = stage` rather than mirroring those services. Stage and prod live in the **same** `breakpilot-app` Orca cluster on **different** VMs — same config, different physical workloads.
|
||||
|
||||
---
|
||||
|
||||
## 2. Service-to-VM Mapping
|
||||
|
||||
Each VM is owned by one Orca cluster (see §1). Service manifests live in that cluster's repo at `platform/breakpilot-<name>/services/...`.
|
||||
|
||||
```
|
||||
vm-edge (prod, m2.small 8 GB, public IP)
|
||||
vm-edge (cluster: breakpilot-edge, prod, m2.small 8 GB, public IP)
|
||||
├── orca-proxy (Orca-managed; wildcard TLS terminator)
|
||||
├── powerdns-auth (Orca-managed; authoritative DNS for yourplatform.com)
|
||||
├── keycloak-26 (Orca-managed; JVM, ~1.5 GB heap)
|
||||
├── powerdns-auth (Orca-managed; authoritative DNS for breakpilot.com)
|
||||
├── keycloak-26 (Orca-managed; JVM, ~1.5 GB heap; repurposed from CERTifAI KC at M2.1)
|
||||
├── postgres-keycloak (Orca-managed; dedicated PG instance for Keycloak only)
|
||||
├── infisical (Orca-managed)
|
||||
├── postgres-infisical (Orca-managed; dedicated PG instance for Infisical only)
|
||||
├── redis-infisical (Orca-managed; ephemeral)
|
||||
└── gitea (Orca-managed; SQLite backend to avoid a third PG)
|
||||
|
||||
vm-control (prod, m2.medium 16 GB)
|
||||
vm-control (cluster: breakpilot-control, prod, m2.medium 16 GB)
|
||||
├── customer-portal (Orca-managed; Next.js)
|
||||
├── tenant-registry (Orca-managed; Go)
|
||||
├── orca-controller (Orca core process; NOT a managed container)
|
||||
@@ -57,9 +61,9 @@ vm-control (prod, m2.medium 16 GB)
|
||||
├── frappe-hd (same bench as ERPNext)
|
||||
├── mariadb (Orca-managed; for ERPNext)
|
||||
├── redis-erpnext (Orca-managed)
|
||||
└── stalwart-mail (Orca-managed; SMTP/IMAP/JMAP on mail.yourplatform.com)
|
||||
└── stalwart-mail (Orca-managed; SMTP/IMAP/JMAP on mail.breakpilot.com)
|
||||
|
||||
vm-data (prod, m2.medium 16 GB)
|
||||
vm-app-prod (cluster: breakpilot-app, prod, m2.medium 16 GB)
|
||||
├── certifai-dashboard (Orca-managed)
|
||||
├── mongodb (Orca-managed)
|
||||
├── litellm (Orca-managed)
|
||||
@@ -70,22 +74,23 @@ vm-data (prod, m2.medium 16 GB)
|
||||
├── qdrant (Orca-managed)
|
||||
└── minio (Orca-managed)
|
||||
|
||||
stage (stage, m2.small 8 GB, public IP)
|
||||
vm-app-stage (cluster: breakpilot-app — SAME cluster as vm-app-prod, stage, m2.small 8 GB, public IP)
|
||||
├── orca-proxy (light; only routes to stage app)
|
||||
├── customer-portal (NEW VERSION under test)
|
||||
├── tenant-registry (NEW VERSION under test, talks to ephemeral PG below)
|
||||
├── tenant-registry (placeholder; stage actually calls PROD tenant-registry — manifest exists for parity, see §5 isolation rules)
|
||||
├── certifai-dashboard (NEW VERSION under test)
|
||||
├── backend-compliance (NEW VERSION under test)
|
||||
├── ai-compliance-sdk (NEW VERSION under test)
|
||||
├── admin-compliance (NEW VERSION under test)
|
||||
├── litellm (light; same image as prod)
|
||||
├── postgres-app-stage (ephemeral; lives entirely on stage VM)
|
||||
├── postgres-app-stage (ephemeral; lives entirely on vm-app-stage)
|
||||
├── mongodb-stage (ephemeral)
|
||||
└── qdrant-stage (ephemeral, tiny corpus)
|
||||
|
||||
Calls OUT to prod:
|
||||
→ auth.yourplatform.com (Keycloak token issuance, under stage client_id)
|
||||
→ mail.yourplatform.com (Stalwart SMTP, recipient filter forces +stage@ only)
|
||||
→ auth.breakpilot.com (PROD Keycloak in breakpilot-edge, under stage client_id, tenant.kind = "stage")
|
||||
→ registry.breakpilot.com (PROD tenant-registry in breakpilot-control, read-only for stage tenants)
|
||||
→ mail.breakpilot.com (PROD Stalwart in breakpilot-control, recipient filter forces +stage@ only)
|
||||
→ Polar SANDBOX webhook URL (NEVER prod Polar)
|
||||
→ no calls to prod Postgres-app, MariaDB, MongoDB
|
||||
```
|
||||
@@ -107,8 +112,8 @@ stage (stage, m2.small 8 GB, public IP)
|
||||
```
|
||||
INTERNET
|
||||
│
|
||||
(yourplatform.com — authoritative on vm-edge PowerDNS;
|
||||
stage.yourplatform.com — authoritative same zone)
|
||||
(breakpilot.com — authoritative on vm-edge PowerDNS;
|
||||
stage.breakpilot.com — authoritative same zone)
|
||||
│
|
||||
┌─────────────┴─────────────┐
|
||||
│ │
|
||||
@@ -129,7 +134,7 @@ stage (stage, m2.small 8 GB, public IP)
|
||||
┌────────┴─────────┐
|
||||
│ │
|
||||
┌──────▼───────┐ ┌───────▼──────┐
|
||||
│ vm-control │ │ vm-data │
|
||||
│ vm-control │ │ vm-app-prod │
|
||||
│ │ │ │
|
||||
│ portal │ │ certifai │
|
||||
│ tenant-reg │ │ mongodb │
|
||||
@@ -143,15 +148,15 @@ stage (stage, m2.small 8 GB, public IP)
|
||||
└──────────────┘
|
||||
|
||||
Orca-Proxy routing (vm-edge, by Host header):
|
||||
auth.yourplatform.com → 127.0.0.1:8443 (Keycloak, local on vm-edge)
|
||||
erp.yourplatform.com → vm-control:8000 (ERPNext) [allowlist: our IPs only]
|
||||
git.yourplatform.com → vm-edge:3000 (Gitea, local) [allowlist: our IPs only]
|
||||
mail.yourplatform.com → vm-control:587 (Stalwart submission) [allowlist: VM internal only]
|
||||
ns1.yourplatform.com → 127.0.0.1:53 (PowerDNS, local)
|
||||
*.yourplatform.com → vm-control:3000 (customer portal)
|
||||
auth.breakpilot.com → 127.0.0.1:8443 (Keycloak, local on vm-edge)
|
||||
erp.breakpilot.com → vm-control:8000 (ERPNext) [allowlist: our IPs only]
|
||||
git.breakpilot.com → vm-edge:3000 (Gitea, local) [allowlist: our IPs only]
|
||||
mail.breakpilot.com → vm-control:587 (Stalwart submission) [allowlist: VM internal only]
|
||||
ns1.breakpilot.com → 127.0.0.1:53 (PowerDNS, local)
|
||||
*.breakpilot.com → vm-control:3000 (customer portal)
|
||||
|
||||
Orca-Proxy routing (stage, by Host header):
|
||||
*.stage.yourplatform.com → 127.0.0.1:3000 (stage portal — all subdomains route here)
|
||||
*.stage.breakpilot.com → 127.0.0.1:3000 (stage portal — all subdomains route here)
|
||||
```
|
||||
|
||||
---
|
||||
@@ -166,7 +171,7 @@ Block volumes (Ceph 3x replicated, €0.10/GiB/mo) mounted to each VM.
|
||||
├──────────────┼───────────────────────────────────────────┼─────────┼─────────────────────┤
|
||||
│ vm-edge │ pg-keycloak + pg-infisical + Gitea repos │ +50 GB │ Slow │
|
||||
│ vm-control │ MariaDB (ERPNext) + Stalwart mail spool │ +250 GB │ Medium │
|
||||
│ vm-data │ MongoDB + pg-app + Qdrant + MinIO │ +500 GB │ Fast (scales w/ N) │
|
||||
│ vm-app-prod │ MongoDB + pg-app + Qdrant + MinIO │ +500 GB │ Fast (scales w/ N) │
|
||||
│ stage │ pg-stage + mongo-stage + qdrant-stage │ +50 GB │ Resets per release │
|
||||
└──────────────┴───────────────────────────────────────────┴─────────┴─────────────────────┘
|
||||
|
||||
@@ -200,10 +205,10 @@ All backups ship to **SysEleven Object Storage** (S3-compatible, geo-redundant D
|
||||
│ Infisical store │ encrypted → S3 │ Daily │ 30 days │ Infra Plane │
|
||||
│ MariaDB (vm-control) │ mysqldump → S3 │ Every 6h │ 30 days │ Control Plane │
|
||||
│ Stalwart queue/store │ tar → S3 │ Daily │ 7 days │ Control Plane │
|
||||
│ pg-app (vm-data) │ pg_dump → S3-geo │ Every 6h │ 30 days │ Data Plane (owns RPO)│
|
||||
│ MongoDB (vm-data) │ mongodump → S3 │ Daily │ 30 days │ Data Plane │
|
||||
│ MinIO (vm-data) │ mc mirror → S3 │ Daily │ 90 days │ Data Plane │
|
||||
│ Qdrant (vm-data) │ API snap → S3 │ Daily │ 14 days │ Data Plane (rebuild) │
|
||||
│ pg-app (vm-app-prod) │ pg_dump → S3-geo │ Every 6h │ 30 days │ App Plane (owns RPO) │
|
||||
│ MongoDB (vm-app-prod) │ mongodump → S3 │ Daily │ 30 days │ App Plane │
|
||||
│ MinIO (vm-app-prod) │ mc mirror → S3 │ Daily │ 90 days │ App Plane │
|
||||
│ Qdrant (vm-app-prod) │ API snap → S3 │ Daily │ 14 days │ App Plane (rebuild) │
|
||||
│ stage * │ no backup │ — │ — │ — (ephemeral) │
|
||||
│ Orca config (IaC) │ Gitea (VCS) │ On commit │ Forever │ Infra Plane │
|
||||
└───────────────────────┴──────────────────┴────────────┴────────────┴──────────────────────┘
|
||||
@@ -254,11 +259,11 @@ STAGE_ISOLATION— stage tenant cannot mutate any prod data; reads-only against
|
||||
|
||||
### Plane ownership of constraints
|
||||
|
||||
Even though planes now share VMs, the **ownership model is unchanged** — the plane that owns a constraint owns it regardless of which VM hosts the service. The Infra Plane (now collapsed onto vm-edge alongside the Identity plane) still mechanically enforces backup, IaC, secrets, and network constraints.
|
||||
Planes are the **SLA + ownership abstraction**; clusters are the **operational unit**. The plane that owns a constraint owns it regardless of which cluster hosts the service. Identity and Infra are co-tenant in the `breakpilot-edge` cluster on `vm-edge`; they remain distinct planes for purposes of SLA + ownership accounting.
|
||||
|
||||
```
|
||||
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
||||
║ IDENTITY PLANE (on vm-edge) ║
|
||||
║ IDENTITY PLANE (cluster: breakpilot-edge, on vm-edge) ║
|
||||
║ ║
|
||||
║ Owns / defines: ║
|
||||
║ AVAILABILITY — must be ≥ 99.5% (root dep for everything) ║
|
||||
@@ -268,11 +273,16 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
||||
║ STAGE_ISOLATION— rate-limits stage_client_id; rejects stage JWTs in prod audiences ║
|
||||
║ ║
|
||||
║ Co-tenant note: shares vm-edge with Infra Plane services. JVM heap pinned to 1.5 GB ║
|
||||
║ in Orca manifest so it cannot starve PowerDNS / Infisical. ║
|
||||
║ in Orca manifest (-Xmx1500m) so it cannot starve PowerDNS / Infisical. Escape hatch ║
|
||||
║ if the heap fight shows up: flip placement.node to a new vm-edge-identity VM inside ║
|
||||
║ the SAME breakpilot-edge cluster — no schema migration, just a container move. ║
|
||||
║ ║
|
||||
║ KC instance is REPURPOSED from the existing CERTifAI Keycloak at M2.1 (realm export ║
|
||||
║ + user database move; hostname swap), not stood up fresh. ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
||||
║ CONTROL PLANE (on vm-control) ║
|
||||
║ CONTROL PLANE (cluster: breakpilot-control, on vm-control) ║
|
||||
║ ║
|
||||
║ Owns / defines: ║
|
||||
║ RPO (tenant) — tenant registry & compliance schemas RPO ≤ 6h ║
|
||||
@@ -288,10 +298,13 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
||||
║ mariadb: 3 GB memory cap ║
|
||||
║ stalwart: 1 GB memory cap ║
|
||||
║ tenant-registry: 500 MB ║
|
||||
║ ║
|
||||
║ Stage callers: vm-app-stage in breakpilot-app calls THIS tenant-registry under ║
|
||||
║ tenant.kind = "stage" (read-only for stage tenants). No duplicated tenant-registry. ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
||||
║ DATA PLANE (on vm-data) ║
|
||||
║ APP PLANE (was "Data Plane"; cluster: breakpilot-app, on vm-app-prod + vm-app-stage) ║
|
||||
║ ║
|
||||
║ Owns / defines: ║
|
||||
║ DATA_RESIDENCY — all customer data (MongoDB, pg-app, MinIO) must stay EU ║
|
||||
@@ -300,12 +313,17 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
||||
║ AUDIT_TRAIL — product-level actions ║
|
||||
║ AVAILABILITY — CERTifAI ≥ 99.5%; compliance ≥ 99.5% ║
|
||||
║ ║
|
||||
║ Co-tenant note: this VM is the SCALE driver. When vm-data hits 80% RAM, bump flavor ║
|
||||
║ Co-tenant note: vm-app-prod is the SCALE driver. When it hits 80% RAM, bump flavor ║
|
||||
║ (m2.medium → m2.large → m2.xlarge). See §13 Growth Trajectory. ║
|
||||
║ ║
|
||||
║ Stage VM (vm-app-stage) lives in the SAME cluster — same config, different physical ║
|
||||
║ workload. Stage authenticates against the prod Identity plane (breakpilot-edge KC) ║
|
||||
║ and reads tenant config from the prod Control plane (breakpilot-control ║
|
||||
║ tenant-registry) — never duplicates identity or control. ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
||||
║ INFRA PLANE (on vm-edge, alongside Identity) ║
|
||||
║ INFRA PLANE (cluster: breakpilot-edge, on vm-edge, alongside Identity) ║
|
||||
║ ║
|
||||
║ Owns / enforces ALL of: ║
|
||||
║ BACKUP — executes all backup jobs (pg_dump, mongodump, mc mirror) ║
|
||||
@@ -317,6 +335,9 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
||||
║ AVAILABILITY — Orca restart policies, health checks ║
|
||||
║ COLD_START — enforces startup ordering (see §10 Scenario F) ║
|
||||
║ STAGE_ISOLATION— Infisical secret-path scoping for stage_app identity ║
|
||||
║ ║
|
||||
║ Per-cluster note: each of the three cluster repos has its own [backup] block + S3 ║
|
||||
║ bucket. A runaway backup on breakpilot-app cannot fill breakpilot-edge's bucket. ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
||||
```
|
||||
|
||||
@@ -339,14 +360,14 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
||||
│ Frappe HD │ 99% │ 60 min │ 24h │ vm-control │
|
||||
│ MariaDB │ 99.5% │ 20 min │ 6h │ vm-control │
|
||||
│ Stalwart Mail │ 99% │ 60 min │ 24h │ vm-control │
|
||||
│ CERTifAI │ 99.5% │ 10 min │ 24h │ vm-data │
|
||||
│ MongoDB │ 99.5% │ 20 min │ 24h │ vm-data │
|
||||
│ LiteLLM │ 99% │ 5 min │ N/A │ vm-data │
|
||||
│ backend-compliance │ 99.5% │ 10 min │ 6h │ vm-data │
|
||||
│ ai-compliance-sdk │ 99.5% │ 10 min │ 6h │ vm-data │
|
||||
│ pg-app │ 99.9% │ 20 min │ 6h │ vm-data (SPOF — RISK-1) │
|
||||
│ MinIO │ 99.5% │ 30 min │ 24h │ vm-data │
|
||||
│ Qdrant │ 99% │ 2h │ 24h │ vm-data (rebuildable) │
|
||||
│ CERTifAI │ 99.5% │ 10 min │ 24h │ vm-app-prod │
|
||||
│ MongoDB │ 99.5% │ 20 min │ 24h │ vm-app-prod │
|
||||
│ LiteLLM │ 99% │ 5 min │ N/A │ vm-app-prod │
|
||||
│ backend-compliance │ 99.5% │ 10 min │ 6h │ vm-app-prod │
|
||||
│ ai-compliance-sdk │ 99.5% │ 10 min │ 6h │ vm-app-prod │
|
||||
│ pg-app │ 99.9% │ 20 min │ 6h │ vm-app-prod (SPOF — RISK-1) │
|
||||
│ MinIO │ 99.5% │ 30 min │ 24h │ vm-app-prod │
|
||||
│ Qdrant │ 99% │ 2h │ 24h │ vm-app-prod (rebuildable) │
|
||||
│ stage (any service) │ 95% │ best ef.│ N/A │ stage (ephemeral; no SLA) │
|
||||
└───────────────────────┴──────────────┴─────────┴─────────┴────────────────────────────────┘
|
||||
```
|
||||
@@ -433,13 +454,13 @@ Arrows = "requires to function." Dashed = soft (degrades, doesn't fail).
|
||||
│ │ mariadb + redis-erp ──► erpnext + frappe-hd │ │
|
||||
│ │ (intra) ─────────────► stalwart │ │
|
||||
│ │ ──────────────────────► customer-portal │ │
|
||||
│ │ ──────────────────────► tenant-registry ──► pg-app (vm-data)│ │
|
||||
│ │ ──────────────────────► tenant-registry ──► pg-app (vm-app-prod)│ │
|
||||
│ └────────────────────────────────────────────────────────────┘ │
|
||||
│ │ tenant-registry API │
|
||||
└────────────────────────────┼─────────────────────────────────────┘
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ vm-data (Data) │
|
||||
│ vm-app-prod (Data) │
|
||||
│ ┌────────────────────────────────────────────────────────────┐ │
|
||||
│ │ mongodb ───► certifai ◄── (vm-edge JWKS, vm-edge secrets) │ │
|
||||
│ │ litellm ───► certifai, ai-compliance-sdk │ │
|
||||
@@ -470,9 +491,9 @@ Arrows = "requires to function." Dashed = soft (degrades, doesn't fail).
|
||||
├──► keycloak (vm-edge) ──► pg-keycloak (intra-VM)
|
||||
│
|
||||
└──► customer-portal (vm-control)
|
||||
├──► tenant-registry (vm-control) ──► pg-app (vm-data)
|
||||
├──► certifai (vm-data) ──► mongodb (intra-VM)
|
||||
└──► backend-compliance (vm-data) ──► pg-app (intra-VM)
|
||||
├──► tenant-registry (vm-control) ──► pg-app (vm-app-prod)
|
||||
├──► certifai (vm-app-prod) ──► mongodb (intra-VM)
|
||||
└──► backend-compliance (vm-app-prod) ──► pg-app (intra-VM)
|
||||
──► ai-sdk ──► qdrant + minio
|
||||
──► litellm ──► [external AI APIs]
|
||||
```
|
||||
@@ -515,7 +536,7 @@ Impact: customer-portal: DOWN → /[slug]/* all return 503
|
||||
ERPNext + Frappe HD: DOWN → we cannot create sales orders or read tickets
|
||||
Stalwart: DOWN → no outbound emails (trial nudges, exports, ticket replies)
|
||||
MariaDB: DOWN → ERPNext queries fail; backups paused
|
||||
Products (CERTifAI, compliance): UNAFFECTED (on vm-data, JWTs still validate)
|
||||
Products (CERTifAI, compliance): UNAFFECTED (on vm-app-prod, JWTs still validate)
|
||||
Existing logged-in users: can use products directly via product subdomain
|
||||
IF they bookmark it; portal home is 503.
|
||||
Cascade: T+0: portal 503; new tenant onboarding blocked (registry down)
|
||||
@@ -532,7 +553,7 @@ Cost of fix at Tier B/C: split vm-control → vm-portal + vm-ops (ERPNext)
|
||||
— €64/mo extra at m2.small
|
||||
```
|
||||
|
||||
### Scenario C — vm-data fails
|
||||
### Scenario C — vm-app-prod fails
|
||||
|
||||
```
|
||||
Impact: tenant-registry queries: FAIL (pg-app down) → portal returns 503 for tenant lookup
|
||||
@@ -543,7 +564,7 @@ Impact: tenant-registry queries: FAIL (pg-app down) → portal returns 503 fo
|
||||
Cascade: T+0: products down; portal degraded
|
||||
T+15m: support tickets pile up
|
||||
Note: prod is partial — users see error pages but ERPNext + auth still work
|
||||
Recovery: Restart vm-data containers. If pg-app corrupt: restore from pg_dump (RPO 6h).
|
||||
Recovery: Restart vm-app-prod containers. If pg-app corrupt: restore from pg_dump (RPO 6h).
|
||||
RTO target: 20 min
|
||||
Mitigation: This is the SCALE-event VM. RISK-1 below makes this the worst SPOF:
|
||||
one pg-app instance owns tenant_registry + compliance schemas.
|
||||
@@ -561,11 +582,11 @@ Impact: CERTifAI: AI features fail (summarization, chat completion).
|
||||
Compliance CRUD: UNAFFECTED.
|
||||
Cascade: Soft degradation only. Products show "AI features temporarily unavailable" banner.
|
||||
Deadlock: None.
|
||||
Recovery: Restart LiteLLM on vm-data (stateless, ~30s).
|
||||
Recovery: Restart LiteLLM on vm-app-prod (stateless, ~30s).
|
||||
Severity: MEDIUM — graceful degradation by design
|
||||
```
|
||||
|
||||
### Scenario E — Stage VM compromised or buggy
|
||||
### Scenario E — vm-app-stage compromised or buggy
|
||||
|
||||
```
|
||||
Impact: On stage itself: stage portal serves bad data; stage testers see errors.
|
||||
@@ -587,7 +608,7 @@ Severity: LOW (in prod) / HIGH (on stage, but stage SLA is 95%)
|
||||
Three VMs boot at once. Services must start in dependency order or services
|
||||
crash-loop until their deps are ready.
|
||||
|
||||
DEADLOCK RISK: vm-control services (portal, tenant-registry) start before vm-data
|
||||
DEADLOCK RISK: vm-control services (portal, tenant-registry) start before vm-app-prod
|
||||
services (pg-app, certifai, compliance). They'll crash-loop ~2-5min
|
||||
with backoff retries.
|
||||
Same for ERPNext on vm-control trying to reach Keycloak on vm-edge.
|
||||
@@ -598,7 +619,7 @@ RESOLUTION: Orca enforces cross-VM startup ordering via health-check dependencie
|
||||
|
||||
Required cold start sequence:
|
||||
|
||||
Phase 0 — Data roots on vm-data (parallel):
|
||||
Phase 0 — Data roots on vm-app-prod (parallel):
|
||||
pg-app, mongodb, qdrant, minio
|
||||
Phase 0 — Data roots on vm-control (parallel):
|
||||
mariadb, redis-erpnext
|
||||
@@ -613,7 +634,7 @@ Required cold start sequence:
|
||||
keycloak (needs: pg-keycloak [Phase 0], infisical [Phase 1])
|
||||
gitea (needs: sqlite; ready from Phase 0)
|
||||
|
||||
Phase 3 — Control on vm-control + Data services on vm-data (parallel):
|
||||
Phase 3 — Control on vm-control + Data services on vm-app-prod (parallel):
|
||||
tenant-registry (needs: keycloak [Phase 2], pg-app [Phase 0, remote])
|
||||
erpnext + frappe-hd (needs: mariadb, redis-erpnext [Phase 0], keycloak [Phase 2])
|
||||
stalwart (needs: infisical [Phase 1])
|
||||
@@ -642,7 +663,7 @@ Impact: Portal cannot resolve tenant from subdomain → /[slug]/* all 503
|
||||
Cascade: New logins degraded.
|
||||
Existing sessions continue.
|
||||
Deadlock: None.
|
||||
Recovery: Restart tenant-registry on vm-control. pg-app on vm-data must be healthy.
|
||||
Recovery: Restart tenant-registry on vm-control. pg-app on vm-app-prod must be healthy.
|
||||
RTO target: ≤ 60s
|
||||
Mitigation: Portal caches slug → tenant mapping with 60s TTL.
|
||||
Short outage invisible to customers.
|
||||
@@ -683,7 +704,7 @@ stage-app │ │ │ │ │ │ │
|
||||
## 12. Open Infrastructure Risks (Priority Order)
|
||||
|
||||
```
|
||||
RISK-1 pg-app (vm-data) is a single instance serving tenant_registry + compliance schemas.
|
||||
RISK-1 pg-app (vm-app-prod) is a single instance serving tenant_registry + compliance schemas.
|
||||
One crash blocks portal AND compliance product simultaneously.
|
||||
→ Mitigation: split into pg-registry + pg-compliance at Tier B (200 customers).
|
||||
Move pg-registry to its own DBaaS PostgreSQL cluster (€213/mo).
|
||||
@@ -738,10 +759,10 @@ The locked 4-VM topology is right for 5–~200 customers. Past that, expect to a
|
||||
```
|
||||
Tier A (5–200 cust): 4 VMs as locked €192/mo compute (36M upfront)
|
||||
↓
|
||||
Tier B (200–500): Bump vm-data m2.med → m2.large +€64/mo
|
||||
Tier B (200–500): Bump vm-app-prod m2.med → m2.large +€64/mo
|
||||
Add cold-standby vm-edge-spare +€0 (idle, paid only on swap)
|
||||
↓
|
||||
Tier C (500–1000): Split vm-data: vm-data + vm-data-db +€64/mo
|
||||
Tier C (500–1000): Split vm-app-prod: vm-app-prod + vm-app-prod-db +€64/mo
|
||||
(postgres-app moves to its own VM, or DBaaS cluster +€213/mo)
|
||||
Split vm-control: vm-control + vm-ops +€64/mo
|
||||
(ERPNext + MariaDB + Stalwart move to vm-ops)
|
||||
@@ -749,7 +770,7 @@ Tier C (500–1000): Split vm-data: vm-data + vm-data-db +€64/mo
|
||||
Tier D (1000–2000): Split vm-edge: vm-edge + vm-identity + vm-secrets +€96/mo
|
||||
HA Keycloak active-passive on 2× vm-identity +€32/mo
|
||||
Octavia Load Balancer Double Instance +€58/mo
|
||||
vm-data m2.large → m2.xlarge or 2× +€128–256/mo
|
||||
vm-app-prod m2.large → m2.xlarge or 2× +€128–256/mo
|
||||
↓
|
||||
Final topology ≈ 8 prod VMs + DBaaS
|
||||
```
|
||||
|
||||
+47
-33
@@ -1,10 +1,12 @@
|
||||
# Platform Architecture — B2B Customer Portal
|
||||
**Status:** Design Draft
|
||||
**Authors:** Sharang, Benjamin
|
||||
**Date:** 2026-05-11
|
||||
**Date:** 2026-05-11 (cluster split: 2026-06-30)
|
||||
|
||||
---
|
||||
|
||||
> **2026-06-30 cluster-split decision.** The four-plane vocabulary in this document still applies as the **SLA + ownership abstraction**. What changed is the operational shape: planes are now hosted by **three Orca clusters** (one cluster per plane, with Identity and Infra co-tenant in the same cluster on a single VM). The "Data plane" is renamed **App plane** to reflect that it carries product workloads, not just data stores. Cluster repos: `platform/breakpilot-edge` (Identity + Infra), `platform/breakpilot-control` (Control), `platform/breakpilot-app` (App, with prod + stage VMs). Multi-VM rollout is gated on the legal entity being established. See `INFRASTRUCTURE.md §1`, `IMPLEMENTATION_PLAN.md §1.1 / M1.2`, and `platform/orca-platform/clusters/README.md` for the operational details.
|
||||
|
||||
## 1. Vision
|
||||
|
||||
We sell CERTifAI and breakpilot-compliance as modular B2B building blocks. Customers buy one or both and operate them inside a unified customer portal — without needing to understand that they are separate products under the hood.
|
||||
@@ -28,24 +30,32 @@ Out of scope: breakpilot-dataroom, breakpilot-lehrer, breakpilot-pitch-deck.
|
||||
|
||||
## 3. The Four Planes
|
||||
|
||||
Four logical planes — but only **three Orca clusters**. Identity and Infra are co-tenant in the `breakpilot-edge` cluster on `vm-edge` per the 2026-06-30 cluster-split decision. The plane abstraction owns SLA and audit responsibilities; the cluster abstraction owns operational concerns (failure domain, backup policy, CI scope, IaC repo).
|
||||
|
||||
```
|
||||
╔══════════════════════════════════════════════════════════════════╗
|
||||
║ PLANE 1 — IDENTITY (logical root, all auth flows through here) ║
|
||||
╚══════════════════════════════════════════════════════════════════╝
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ PLANE 1 — IDENTITY cluster: breakpilot-edge VM: vm-edge ║
|
||||
║ (logical root, all auth flows through here) ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
↓ JWT
|
||||
╔══════════════════════════════════════════════════════════════════╗
|
||||
║ PLANE 2 — CONTROL (portal + ERPNext + tenant registry) ║
|
||||
╚══════════════════════════════════════════════════════════════════╝
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ PLANE 2 — CONTROL cluster: breakpilot-control VM: vm-control ║
|
||||
║ (portal + ERPNext + tenant registry) ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
↓ tenant-scoped API calls
|
||||
╔══════════════════════════════════════════════════════════════════╗
|
||||
║ PLANE 3 — DATA (CERTifAI + breakpilot-compliance) ║
|
||||
╚══════════════════════════════════════════════════════════════════╝
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ PLANE 3 — APP (was DATA) cluster: breakpilot-app VMs: vm-app-prod ║
|
||||
║ (CERTifAI + breakpilot-compliance + data stores) + vm-app-stage║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
↓ everything runs on
|
||||
╔══════════════════════════════════════════════════════════════════╗
|
||||
║ PLANE 4 — INFRA (Orca + VMs + Gitea + Infisical + LiteLLM) ║
|
||||
╚══════════════════════════════════════════════════════════════════╝
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ PLANE 4 — INFRA cluster: breakpilot-edge VM: vm-edge (co-tenant) ║
|
||||
║ (Orca + Gitea + Infisical + PowerDNS + Orca-Proxy) ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
```
|
||||
|
||||
Stage and prod share the same App cluster on different VMs (`vm-app-stage` and `vm-app-prod`). Stage authenticates via the prod Identity plane (`auth.breakpilot.com`) and reads tenant config from the prod Control plane (`tenant-registry`) under `tenant.kind = "stage"` — Identity and Control are **not** duplicated for stage.
|
||||
|
||||
---
|
||||
|
||||
## 4. Plane 1 — Identity
|
||||
@@ -94,7 +104,7 @@ org_roles — [IT_ADMIN, USER, ...] roles within their org
|
||||
realm_roles — [customer] | [BREAKPILOT_ADMIN] | [SUPPORT_ENGINEER] | [SALES_REP]
|
||||
products — [certifai, compliance] entitlements (injected by protocol mapper)
|
||||
plan — starter | professional | enterprise
|
||||
iss — https://auth.yourplatform.com/realms/breakpilot-prod
|
||||
iss — https://auth.breakpilot.com/realms/breakpilot-prod
|
||||
```
|
||||
|
||||
The `products` and `plan` claims are added by a Keycloak **protocol mapper** that reads live entitlements from the Tenant Registry at token issuance. Products do not need to call back to the registry on every request.
|
||||
@@ -108,12 +118,12 @@ Three distinct services. Clear separation of responsibility.
|
||||
### 5a. Customer Portal
|
||||
|
||||
**Technology:** Next.js 15 (new service)
|
||||
**Deployed at:** `*.yourplatform.com` via Orca-Proxy wildcard routing
|
||||
**Deployed at:** `*.breakpilot.com` via Orca-Proxy wildcard routing
|
||||
|
||||
The front door for all customers and for us. Owns no business logic — it is a routing, auth, and UI layer.
|
||||
|
||||
**Subdomain routing:**
|
||||
- DNS wildcard `*.yourplatform.com` → Orca-Proxy
|
||||
- DNS wildcard `*.breakpilot.com` → Orca-Proxy
|
||||
- Orca-Proxy reads `Host` header → routes all traffic to the portal container
|
||||
- Portal reads `Host` → extracts tenant slug → looks up Tenant Registry
|
||||
|
||||
@@ -190,7 +200,7 @@ The front door for all customers and for us. Owns no business logic — it is a
|
||||
### 5b. ERPNext
|
||||
|
||||
**Technology:** Frappe + ERPNext (self-hosted via Orca)
|
||||
**Access:** `erp.yourplatform.com` — us only (IP-restricted at Orca-Proxy)
|
||||
**Access:** `erp.breakpilot.com` — us only (IP-restricted at Orca-Proxy)
|
||||
**Auth:** Keycloak OIDC — we log in with our existing accounts, no separate password
|
||||
|
||||
ERPNext is our **business operations backbone**. We do not build CRM, invoicing, or HR — we configure ERPNext for these.
|
||||
@@ -262,7 +272,7 @@ api_keys portal-owned. tenant_id, product, scopes, name,
|
||||
|
||||
### 5d. Demo Tenant (Shared)
|
||||
|
||||
**Slug:** `demo` — reachable at `demo.yourplatform.com`
|
||||
**Slug:** `demo` — reachable at `demo.breakpilot.com`
|
||||
**Status:** `demo` (never transitions; never billed)
|
||||
**Owner:** us (`BREAKPILOT_ADMIN` curates content; `SALES_REP` reads + logs in)
|
||||
|
||||
@@ -299,13 +309,15 @@ all real-tenant flows work otherwise same flows, same code paths
|
||||
|
||||
**Support flow:**
|
||||
- Customer submits ticket via `/[slug]/support/` (Frappe HD customer portal, embedded or linked)
|
||||
- Agent (us) triages in Frappe HD agent UI at `erp.yourplatform.com`
|
||||
- Agent (us) triages in Frappe HD agent UI at `erp.breakpilot.com`
|
||||
- If technical: agent clicks "Escalate to Engineering" → Frappe server script creates a Gitea issue in the relevant repo via Gitea REST API → issue URL stored on ticket
|
||||
- When Gitea issue is closed → Gitea webhook → Frappe HD → ticket marked "Resolved"
|
||||
|
||||
---
|
||||
|
||||
## 6. Plane 3 — Data
|
||||
## 6. Plane 3 — App (was "Data")
|
||||
|
||||
Renamed 2026-06-30. The plane carries product workloads end-to-end (dashboards, APIs, AI services) as well as the data stores those products own — "App" reflects the full surface, not just the storage tier. SLA targets are unchanged (CERTifAI ≥ 99.5%, compliance ≥ 99.5%, RPO ≤ 6h–24h per data class). Lives in the `breakpilot-app` cluster on `vm-app-prod` (prod) and `vm-app-stage` (stage).
|
||||
|
||||
### CERTifAI
|
||||
|
||||
@@ -334,18 +346,20 @@ GDPR and AI-Act compliance automation platform. After updates, tenant identity c
|
||||
|
||||
## 7. Plane 4 — Infra
|
||||
|
||||
**Orchestration:** Orca manages all containers on Hetzner VMs
|
||||
Co-tenant with the **Identity plane** in the `breakpilot-edge` cluster on `vm-edge`. Single-VM core per the 2026-06-30 cluster-split decision. Keycloak JVM heap is pinned (`-Xmx1500m`) so it cannot starve PowerDNS / Infisical. Escape hatch if the heap fight shows up post-launch: peel Keycloak onto its own VM inside the same cluster — no schema migration, just a `placement.node` flip.
|
||||
|
||||
**Orchestration:** Orca manages all containers on SysEleven VMs. Per-node ingress (every node binds 80/443 and runs its own ACME), one Orca cluster per plane (3 clusters total), one IaC repo per cluster (`platform/breakpilot-{edge,control,app}`).
|
||||
**Secrets:** Infisical — every service has a machine identity, pulls its own secrets at startup
|
||||
**CI/CD:** Gitea Actions → Docker build → push to private registry → Orca redeploy webhook
|
||||
**Routing:** Orca-Proxy handles all TLS termination and subdomain routing
|
||||
**CI/CD:** Gitea Actions → Docker build → push to private registry → Orca redeploy webhook (one CI/release cadence per cluster repo, plus per-product repos for the deployed services)
|
||||
**Routing:** Orca-Proxy handles TLS termination and subdomain routing on each node (no central proxy SPOF)
|
||||
|
||||
```
|
||||
Orca-Proxy routing table:
|
||||
auth.yourplatform.com → Keycloak
|
||||
erp.yourplatform.com → ERPNext + Frappe HD (IP-restricted)
|
||||
git.yourplatform.com → Gitea
|
||||
secrets.yourplatform.com → Infisical (IP-restricted)
|
||||
*.yourplatform.com → customer-portal (wildcard, Host → tenant)
|
||||
auth.breakpilot.com → Keycloak
|
||||
erp.breakpilot.com → ERPNext + Frappe HD (IP-restricted)
|
||||
git.breakpilot.com → Gitea
|
||||
secrets.breakpilot.com → Infisical (IP-restricted)
|
||||
*.breakpilot.com → customer-portal (wildcard, Host → tenant)
|
||||
```
|
||||
|
||||
**Services managed by Orca:**
|
||||
@@ -436,7 +450,7 @@ Data Stores
|
||||
```
|
||||
USER ORCA-PROXY PORTAL KEYCLOAK CUSTOMER IdP
|
||||
│ │ │ │ │
|
||||
│ acme.yourplatform.com │ │ │ │
|
||||
│ acme.breakpilot.com │ │ │ │
|
||||
│───────────────────────►│ │ │ │
|
||||
│ │ Host=acme.* │ │ │
|
||||
│ │───────────────►│ │ │
|
||||
@@ -458,7 +472,7 @@ Data Stores
|
||||
```
|
||||
USER PORTAL KEYCLOAK
|
||||
│ │ │
|
||||
│ acme.yourplatform│ │
|
||||
│ acme.breakpilot │ │
|
||||
│──────────────────►│ │
|
||||
│ │ redirect + PKCE │
|
||||
│ │─────────────────►│
|
||||
@@ -671,7 +685,7 @@ Data Stores
|
||||
│ │ impersonated_by │ │
|
||||
│ │ claim) │ │
|
||||
│ │ │
|
||||
│ new tab: acme.yourplatform.com │ │
|
||||
│ new tab: acme.breakpilot.com │ │
|
||||
│──────────────────────────────────────────────────────────►│
|
||||
│ │ [orange banner] │
|
||||
│ │ Impersonating │
|
||||
@@ -749,7 +763,7 @@ Data Stores
|
||||
│ │ │ │
|
||||
│ open Zoom with prospect, share screen │
|
||||
│ │
|
||||
│ demo.yourplatform.com │
|
||||
│ demo.breakpilot.com │
|
||||
│────────────────────────────────►│ │
|
||||
│ │ │ Host: demo │
|
||||
│ │ │ → slug = demo │
|
||||
@@ -796,7 +810,7 @@ Data Stores
|
||||
```
|
||||
PROSPECT PORTAL TENANT REGISTRY ERPNEXT KEYCLOAK
|
||||
│ │ │ │ │
|
||||
│ yourplatform.com/start │ │ │
|
||||
│ breakpilot.com/start │ │ │
|
||||
│──────────────►│ │ │ │
|
||||
│ form: email, company, password │ │ │
|
||||
│──────────────►│ │ │ │
|
||||
|
||||
+19
-19
@@ -279,7 +279,7 @@ Products that ship custom styling must respect the `theme` attribute and the pre
|
||||
|
||||
```
|
||||
Product publishes a bundle at:
|
||||
https://cdn.yourplatform.com/products/{name}/{version}/element.js
|
||||
https://cdn.breakpilot.com/products/{name}/{version}/element.js
|
||||
|
||||
Portal loads it lazily via dynamic import when the user navigates to /[tenant]/products/{name}.
|
||||
Portal caches the bundle URL per product version (declared in tenant_products.config).
|
||||
@@ -343,12 +343,12 @@ The product ships NO frontend code. The portal renders a generic management UI f
|
||||
│ CODE SAMPLES │
|
||||
│ ──────────────────────────────────────────────────── │
|
||||
│ [curl] [JS] [Python] │
|
||||
│ curl -X POST https://notetaker-api.yourplatform.com/v1 │
|
||||
│ curl -X POST https://notetaker-api.breakpilot.com/v1 │
|
||||
│ -H "Authorization: ApiKey k_xxx" │
|
||||
│ -H "X-Tenant: acme" │
|
||||
│ -d '{...}' │
|
||||
│ │
|
||||
│ DOCS ► developers.yourplatform.com/products/notetaker │
|
||||
│ DOCS ► developers.breakpilot.com/products/notetaker │
|
||||
└──────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
@@ -394,7 +394,7 @@ An MCP (Model Context Protocol) server exposes the product's capabilities as too
|
||||
|
||||
```
|
||||
1. ONE MCP server per product
|
||||
Endpoint: https://mcp.{product}.yourplatform.com (or unified mcp.yourplatform.com/{product})
|
||||
Endpoint: https://mcp.{product}.breakpilot.com (or unified mcp.breakpilot.com/{product})
|
||||
|
||||
2. Authentication via SCOPED API KEY
|
||||
Customer IT Admin generates API key in /[tenant]/settings/api-keys.
|
||||
@@ -438,7 +438,7 @@ Enterprise customers automatically get MCP enabled. Starter/Pro customers see "A
|
||||
|
||||
## 7. Documentation Contract
|
||||
|
||||
A product ships five required documents. They are published at `developers.yourplatform.com/products/{name}/`.
|
||||
A product ships five required documents. They are published at `developers.breakpilot.com/products/{name}/`.
|
||||
|
||||
```
|
||||
1. README What does it do? Value prop in 200 words.
|
||||
@@ -763,16 +763,16 @@ product:
|
||||
vendor: breakpilot # we; future third-parties will use their slug
|
||||
contract_version: "1.0"
|
||||
product_version: "1.4.2"
|
||||
repo: git.yourplatform.com/sharang/certifai
|
||||
repo: git.breakpilot.com/sharang/certifai
|
||||
|
||||
catalog:
|
||||
# Renders in /[tenant]/catalog and /backstage/products
|
||||
category: "AI Infrastructure" # AI Infrastructure | Compliance | Productivity | Security | Data
|
||||
tagline: "GDPR-compliant LLMs without leaving the EU"
|
||||
hero_image: https://cdn.yourplatform.com/products/certifai/hero.png
|
||||
hero_image: https://cdn.breakpilot.com/products/certifai/hero.png
|
||||
screenshots:
|
||||
- https://cdn.yourplatform.com/products/certifai/dashboard.png
|
||||
- https://cdn.yourplatform.com/products/certifai/agents.png
|
||||
- https://cdn.breakpilot.com/products/certifai/dashboard.png
|
||||
- https://cdn.breakpilot.com/products/certifai/agents.png
|
||||
pricing_summary: "From €X/seat/month — included on Professional and Enterprise plans"
|
||||
available_on_plans: [trial, professional, enterprise] # 'trial' opt-in for self-serve
|
||||
trial_days: 14
|
||||
@@ -784,7 +784,7 @@ catalog:
|
||||
|
||||
demo:
|
||||
supported: true # MUST be true unless explicitly waived
|
||||
seed_data_url: https://cdn.yourplatform.com/products/certifai/demo/seed-v3.tar.gz
|
||||
seed_data_url: https://cdn.breakpilot.com/products/certifai/demo/seed-v3.tar.gz
|
||||
reset_endpoint: /v1/tenants/demo/reset # called nightly by portal cron
|
||||
persona_hints: # for sales rep talk track
|
||||
- "GDPR officer at a 200-person SaaS"
|
||||
@@ -807,7 +807,7 @@ identity:
|
||||
frontend:
|
||||
type: interactive # interactive | widget | headless
|
||||
tag: certifai-dashboard
|
||||
bundle_url: https://cdn.yourplatform.com/products/certifai/{version}/element.js
|
||||
bundle_url: https://cdn.breakpilot.com/products/certifai/{version}/element.js
|
||||
bundle_size_kb: 380
|
||||
routes:
|
||||
- path: /
|
||||
@@ -828,7 +828,7 @@ backend:
|
||||
mcp:
|
||||
enabled: true
|
||||
required_plan: enterprise
|
||||
endpoint: https://mcp.yourplatform.com/certifai
|
||||
endpoint: https://mcp.breakpilot.com/certifai
|
||||
tools:
|
||||
- name: list_ai_agents
|
||||
description: "Returns AI agents configured for the tenant"
|
||||
@@ -864,7 +864,7 @@ backup:
|
||||
retention_days: 30
|
||||
|
||||
infra:
|
||||
image: registry.yourplatform.com/certifai-dashboard
|
||||
image: registry.breakpilot.com/certifai-dashboard
|
||||
vm: vm-certifai
|
||||
replicas: 1
|
||||
resource_limits:
|
||||
@@ -909,7 +909,7 @@ The example above shows an `interactive` product. Headless and widget products d
|
||||
frontend:
|
||||
type: widget
|
||||
tag: status-monitor-widget
|
||||
bundle_url: https://cdn.yourplatform.com/products/status/{version}/widget.js
|
||||
bundle_url: https://cdn.breakpilot.com/products/status/{version}/widget.js
|
||||
bundle_size_kb: 38
|
||||
dimensions:
|
||||
width: 400
|
||||
@@ -954,7 +954,7 @@ frontend:
|
||||
- language: curl
|
||||
title: "Create a session"
|
||||
snippet: |
|
||||
curl -X POST https://notetaker-api.yourplatform.com/v1/sessions \
|
||||
curl -X POST https://notetaker-api.breakpilot.com/v1/sessions \
|
||||
-H "Authorization: ApiKey k_xxx" \
|
||||
-H "X-Tenant: acme" \
|
||||
-d '{"audio_url": "...", "language": "en"}'
|
||||
@@ -963,7 +963,7 @@ frontend:
|
||||
snippet: |
|
||||
import requests
|
||||
requests.post(
|
||||
"https://notetaker-api.yourplatform.com/v1/sessions",
|
||||
"https://notetaker-api.breakpilot.com/v1/sessions",
|
||||
headers={"Authorization": "ApiKey k_xxx", "X-Tenant": "acme"},
|
||||
json={"audio_url": "...", "language": "en"},
|
||||
)
|
||||
@@ -986,7 +986,7 @@ Products can call each other directly. Auth is via short-lived service tokens is
|
||||
1. Compliance product needs to list AI agents for an AI Act assessment.
|
||||
|
||||
2. Compliance backend requests a service token:
|
||||
POST https://auth.yourplatform.com/realms/breakpilot-prod/protocol/openid-connect/token
|
||||
POST https://auth.breakpilot.com/realms/breakpilot-prod/protocol/openid-connect/token
|
||||
Body: grant_type=client_credentials
|
||||
client_id=compliance-svc
|
||||
client_secret=<from Infisical>
|
||||
@@ -1135,7 +1135,7 @@ A product is "ready to ship to a customer" when all boxes are ticked.
|
||||
☐ All tools tenant-scoped and audited
|
||||
|
||||
☐ Documentation
|
||||
☐ README published at developers.yourplatform.com/products/{name}
|
||||
☐ README published at developers.breakpilot.com/products/{name}
|
||||
☐ API reference auto-generated and live
|
||||
☐ Integration guide for customer IT
|
||||
☐ Operational runbook for us
|
||||
@@ -1220,7 +1220,7 @@ Effort estimate: 3-5 weeks of focused work
|
||||
```
|
||||
- Design tokens package (@breakpilot/design-tokens) — needs to exist before web components ship
|
||||
- CDN for product bundles — pick provider (Hetzner Object Storage + Cloudflare?)
|
||||
- MCP gateway — single mcp.yourplatform.com vs. per-product subdomains
|
||||
- MCP gateway — single mcp.breakpilot.com vs. per-product subdomains
|
||||
- Third-party manifest signing — defer until first real third-party conversation
|
||||
- Inter-product event bus — explicitly deferred; service tokens cover the use cases for now
|
||||
- Contract testing — automate manifest + openapi validation in Gitea Actions
|
||||
|
||||
@@ -39,8 +39,8 @@ For IaC: list the make targets.}}
|
||||
| Env | URL | How |
|
||||
|---|---|---|
|
||||
| dev | `http://localhost:3000` | `make dev` |
|
||||
| stage | `https://docs.stage.yourplatform.com` | auto on merge to `main` |
|
||||
| prod | `https://docs.yourplatform.com` | manual: tag `vX.Y.Z` + sign-off |
|
||||
| stage | `https://docs.stage.breakpilot.com` | auto on merge to `main` |
|
||||
| prod | `https://docs.breakpilot.com` | manual: tag `vX.Y.Z` + sign-off |
|
||||
|
||||
Rollback: `orca rollout undo docs --env={{env}}`.
|
||||
|
||||
@@ -48,7 +48,7 @@ Rollback: `orca rollout undo docs --env={{env}}`.
|
||||
|
||||
- Traces, logs, metrics: [SigNoz](https://signoz.meghsakha.com) — service name `docs`
|
||||
- Audit events: Tenant Registry `/audit` (Retraced-shape schema)
|
||||
- On-call: `oncall@yourplatform.com` · runbook at `platform/docs/runbooks/docs.md`
|
||||
- On-call: `oncall@breakpilot.com` · runbook at `platform/docs/runbooks/docs.md`
|
||||
|
||||
## Contributing
|
||||
|
||||
|
||||
Reference in New Issue
Block a user