Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5992e4b8e0 | ||
|
|
cfb94d3259 | ||
|
|
1fd03592c0 | ||
|
|
a8593091c2 | ||
|
|
03a5b4846e | ||
|
|
1ed2dcee57 |
+37
-10
@@ -1,6 +1,7 @@
|
|||||||
# CI skeleton (TypeScript shape; no app code yet).
|
# CI skeleton — applies to every repo until per-stack code lands.
|
||||||
# Lights up to commitlint + gitleaks + trivy fs scan. Add lint/test/build jobs
|
# The `shared` job runs on every push + PR; per-stack `test`/`image`/`e2e`
|
||||||
# when this repo grows real package code.
|
# jobs light up automatically when the repo gains go.sum / package.json /
|
||||||
|
# Cargo.toml etc. — see the conditional gates below.
|
||||||
name: ci
|
name: ci
|
||||||
|
|
||||||
on:
|
on:
|
||||||
@@ -18,14 +19,40 @@ jobs:
|
|||||||
|
|
||||||
- name: commitlint (PR only)
|
- name: commitlint (PR only)
|
||||||
if: github.event_name == 'pull_request'
|
if: github.event_name == 'pull_request'
|
||||||
uses: wagoid/commitlint-github-action@v6
|
shell: bash
|
||||||
|
env:
|
||||||
|
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||||
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
pattern='^(feat|fix|docs|chore|refactor|test|perf|build|ci|revert)(\([a-z0-9_/.-]+\))?!?: .{1,72}$'
|
||||||
|
bad=0
|
||||||
|
while IFS= read -r subject; do
|
||||||
|
if ! [[ "$subject" =~ $pattern ]]; then
|
||||||
|
echo "::error::Commit subject does not match Conventional Commits: $subject"
|
||||||
|
bad=$((bad+1))
|
||||||
|
fi
|
||||||
|
done < <(git log --format=%s "${BASE_SHA}..${HEAD_SHA}")
|
||||||
|
if [ "$bad" -gt 0 ]; then
|
||||||
|
echo "::error::$bad commit(s) failed commitlint. See https://www.conventionalcommits.org/"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "commitlint: all commit subjects OK"
|
||||||
|
|
||||||
- name: gitleaks
|
- name: gitleaks
|
||||||
uses: gitleaks/gitleaks-action@v2
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
GL_VERSION=8.18.4
|
||||||
|
curl -fsSL "https://github.com/gitleaks/gitleaks/releases/download/v${GL_VERSION}/gitleaks_${GL_VERSION}_linux_x64.tar.gz" \
|
||||||
|
| tar -xz -C /tmp gitleaks
|
||||||
|
/tmp/gitleaks detect --source . --no-banner --redact --verbose --exit-code 1
|
||||||
|
|
||||||
- name: trivy fs scan
|
- name: trivy fs scan
|
||||||
uses: aquasecurity/trivy-action@master
|
shell: bash
|
||||||
with:
|
run: |
|
||||||
scan-type: fs
|
set -euo pipefail
|
||||||
severity: HIGH,CRITICAL
|
TRIVY_VERSION=0.70.0
|
||||||
exit-code: 1
|
curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \
|
||||||
|
| tar -xz -C /tmp trivy
|
||||||
|
/tmp/trivy fs --severity HIGH,CRITICAL --exit-code 1 --no-progress --skip-dirs node_modules,target,dist .
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ jobs:
|
|||||||
runs-on: docker
|
runs-on: docker
|
||||||
environment:
|
environment:
|
||||||
name: production # Gitea Environments — requires sign-off per branch protection
|
name: production # Gitea Environments — requires sign-off per branch protection
|
||||||
url: https://yourplatform.com
|
url: https://breakpilot.com
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with: { fetch-depth: 0 }
|
with: { fetch-depth: 0 }
|
||||||
@@ -22,7 +22,7 @@ jobs:
|
|||||||
|
|
||||||
- name: verify stage soak (>= 24h on this image)
|
- name: verify stage soak (>= 24h on this image)
|
||||||
run: |
|
run: |
|
||||||
IMG=registry.yourplatform.com/${{ github.event.repository.name }}:env-stage
|
IMG=repo.breakpilot.com/breakpilot/${{ github.event.repository.name }}:env-stage
|
||||||
SOAK_SECONDS=$(orca image-age --env=stage --image $IMG)
|
SOAK_SECONDS=$(orca image-age --env=stage --image $IMG)
|
||||||
if [ "$SOAK_SECONDS" -lt 86400 ]; then
|
if [ "$SOAK_SECONDS" -lt 86400 ]; then
|
||||||
echo "Stage soak only $SOAK_SECONDS s, < 24h. Aborting."
|
echo "Stage soak only $SOAK_SECONDS s, < 24h. Aborting."
|
||||||
@@ -34,12 +34,12 @@ jobs:
|
|||||||
- name: re-tag image as semver + env-prod
|
- name: re-tag image as semver + env-prod
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v3
|
||||||
with:
|
with:
|
||||||
registry: registry.yourplatform.com
|
registry: repo.breakpilot.com
|
||||||
username: ${{ secrets.REGISTRY_USER }}
|
username: ${{ secrets.REGISTRY_USER }}
|
||||||
password: ${{ secrets.REGISTRY_PASS }}
|
password: ${{ secrets.REGISTRY_PASS }}
|
||||||
|
|
||||||
- run: |
|
- run: |
|
||||||
IMG=registry.yourplatform.com/${{ github.event.repository.name }}
|
IMG=repo.breakpilot.com/breakpilot/${{ github.event.repository.name }}
|
||||||
docker pull $IMG:env-stage
|
docker pull $IMG:env-stage
|
||||||
docker tag $IMG:env-stage $IMG:v${{ steps.v.outputs.version }}
|
docker tag $IMG:env-stage $IMG:v${{ steps.v.outputs.version }}
|
||||||
docker tag $IMG:env-stage $IMG:env-prod
|
docker tag $IMG:env-stage $IMG:env-prod
|
||||||
@@ -67,7 +67,7 @@ jobs:
|
|||||||
curl -X POST -H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \
|
curl -X POST -H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-d "$(jq -Rs '{tag_name:"v${{ steps.v.outputs.version }}", name:"v${{ steps.v.outputs.version }}", body:.}' < RELEASE_NOTES.md)" \
|
-d "$(jq -Rs '{tag_name:"v${{ steps.v.outputs.version }}", name:"v${{ steps.v.outputs.version }}", body:.}' < RELEASE_NOTES.md)" \
|
||||||
https://gitea.meghsakha.com/api/v1/repos/${{ github.repository }}/releases
|
https://git.breakpilot.com/api/v1/repos/${{ github.repository }}/releases
|
||||||
|
|
||||||
rollback-on-failure:
|
rollback-on-failure:
|
||||||
needs: promote
|
needs: promote
|
||||||
|
|||||||
+3
-1
@@ -6,12 +6,14 @@ Generated section is appended on release tag via `git-cliff` (see `.gitea/workfl
|
|||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
- chore: smoke PR for M0.2 CI verification
|
-
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
- chore(domain): yourplatform.com → breakpilot.com
|
||||||
-
|
-
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
- ci: rework workflow for Gitea Actions (bash commitlint, inline gitleaks binary, per-stack jobs gated on real code)
|
||||||
-
|
-
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
|
|||||||
+1
-1
@@ -86,4 +86,4 @@ When reviewing, check in this order:
|
|||||||
|
|
||||||
## Questions
|
## Questions
|
||||||
|
|
||||||
`#engineering` channel · `oncall@yourplatform.com` · or open a PR with a `[WIP]` prefix and ask in the description.
|
`#engineering` channel · `oncall@breakpilot.com` · or open a PR with a `[WIP]` prefix and ask in the description.
|
||||||
|
|||||||
+115
-81
@@ -19,18 +19,26 @@ This is the build plan for an AI coding agent (Claude Code, executing PRs agains
|
|||||||
## 1. Cross-cutting conventions (apply to every PR in every repo)
|
## 1. Cross-cutting conventions (apply to every PR in every repo)
|
||||||
|
|
||||||
### 1.1 Repo strategy
|
### 1.1 Repo strategy
|
||||||
Polyrepo under a new Gitea org `gitea.meghsakha.com/platform/`. One repo per deployable unit. Existing product repos stay where they are.
|
Polyrepo under the Gitea org `gitea.meghsakha.com/platform/`. One repo per deployable unit. Existing product repos stay where they are.
|
||||||
|
|
||||||
**Repos to create:**
|
**Infrastructure repos.** The IaC is split by Orca cluster — one cluster per plane, one repo per cluster — per the 2026-06-30 cluster-split decision (see [Plane model memory](../../../../home/sharang/.claude/projects/-home-sharang-workspace/memory/project_breakpilot_plane_model.md) and `INFRASTRUCTURE.md §6`). Until the legal entity is established and SysEleven / Hetzner business contracts can be signed, the cluster manifests live as design-only staging directories under `platform/orca-platform/clusters/`. At migration time each `clusters/<name>/` subdir becomes its own repo:
|
||||||
|
|
||||||
| Repo | Purpose | Created in |
|
| Repo | Purpose | Created in |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `platform/orca-platform` | IaC for VMs, Orca manifests, DNS, TLS, backups | M1.1 |
|
| `platform/orca-platform` | IaC staging — cluster.toml templates, per-cluster service manifests under `clusters/breakpilot-{edge,control,app}/`, overlays, DNS zones, dev-compose | M1.1 |
|
||||||
| `platform/tenant-registry` | Go service: tenant glue, audit, API keys | M4.1 |
|
| `platform/breakpilot-edge` | Identity + Infra cluster (vm-edge): Keycloak, Gitea, Infisical, PowerDNS, Orca-Proxy | M1.2 (split from `orca-platform/clusters/breakpilot-edge/`) |
|
||||||
| `platform/portal` | Next.js 15: customer area + backstage | M5.1 |
|
| `platform/breakpilot-control` | Control cluster (vm-control): portal, tenant-registry, ERPNext, MariaDB, Stalwart, Frappe HD | M1.2 (split from `orca-platform/clusters/breakpilot-control/`) |
|
||||||
| `platform/docs` | Architecture, integration spec, this plan, runbooks | M0.1 |
|
| `platform/breakpilot-app` | App cluster (vm-app-prod + vm-app-stage): CERTifAI, compliance-*, Mongo, MinIO, Qdrant, LiteLLM, pg-app | M1.2 (split from `orca-platform/clusters/breakpilot-app/`) |
|
||||||
| `platform/seed-data` | Demo tenant fixtures per product | M13.1 |
|
|
||||||
| `platform/design-tokens` | CSS variables / fonts (consumed by product web comps) | M5.1 |
|
**Product / business repos:**
|
||||||
|
|
||||||
|
| Repo | Purpose | Created in |
|
||||||
|
|---|---|---|
|
||||||
|
| `platform/tenant-registry` | Go service: tenant glue, audit, API keys | M4.1 |
|
||||||
|
| `platform/portal` | Next.js 15: customer area + backstage | M5.1 |
|
||||||
|
| `platform/docs` | Architecture, integration spec, this plan, runbooks | M0.1 |
|
||||||
|
| `platform/seed-data` | Demo tenant fixtures per product | M13.1 |
|
||||||
|
| `platform/design-tokens` | CSS variables / fonts (consumed by product web comps) | M5.1 |
|
||||||
|
|
||||||
**Existing repos that get changes (no new repos):**
|
**Existing repos that get changes (no new repos):**
|
||||||
- `benjamin_boenisch/certifai` — M6.1 / M6.2 / M6.3
|
- `benjamin_boenisch/certifai` — M6.1 / M6.2 / M6.3
|
||||||
@@ -110,18 +118,26 @@ Per language defaults:
|
|||||||
- **Database migrations** are forward-only and run as an init container before the service starts. Migrations that delete columns require two releases (1: stop writing, 2: drop).
|
- **Database migrations** are forward-only and run as an init container before the service starts. Migrations that delete columns require two releases (1: stop writing, 2: drop).
|
||||||
|
|
||||||
### 1.8 Environments
|
### 1.8 Environments
|
||||||
Three Orca clusters, all on the same hardware until volume justifies separation:
|
Three Orca clusters, **one per plane** (not one per env). Stage and prod live in the same `breakpilot-app` cluster on different VMs:
|
||||||
|
|
||||||
| Env | Cluster name | Purpose | Data | Auto-deploy? |
|
| Cluster | VMs | Plane | Envs hosted |
|
||||||
|---|---|---|---|---|
|
|---|---|---|---|
|
||||||
| dev | local | Developer machine, docker-compose | fixtures | n/a |
|
| `breakpilot-edge` | `vm-edge` (1) | Identity + Infra | shared by stage + prod (single Keycloak realm, `tenant.kind` differentiates) |
|
||||||
| stage | `orca-stage` | Pre-prod validation | seeded demo + synthetic customers | yes (on merge to main) |
|
| `breakpilot-control` | `vm-control` (1) | Control | shared by stage + prod (single tenant-registry, `tenant.kind` differentiates) |
|
||||||
| prod | `orca-prod` | Live customer traffic | real | tag + gate |
|
| `breakpilot-app` | `vm-app-prod`, `vm-app-stage` (2) | App | prod on `vm-app-prod`, stage on `vm-app-stage` |
|
||||||
|
|
||||||
|
| Env | Where it runs | Data | Auto-deploy? |
|
||||||
|
|---|---|---|---|
|
||||||
|
| dev | Developer machine, docker-compose (`platform/orca-platform/dev/`) | fixtures | n/a |
|
||||||
|
| stage | `vm-app-stage` in `breakpilot-app`; calls prod KC + prod tenant-registry under `tenant.kind = "stage"` | seeded demo + synthetic customers | yes (on merge to main) |
|
||||||
|
| prod | `vm-app-prod` in `breakpilot-app` + `vm-control` + `vm-edge` | real | tag + gate |
|
||||||
|
|
||||||
|
Identity and Control are NOT duplicated for stage. Stage workloads authenticate against the prod Keycloak realm with `tenant.kind = "stage"` and read tenant config from the prod `tenant-registry` (read-only for stage tenants) per `INFRASTRUCTURE.md §5`.
|
||||||
|
|
||||||
Domain pattern:
|
Domain pattern:
|
||||||
- dev: `*.localhost` (mkcert)
|
- dev: `*.localhost` (mkcert)
|
||||||
- stage: `*.stage.yourplatform.com`
|
- stage: `*.stage.breakpilot.com`
|
||||||
- prod: `*.yourplatform.com`
|
- prod: `*.breakpilot.com`
|
||||||
|
|
||||||
### 1.9 Observability + audit
|
### 1.9 Observability + audit
|
||||||
- **SigNoz** (already running at `signoz.meghsakha.com`) for traces, logs, metrics. Every service ships OTel SDK from day one.
|
- **SigNoz** (already running at `signoz.meghsakha.com`) for traces, logs, metrics. Every service ships OTel SDK from day one.
|
||||||
@@ -157,6 +173,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
- **Tests:** n/a
|
- **Tests:** n/a
|
||||||
- **Gate:** standard
|
- **Gate:** standard
|
||||||
- **Effort:** S
|
- **Effort:** S
|
||||||
|
- **Note:** the three cluster repos (`platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app`) are NOT created at M0.1 — their service manifests live as design-only directories under `platform/orca-platform/clusters/` until M1.2 splits them.
|
||||||
|
|
||||||
### M0.2 — CI templates + branch protection
|
### M0.2 — CI templates + branch protection
|
||||||
- **Depends on:** M0.1
|
- **Depends on:** M0.1
|
||||||
@@ -169,66 +186,81 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
|
|
||||||
### M0.3 — Self-hosted DNS + wildcard TLS
|
### M0.3 — Self-hosted DNS + wildcard TLS
|
||||||
- **Depends on:** M1.2 (vm-edge must exist before PowerDNS lands)
|
- **Depends on:** M1.2 (vm-edge must exist before PowerDNS lands)
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/breakpilot-edge`
|
||||||
- **Deliverables:**
|
- **Deliverables:**
|
||||||
- **PowerDNS Authoritative** on `vm-edge` (Orca-managed). PostgreSQL backend on same VM (small; ~100 records).
|
- **PowerDNS Authoritative** on `vm-edge` (Orca-managed). PostgreSQL backend on same VM (small; ~100 records).
|
||||||
- At the registrar (Benjamin's account): set `ns1.yourplatform.com` and `ns2.yourplatform.com` glue records pointing at vm-edge public IP; delegate the domain to those NS.
|
- At the registrar (Benjamin's account): set `ns1.breakpilot.com` and `ns2.breakpilot.com` glue records pointing at vm-edge public IP; delegate the domain to those NS.
|
||||||
- Zone file committed in `orca-platform/dns/yourplatform.com.zone`; Orca syncs into PowerDNS on apply.
|
- Zone file committed in `orca-platform/dns/breakpilot.com.zone`; Orca syncs into PowerDNS on apply.
|
||||||
- Records: apex `yourplatform.com`, wildcards `*.yourplatform.com` + `*.stage.yourplatform.com`, plus `auth.`, `erp.`, `mcp.`, `cdn.`, `mail.`, `ns1.`, `ns2.`, SPF/DKIM/DMARC TXT records (for M3.2).
|
- Records: apex `breakpilot.com`, wildcards `*.breakpilot.com` + `*.stage.breakpilot.com`, plus `auth.`, `erp.`, `mcp.`, `cdn.`, `mail.`, `ns1.`, `ns2.`, SPF/DKIM/DMARC TXT records (for M3.2).
|
||||||
- Wildcard TLS via Let's Encrypt **DNS-01 against PowerDNS** (Lego's `--dns=pdns` provider); ACME credentials in Infisical at `/prod/orca-proxy/PDNS_API_KEY`.
|
- Wildcard TLS via Let's Encrypt **DNS-01 against PowerDNS** (Lego's `--dns=pdns` provider); ACME credentials in Infisical at `/prod/orca-proxy/PDNS_API_KEY`.
|
||||||
- Orca-Proxy reloads the cert via watch on the secret file; renewal cron runs at 02:00 daily.
|
- Orca-Proxy reloads the cert via watch on the secret file; renewal cron runs at 02:00 daily.
|
||||||
- **Acceptance:** `dig @1.1.1.1 anything.yourplatform.com` returns an answer; `curl https://anything.yourplatform.com` returns 404 from Orca-Proxy (no TLS error).
|
- **Acceptance:** `dig @1.1.1.1 anything.breakpilot.com` returns an answer; `curl https://anything.breakpilot.com` returns 404 from Orca-Proxy (no TLS error).
|
||||||
- **Tests:** ACME renewal dry-run; PowerDNS zone-diff check in CI; reach via stage and prod subdomains; cert expiry page wired to SigNoz alert.
|
- **Tests:** ACME renewal dry-run; PowerDNS zone-diff check in CI; reach via stage and prod subdomains; cert expiry page wired to SigNoz alert.
|
||||||
- **Gate:** standard + manual DNS-delegation check by both founders (irreversible from registrar side without 24–48h propagation)
|
- **Gate:** standard + manual DNS-delegation check by both founders (irreversible from registrar side without 24–48h propagation)
|
||||||
- **Effort:** M (was S — registrar delegation + PowerDNS adds setup time vs. Cloudflare)
|
- **Effort:** M (was S — registrar delegation + PowerDNS adds setup time vs. Cloudflare)
|
||||||
|
|
||||||
### M1.1 — `orca-platform` repo (IaC)
|
### M1.1 — `orca-platform` repo (IaC staging)
|
||||||
- **Depends on:** M0.1, M0.2
|
- **Depends on:** M0.1, M0.2
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/orca-platform`
|
||||||
- **Deliverables:** directory layout per `INFRASTRUCTURE.md`; one Orca manifest per VM × service; per-env overlays (`overlays/dev`, `overlays/stage`, `overlays/prod`); a `Makefile` with `make plan` / `make apply` per env.
|
- **Deliverables:** per-cluster manifest layout under `clusters/breakpilot-{edge,control,app}/services/`; per-env overlays (`overlays/dev`, `overlays/stage`, `overlays/prod`); `cluster.toml.tmpl` template parameterized by cluster; a `Makefile` with `make plan` / `make apply` per env; `scripts/validate.sh` enforces a per-cluster `placement.node` whitelist (`breakpilot-edge → vm-edge`, `breakpilot-control → vm-control`, `breakpilot-app → {vm-app-prod, vm-app-stage}`).
|
||||||
- **Acceptance:** `make plan ENV=stage` produces a no-op diff once applied.
|
- **Acceptance:** `make plan ENV=stage` and `make plan ENV=prod` both resolve cleanly; `make validate` passes the per-cluster node check on all manifests.
|
||||||
- **Tests:** `orca validate` runs in CI; PRs that break a manifest fail.
|
- **Tests:** `make validate` runs in CI; PRs that break a manifest or cross-place a service into the wrong cluster's node fail.
|
||||||
- **Gate:** standard
|
- **Gate:** standard
|
||||||
- **Effort:** M
|
- **Effort:** M
|
||||||
|
- **Status (2026-06-30):** layout landed; cluster split refactor merged via PR #6 (`6be727d4`). Manifests under `clusters/breakpilot-*/services/` are stubs awaiting per-milestone fill-in. `make apply` is a no-op until M1.2.
|
||||||
|
|
||||||
### M1.2 — Provision VMs (locked topology)
|
### M1.2 — Provision VMs + split cluster repos (locked topology)
|
||||||
- **Depends on:** M1.1 (Orca manifest layout)
|
- **Depends on:** M1.1 (manifest layout exists); **legal entity established** (gating SysEleven / Hetzner business contracts; no commercial activity before this).
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/orca-platform` (source of split), `platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app` (created in this milestone).
|
||||||
- **Deliverables:** the **4 VMs** from `INFRASTRUCTURE.md §1` provisioned on SysEleven (DUS2):
|
- **Deliverables:** **4 VMs across 3 clusters**, one cluster per plane (per the 2026-06-30 cluster-split decision):
|
||||||
- **stage** (m2.small, public IP) — runs app-plane code only, calls prod KC + Stalwart
|
|
||||||
- **vm-edge** (m2.small, public IP) — Identity + Infra planes (orca-proxy, PowerDNS, Keycloak, pg-keycloak, Infisical, pg-infisical, Gitea)
|
| Cluster | VM | Flavor | Plane | Services |
|
||||||
- **vm-control** (m2.medium) — Control plane (portal, tenant-registry, ERPNext, Frappe HD, MariaDB, Stalwart)
|
|---|---|---|---|---|
|
||||||
- **vm-data** (m2.medium) — Data plane (CERTifAI, MongoDB, LiteLLM, compliance ×3, pg-app, Qdrant, MinIO)
|
| `breakpilot-edge` | `vm-edge` | m2.small, public IP | Identity + Infra | orca-proxy, PowerDNS, Keycloak (repurposed from CERTifAI), pg-keycloak, Infisical, pg-infisical, redis-infisical, Gitea |
|
||||||
- Private network 10.0.0.0/16 between all four. Public ingress only via vm-edge (and stage's own IP for tester access).
|
| `breakpilot-control` | `vm-control` | m2.medium | Control | portal, tenant-registry, ERPNext, Frappe HD, MariaDB, redis-erpnext, Stalwart |
|
||||||
- SSH disabled; only `orca exec` for shell access.
|
| `breakpilot-app` | `vm-app-prod` | m2.medium | App (prod) | CERTifAI, compliance ×3, MongoDB, LiteLLM, pg-app, Qdrant, MinIO |
|
||||||
- **Acceptance:** every VM reachable from Orca control plane; private-network connectivity verified; resource limits per service set in manifest per `INFRASTRUCTURE.md §6` co-tenant notes.
|
| `breakpilot-app` | `vm-app-stage` | m2.small, public IP | App (stage) | slim parallel set for staging; calls prod KC + prod tenant-registry under `tenant.kind = "stage"` |
|
||||||
- **Tests:** cold-start sequence from `INFRASTRUCTURE.md §10 Scenario F` runs successfully on stage VMs.
|
|
||||||
- **Gate:** standard + manual sign-off (touches infra spend and 36M commitment decision)
|
- **Split the cluster repos:** for each cluster, lift `platform/orca-platform/clusters/breakpilot-<name>/` into its own Gitea repo `platform/breakpilot-<name>` with the §1.2 scaffolding + its own `cluster.toml` + its own `services/` tree. Empty out (or stub) the original `clusters/breakpilot-<name>/` in `orca-platform` and leave a pointer README.
|
||||||
- **Effort:** M
|
- **Repurpose the existing Keycloak.** The CERTifAI Keycloak realm export + user database moves to `vm-edge` rather than standing up a fresh instance — a hostname swap and realm export/import, not a re-bootstrap.
|
||||||
- **Cost impact:** see COST_PLAN.md §3. Initial run: ~€552/mo On-Demand, dropping to ~€310/mo after 36M-upfront commit in Month 4.
|
- **Private network** 10.0.0.0/16 between all four VMs.
|
||||||
|
- **Public ingress** per Orca's per-node-ingress model: every VM binds 80/443 and runs its own Let's Encrypt ACME; A records for a service point at the agent IP that runs the container, not at a master.
|
||||||
|
- **SSH disabled**; only `orca exec` for shell access.
|
||||||
|
- **Acceptance:**
|
||||||
|
- Every VM is reachable from its cluster's Orca master; private-network connectivity verified.
|
||||||
|
- Resource limits per service set in manifest per `INFRASTRUCTURE.md §6` co-tenant notes.
|
||||||
|
- `make validate` (in `orca-platform`) and the new per-cluster repo CIs all pass.
|
||||||
|
- Cold-start sequence from `INFRASTRUCTURE.md §10 Scenario F` runs successfully on `vm-app-stage`.
|
||||||
|
- **Tests:** cold-start drill on `vm-app-stage`; cross-cluster reachability matrix; KC realm-import smoke (stage tenant logs in).
|
||||||
|
- **Gate:** standard + manual sign-off (touches infra spend and 36M commitment decision) + legal-entity confirmation before any contract signing.
|
||||||
|
- **Effort:** L (was M — adds the 3-repo split + KC realm migration on top of provisioning).
|
||||||
|
- **Cost impact:** see COST_PLAN.md §3. Initial run: ~€552/mo On-Demand, dropping to ~€310/mo after 36M-upfront commit in Month 4. `vm-app-stage` is the cheapest VM in the topology so adding it doesn't materially shift the bill.
|
||||||
|
|
||||||
|
**Pre-M1.2 reality (today, single-VM era):** until this milestone lands, everything runs on the existing single VM (`46.225.100.82`) via `~/workspace/orca-infra`. M7.x work (tenant-registry-aware compliance-agent, per-tenant MCP) and other product work is meaningful on the single-VM setup and ports cleanly into `breakpilot-app/services/prod/` at migration time.
|
||||||
|
|
||||||
### M1.3 — Backups, monitoring, on-call
|
### M1.3 — Backups, monitoring, on-call
|
||||||
- **Depends on:** M1.2
|
- **Depends on:** M1.2
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app` (each cluster owns its own `[backup]` block + restore script per `INFRASTRUCTURE.md §6`)
|
||||||
- **Deliverables:** backup cron per VM per `INFRASTRUCTURE.md §3` (Postgres pg_dump, MinIO bucket replication); SigNoz OTel collector running on every VM; alert routing to `oncall@yourplatform.com`; restore runbook in `platform/docs/runbooks/restore.md`.
|
- **Deliverables:** backup cron per VM per `INFRASTRUCTURE.md §3` (Postgres pg_dump on `pg-keycloak` / `pg-app` / `mariadb`, MinIO bucket replication); SigNoz OTel collector running on every VM; alert routing to `oncall@breakpilot.com`; restore runbook in `platform/docs/runbooks/restore.md`. Each cluster repo has its own S3 bucket per `INFRASTRUCTURE.md §6` so a runaway backup on `breakpilot-app` cannot fill the bucket `breakpilot-edge` writes to.
|
||||||
- **Acceptance:** restore drill on stage succeeds (script in `platform/orca-platform/scripts/restore-drill.sh`); SigNoz shows traces from a synthetic request.
|
- **Acceptance:** restore drill on `vm-app-stage` succeeds (script in `breakpilot-app/scripts/restore-drill.sh`); SigNoz shows traces from a synthetic request through `vm-app-stage` → `vm-control` → `vm-edge`.
|
||||||
- **Tests:** disaster-recovery exercise per failure scenario in `INFRASTRUCTURE.md §10` — at least Scenarios A, B, F validated on stage.
|
- **Tests:** disaster-recovery exercise per failure scenario in `INFRASTRUCTURE.md §10` — at least Scenarios A, B, F validated on stage.
|
||||||
- **Gate:** standard + manual sign-off
|
- **Gate:** standard + manual sign-off
|
||||||
- **Effort:** L
|
- **Effort:** L
|
||||||
|
|
||||||
### M2.1 — Keycloak deployment
|
### M2.1 — Keycloak deployment (repurpose from CERTifAI)
|
||||||
- **Depends on:** M1.2, M1.3
|
- **Depends on:** M1.2, M1.3
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/breakpilot-edge`
|
||||||
- **Deliverables:** Keycloak 26 on `vm-identity`, Postgres backing store on `vm-control`, exposed at `auth.yourplatform.com` and `auth.stage.yourplatform.com`. Realm import file in `orca-platform/keycloak/realm-export.json` (committed, source-of-truth).
|
- **Deliverables:** Keycloak 26 on **`vm-edge`** (co-tenant with the rest of infra; JVM heap pinned `-Xmx1500m` so it cannot starve PowerDNS / Infisical, per `INFRASTRUCTURE.md §6`). Postgres backing store (`pg-keycloak`) also on `vm-edge`. Exposed at `auth.breakpilot.com`; stage tenants authenticate against the same instance under `tenant.kind = "stage"` (no separate stage realm). **The CERTifAI Keycloak instance is repurposed here** — its realm export + user database moves to `vm-edge` rather than standing up a fresh instance. Realm import file in `breakpilot-edge/services/keycloak/realm-export.json` (committed, source-of-truth).
|
||||||
- **Acceptance:** master admin login works; realm `breakpilot-prod` exists in both envs.
|
- **Acceptance:** master admin login works; realm `breakpilot-prod` exists; the realm carries the migrated CERTifAI users; a stage tenant test login succeeds with `tenant.kind = "stage"` in the JWT.
|
||||||
- **Tests:** automated realm-state diff in CI (`kcadm` against checked-in export).
|
- **Tests:** automated realm-state diff in CI (`kcadm` against checked-in export); side-by-side comparison of pre-migration and post-migration realm exports flags any silent drift.
|
||||||
- **Gate:** standard + security checklist
|
- **Gate:** standard + security checklist + manual sign-off on the realm-import diff (irreversible without restoring from KC backup).
|
||||||
- **Effort:** M
|
- **Effort:** M
|
||||||
|
- **Note:** the escape hatch — if the JVM/PowerDNS fight shows up in alerts after launch — is to flip Keycloak's `placement.node` to a new `vm-edge-identity` VM inside the same `breakpilot-edge` cluster. No schema migration, just a container move.
|
||||||
|
|
||||||
### M2.2 — Realm configuration: roles + protocol mappers + Organizations
|
### M2.2 — Realm configuration: roles + protocol mappers + Organizations
|
||||||
- **Depends on:** M2.1
|
- **Depends on:** M2.1
|
||||||
- **Repos:** `platform/orca-platform` (realm config)
|
- **Repos:** `platform/breakpilot-edge` (realm config)
|
||||||
- **Deliverables:** Organizations feature enabled; realm roles `BREAKPILOT_ADMIN`, `SUPPORT_ENGINEER`, `SALES_REP`; org roles `IT_ADMIN`, `CXO`, `FINANCE`, `LEGAL`, `USER`; protocol mapper that calls Tenant Registry at token issuance for `products`, `plan`, `tenant_status` claims; SALES_REP guardrail policy (token only issuable with `org_id = demo`).
|
- **Deliverables:** Organizations feature enabled; realm roles `BREAKPILOT_ADMIN`, `SUPPORT_ENGINEER`, `SALES_REP`; org roles `IT_ADMIN`, `CXO`, `FINANCE`, `LEGAL`, `USER`; protocol mapper that calls Tenant Registry at token issuance for `products`, `plan`, `tenant_status`, `tenant_kind` claims; SALES_REP guardrail policy (token only issuable with `org_id = demo`).
|
||||||
- **Acceptance:** a test user gets the expected JWT claims; a SALES_REP user cannot get a JWT for a non-demo org (verified by integration test).
|
- **Acceptance:** a test user gets the expected JWT claims; a SALES_REP user cannot get a JWT for a non-demo org (verified by integration test).
|
||||||
- **Tests:** Keycloak integration suite in `platform/tenant-registry/test/keycloak_test.go`.
|
- **Tests:** Keycloak integration suite in `platform/tenant-registry/test/keycloak_test.go`.
|
||||||
- **Gate:** standard + security checklist
|
- **Gate:** standard + security checklist
|
||||||
@@ -236,32 +268,34 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
|
|
||||||
### M3.1 — Infisical
|
### M3.1 — Infisical
|
||||||
- **Depends on:** M1.2
|
- **Depends on:** M1.2
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/breakpilot-edge`
|
||||||
- **Deliverables:** Infisical on `vm-secrets`, machine identity per service, secret paths laid out per `PRODUCT_INTEGRATION_SPEC.md §9.4`.
|
- **Deliverables:** Infisical on **`vm-edge`** (co-tenant with Keycloak, PowerDNS, Gitea), machine identity per service, secret paths laid out per `PRODUCT_INTEGRATION_SPEC.md §9.4`.
|
||||||
- **Acceptance:** a stub service can read its secrets at startup; rotating a secret in Infisical UI is picked up on next pod start.
|
- **Acceptance:** a stub service can read its secrets at startup; rotating a secret in Infisical UI is picked up on next pod start.
|
||||||
- **Tests:** smoke test container reads secrets.
|
- **Tests:** smoke test container reads secrets.
|
||||||
- **Gate:** standard + security checklist
|
- **Gate:** standard + security checklist
|
||||||
- **Effort:** S
|
- **Effort:** S
|
||||||
|
- **Bootstrap exception (per `INFRASTRUCTURE.md §8 rule 3`):** Keycloak's `KC_DB_URL` lives in Orca env (not Infisical) because both run on `vm-edge` and we'd otherwise have a circular bootstrap dependency.
|
||||||
|
|
||||||
### M3.2 — Stalwart transactional email
|
### M3.2 — Stalwart transactional email
|
||||||
- **Depends on:** M0.3 (needs DNS records under our control), M3.1
|
- **Depends on:** M0.3 (needs DNS records under our control), M3.1
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/breakpilot-control`
|
||||||
- **Deliverables:**
|
- **Deliverables:**
|
||||||
- **Stalwart** on `vm-control` (Orca-managed); reachable at `mail.yourplatform.com`.
|
- **Stalwart** on `vm-control` (Orca-managed); reachable at `mail.breakpilot.com`.
|
||||||
- DNS records added to the zone in M0.3: `mail` A record, MX → mail, SPF (`v=spf1 mx -all`), DKIM (Stalwart-generated public key), DMARC (`p=quarantine; rua=mailto:dmarc@yourplatform.com`), reverse DNS (PTR) configured at the cloud provider for the vm-control public IP — coordinate with vm-edge since outbound mail must egress from a host with a clean PTR.
|
- DNS records added to the zone in M0.3: `mail` A record, MX → mail, SPF (`v=spf1 mx -all`), DKIM (Stalwart-generated public key), DMARC (`p=quarantine; rua=mailto:dmarc@breakpilot.com`), reverse DNS (PTR) configured at the cloud provider for the vm-control public IP — coordinate with vm-edge since outbound mail must egress from a host with a clean PTR.
|
||||||
- SMTP submission service account per platform sender: `noreply@`, `oncall@`, `support@`, `billing@`, `dmarc@`.
|
- SMTP submission service account per platform sender: `noreply@`, `oncall@`, `support@`, `billing@`, `dmarc@`.
|
||||||
- Outbound queue and bounce handler; failed deliveries surface as audit events.
|
- Outbound queue and bounce handler; failed deliveries surface as audit events.
|
||||||
- Webhook receiver at `/inbound/postmaster` for bounce/complaint feedback loops (Gmail FBL, MS SNDS).
|
- Webhook receiver at `/inbound/postmaster` for bounce/complaint feedback loops (Gmail FBL, MS SNDS).
|
||||||
- **IP warming plan**: write a `platform/docs/runbooks/email-warming.md` documenting the 4–8 week ramp from low daily volumes; first 2 weeks of trial nudges (M12.2) explicitly throttled.
|
- **IP warming plan**: write a `platform/docs/runbooks/email-warming.md` documenting the 4–8 week ramp from low daily volumes; first 2 weeks of trial nudges (M12.2) explicitly throttled.
|
||||||
- **Acceptance:** test email from `noreply@yourplatform.com` to `parnerkarsharang@gmail.com` lands in inbox (not spam) on day 1; SPF/DKIM/DMARC all "pass" in Gmail's "show original" view; mail-tester.com score ≥ 9/10.
|
- **Acceptance:** test email from `noreply@breakpilot.com` to `parnerkarsharang@gmail.com` lands in inbox (not spam) on day 1; SPF/DKIM/DMARC all "pass" in Gmail's "show original" view; mail-tester.com score ≥ 9/10.
|
||||||
- **Tests:** automated daily mail-tester check (failure pages on-call); bounce-handling integration test.
|
- **Tests:** automated daily mail-tester check (failure pages on-call); bounce-handling integration test.
|
||||||
- **Gate:** standard + security checklist + manual deliverability sign-off (DKIM keys are load-bearing)
|
- **Gate:** standard + security checklist + manual deliverability sign-off (DKIM keys are load-bearing)
|
||||||
- **Effort:** L (deliverability tuning is the long tail)
|
- **Effort:** L (deliverability tuning is the long tail)
|
||||||
|
|
||||||
**Phase 0 exit criteria:**
|
**Phase 0 exit criteria:**
|
||||||
- Stage cluster boots cold from cron-driven nightly stop/start using only `INFRASTRUCTURE.md §5` ordering.
|
- `vm-app-stage` boots cold from cron-driven nightly stop/start using only `INFRASTRUCTURE.md §5` ordering; `breakpilot-edge` and `breakpilot-control` clusters remain warm throughout (no nightly cycling for identity / control).
|
||||||
- A synthetic HTTPS request to `https://hello.stage.yourplatform.com` reaches a stub container.
|
- A synthetic HTTPS request to `https://hello.stage.breakpilot.com` reaches a stub container on `vm-app-stage`.
|
||||||
- Restore drill on stage Postgres succeeds end-to-end.
|
- Restore drill on `vm-app-prod` Postgres (`pg-app`) and `vm-control` Postgres (`pg-keycloak`, `pg-mariadb`) both succeed end-to-end.
|
||||||
|
- All three cluster repos (`platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app`) have green CI on main and a successful `make validate`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -300,7 +334,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
- **Depends on:** M2.2, M4.3, M0.3
|
- **Depends on:** M2.2, M4.3, M0.3
|
||||||
- **Repos:** `platform/portal`, `platform/design-tokens`
|
- **Repos:** `platform/portal`, `platform/design-tokens`
|
||||||
- **Deliverables:** Next.js 15 app on `vm-control`; middleware reads `Host` → extracts slug → calls Tenant Registry `GET /tenants?slug=` → injects tenant context; Keycloak OIDC login; logout; `design-tokens` package consumed by portal.
|
- **Deliverables:** Next.js 15 app on `vm-control`; middleware reads `Host` → extracts slug → calls Tenant Registry `GET /tenants?slug=` → injects tenant context; Keycloak OIDC login; logout; `design-tokens` package consumed by portal.
|
||||||
- **Acceptance:** visiting `https://acme.stage.yourplatform.com` redirects to Keycloak; after login, user lands on `/acme/dashboard` (empty page) with valid session.
|
- **Acceptance:** visiting `https://acme.stage.breakpilot.com` redirects to Keycloak; after login, user lands on `/acme/dashboard` (empty page) with valid session.
|
||||||
- **Tests:** Playwright e2e: login + logout for an existing test tenant.
|
- **Tests:** Playwright e2e: login + logout for an existing test tenant.
|
||||||
- **Gate:** standard
|
- **Gate:** standard
|
||||||
- **Effort:** M
|
- **Effort:** M
|
||||||
@@ -317,14 +351,14 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
### M5.3 — Playwright e2e harness
|
### M5.3 — Playwright e2e harness
|
||||||
- **Depends on:** M5.2
|
- **Depends on:** M5.2
|
||||||
- **Repos:** `platform/portal`
|
- **Repos:** `platform/portal`
|
||||||
- **Deliverables:** Playwright config that runs against `stage.yourplatform.com` post-deploy; CI job `e2e-stage` triggered after stage deploy; failure pages on-call.
|
- **Deliverables:** Playwright config that runs against `stage.breakpilot.com` post-deploy; CI job `e2e-stage` triggered after stage deploy; failure pages on-call.
|
||||||
- **Acceptance:** breaking change to login is caught in CI within 10 min of merge.
|
- **Acceptance:** breaking change to login is caught in CI within 10 min of merge.
|
||||||
- **Tests:** the suite itself.
|
- **Tests:** the suite itself.
|
||||||
- **Gate:** standard
|
- **Gate:** standard
|
||||||
- **Effort:** S
|
- **Effort:** S
|
||||||
|
|
||||||
**Phase 1 exit criteria:**
|
**Phase 1 exit criteria:**
|
||||||
- A tenant created via `POST /tenants` results in a working login flow at `<slug>.stage.yourplatform.com`.
|
- A tenant created via `POST /tenants` results in a working login flow at `<slug>.stage.breakpilot.com`.
|
||||||
- All Phase 1 routes have a passing Playwright spec running on every stage deploy.
|
- All Phase 1 routes have a passing Playwright spec running on every stage deploy.
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -354,7 +388,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
### M6.3 — CERTifAI: manifest + integration assets
|
### M6.3 — CERTifAI: manifest + integration assets
|
||||||
- **Depends on:** M6.2
|
- **Depends on:** M6.2
|
||||||
- **Repos:** `benjamin_boenisch/certifai`
|
- **Repos:** `benjamin_boenisch/certifai`
|
||||||
- **Deliverables:** `product.manifest.yaml` per `PRODUCT_INTEGRATION_SPEC.md §10` published to `cdn.yourplatform.com`; OpenAPI 3.1 spec; `/v1/health`, `/v1/usage`, `/v1/tenants/:id/export`, `DELETE /v1/tenants/:id/data`, `POST /v1/tenants/demo/reset`; web component `certifai-dashboard` per §5.A.
|
- **Deliverables:** `product.manifest.yaml` per `PRODUCT_INTEGRATION_SPEC.md §10` published to `cdn.breakpilot.com`; OpenAPI 3.1 spec; `/v1/health`, `/v1/usage`, `/v1/tenants/:id/export`, `DELETE /v1/tenants/:id/data`, `POST /v1/tenants/demo/reset`; web component `certifai-dashboard` per §5.A.
|
||||||
- **Acceptance:** CERTifAI appears in the portal catalog; subscribed tenants can open it from the dashboard.
|
- **Acceptance:** CERTifAI appears in the portal catalog; subscribed tenants can open it from the dashboard.
|
||||||
- **Tests:** contract test that manifest validates against schema; web component renders inside portal shadow-DOM host.
|
- **Tests:** contract test that manifest validates against schema; web component renders inside portal shadow-DOM host.
|
||||||
- **Gate:** standard
|
- **Gate:** standard
|
||||||
@@ -390,8 +424,8 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
|
|
||||||
### M8.1 — ERPNext deployment
|
### M8.1 — ERPNext deployment
|
||||||
- **Depends on:** M1.2, M2.1
|
- **Depends on:** M1.2, M2.1
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/breakpilot-control`
|
||||||
- **Deliverables:** Frappe + ERPNext on `vm-control` (separate Postgres database from tenant_registry — see `INFRASTRUCTURE.md` RISK-1); reached at `erp.yourplatform.com`; Keycloak OIDC; IP-restricted at Orca-Proxy.
|
- **Deliverables:** Frappe + ERPNext on `vm-control` (separate Postgres database from tenant_registry — see `INFRASTRUCTURE.md` RISK-1); reached at `erp.breakpilot.com`; Keycloak OIDC; IP-restricted at Orca-Proxy.
|
||||||
- **Acceptance:** us login works; a Customer record can be created manually.
|
- **Acceptance:** us login works; a Customer record can be created manually.
|
||||||
- **Tests:** smoke test for OIDC; backup of Frappe filestore validated.
|
- **Tests:** smoke test for OIDC; backup of Frappe filestore validated.
|
||||||
- **Gate:** standard + manual sign-off (touches `vm-control` resources)
|
- **Gate:** standard + manual sign-off (touches `vm-control` resources)
|
||||||
@@ -399,7 +433,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
|
|
||||||
### M8.2 — ERPNext customization
|
### M8.2 — ERPNext customization
|
||||||
- **Depends on:** M8.1
|
- **Depends on:** M8.1
|
||||||
- **Repos:** `platform/orca-platform/erpnext-app/`
|
- **Repos:** `platform/breakpilot-control` (`erpnext-app/` subdir under the cluster repo)
|
||||||
- **Deliverables:** custom Frappe app with: `tenant_id` field on `Customer`; `sales_owner` field on `Lead`; server scripts for the Sales Order → Tenant Registry webhook; `Cancel` workflow that calls Tenant Registry `/cancel`.
|
- **Deliverables:** custom Frappe app with: `tenant_id` field on `Customer`; `sales_owner` field on `Lead`; server scripts for the Sales Order → Tenant Registry webhook; `Cancel` workflow that calls Tenant Registry `/cancel`.
|
||||||
- **Acceptance:** submitting a Sales Order in ERPNext triggers a tenant activation in stage Tenant Registry.
|
- **Acceptance:** submitting a Sales Order in ERPNext triggers a tenant activation in stage Tenant Registry.
|
||||||
- **Tests:** server-script unit tests (Frappe test harness); integration test exercises the full webhook.
|
- **Tests:** server-script unit tests (Frappe test harness); integration test exercises the full webhook.
|
||||||
@@ -424,7 +458,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
|
|
||||||
### M9.1 — Frappe Helpdesk
|
### M9.1 — Frappe Helpdesk
|
||||||
- **Depends on:** M8.1
|
- **Depends on:** M8.1
|
||||||
- **Repos:** `platform/orca-platform`
|
- **Repos:** `platform/breakpilot-control`
|
||||||
- **Deliverables:** Frappe HD on the same Frappe bench; customer portal embedded at `/[slug]/support/`.
|
- **Deliverables:** Frappe HD on the same Frappe bench; customer portal embedded at `/[slug]/support/`.
|
||||||
- **Acceptance:** a customer user can submit a ticket; we receive it.
|
- **Acceptance:** a customer user can submit a ticket; we receive it.
|
||||||
- **Tests:** Playwright spec for ticket submission.
|
- **Tests:** Playwright spec for ticket submission.
|
||||||
@@ -433,7 +467,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
|
|
||||||
### M9.2 — HD → Gitea escalation
|
### M9.2 — HD → Gitea escalation
|
||||||
- **Depends on:** M9.1
|
- **Depends on:** M9.1
|
||||||
- **Repos:** `platform/orca-platform/erpnext-app/`
|
- **Repos:** `platform/breakpilot-control` (`erpnext-app/` subdir)
|
||||||
- **Deliverables:** server script that on a `Ticket: Escalate to Engineering` action creates a Gitea issue in the matching repo via Gitea REST API; reverse webhook from Gitea on issue close marks ticket resolved.
|
- **Deliverables:** server script that on a `Ticket: Escalate to Engineering` action creates a Gitea issue in the matching repo via Gitea REST API; reverse webhook from Gitea on issue close marks ticket resolved.
|
||||||
- **Acceptance:** the round-trip works for a test ticket on stage.
|
- **Acceptance:** the round-trip works for a test ticket on stage.
|
||||||
- **Tests:** integration test against stage Gitea.
|
- **Tests:** integration test against stage Gitea.
|
||||||
@@ -489,7 +523,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
### M12.2 — Trial lifecycle cron + emails
|
### M12.2 — Trial lifecycle cron + emails
|
||||||
- **Depends on:** M12.1, M3.2 (Stalwart must be deliverability-clean)
|
- **Depends on:** M12.1, M3.2 (Stalwart must be deliverability-clean)
|
||||||
- **Repos:** `platform/tenant-registry`
|
- **Repos:** `platform/tenant-registry`
|
||||||
- **Deliverables:** scheduler in tenant-registry that runs day-7/12/14 emails; status transitions trial → active (on payment) or trial → frozen → archived; SMTP via Stalwart at `mail.yourplatform.com:587`; sender `noreply@yourplatform.com`; HTML + plaintext templates committed under `tenant-registry/templates/email/`; List-Unsubscribe headers per RFC 8058.
|
- **Deliverables:** scheduler in tenant-registry that runs day-7/12/14 emails; status transitions trial → active (on payment) or trial → frozen → archived; SMTP via Stalwart at `mail.breakpilot.com:587`; sender `noreply@breakpilot.com`; HTML + plaintext templates committed under `tenant-registry/templates/email/`; List-Unsubscribe headers per RFC 8058.
|
||||||
- **Acceptance:** in a time-warped stage test (script that advances `trial_ends_at`), all transitions fire in order and all three emails land in Gmail inbox.
|
- **Acceptance:** in a time-warped stage test (script that advances `trial_ends_at`), all transitions fire in order and all three emails land in Gmail inbox.
|
||||||
- **Tests:** integration test with time injection; deliverability spot-check at each release.
|
- **Tests:** integration test with time injection; deliverability spot-check at each release.
|
||||||
- **Gate:** standard
|
- **Gate:** standard
|
||||||
@@ -498,7 +532,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
### M13.1 — Demo tenant seeding
|
### M13.1 — Demo tenant seeding
|
||||||
- **Depends on:** M6.3, M7.2
|
- **Depends on:** M6.3, M7.2
|
||||||
- **Repos:** `platform/seed-data`
|
- **Repos:** `platform/seed-data`
|
||||||
- **Deliverables:** per-product fixture archives (`certifai/seed-v1.tar.gz`, `compliance/seed-v1.tar.gz`); publishing pipeline to `cdn.yourplatform.com`; `catalog.demo.seed_data_url` populated in product manifests.
|
- **Deliverables:** per-product fixture archives (`certifai/seed-v1.tar.gz`, `compliance/seed-v1.tar.gz`); publishing pipeline to `cdn.breakpilot.com`; `catalog.demo.seed_data_url` populated in product manifests.
|
||||||
- **Acceptance:** calling `POST /v1/tenants/demo/reset` on either product restores fixtures.
|
- **Acceptance:** calling `POST /v1/tenants/demo/reset` on either product restores fixtures.
|
||||||
- **Tests:** integration test asserts fixture state after reset.
|
- **Tests:** integration test asserts fixture state after reset.
|
||||||
- **Gate:** standard
|
- **Gate:** standard
|
||||||
@@ -508,7 +542,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
- **Depends on:** M2.2, M13.1
|
- **Depends on:** M2.2, M13.1
|
||||||
- **Repos:** `platform/portal`, `platform/tenant-registry`
|
- **Repos:** `platform/portal`, `platform/tenant-registry`
|
||||||
- **Deliverables:** demo tenant created in stage and prod with `kind=demo, status=demo`; SALES_REP role usable; backstage routes restricted to `/backstage/leads` and `/backstage/demo`; demo tenant audit events tagged `{"demo": true}` and hidden from real-tenant audit views.
|
- **Deliverables:** demo tenant created in stage and prod with `kind=demo, status=demo`; SALES_REP role usable; backstage routes restricted to `/backstage/leads` and `/backstage/demo`; demo tenant audit events tagged `{"demo": true}` and hidden from real-tenant audit views.
|
||||||
- **Acceptance:** sales rep logs in at `demo.yourplatform.com`, walks both products live, [Request Trial] modal creates a CRM Lead with `sales_owner = the rep`.
|
- **Acceptance:** sales rep logs in at `demo.breakpilot.com`, walks both products live, [Request Trial] modal creates a CRM Lead with `sales_owner = the rep`.
|
||||||
- **Tests:** Playwright e2e for the sales walk-through.
|
- **Tests:** Playwright e2e for the sales walk-through.
|
||||||
- **Gate:** standard + security checklist (SALES_REP guardrail enforcement is the load-bearing piece)
|
- **Gate:** standard + security checklist (SALES_REP guardrail enforcement is the load-bearing piece)
|
||||||
- **Effort:** M
|
- **Effort:** M
|
||||||
@@ -543,7 +577,7 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
**Phase 4 exit criteria:**
|
**Phase 4 exit criteria:**
|
||||||
- Every flow P1–P16 from `PLATFORM_ARCHITECTURE.md` has a passing Playwright spec.
|
- Every flow P1–P16 from `PLATFORM_ARCHITECTURE.md` has a passing Playwright spec.
|
||||||
- Stage runs a full lifecycle: sign-up trial → convert → use → cancel → offboard, in an automated nightly job.
|
- Stage runs a full lifecycle: sign-up trial → convert → use → cancel → offboard, in an automated nightly job.
|
||||||
- We can hand a prospect a real demo using `demo.yourplatform.com`.
|
- We can hand a prospect a real demo using `demo.breakpilot.com`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -581,8 +615,8 @@ Every place where a future flag would gate behaviour MUST flow through a single
|
|||||||
### M17.1 — MCP servers (Enterprise)
|
### M17.1 — MCP servers (Enterprise)
|
||||||
- **Depends on:** M6.3, M7.2
|
- **Depends on:** M6.3, M7.2
|
||||||
- **Repos:** `benjamin_boenisch/certifai`, `benjamin_boenisch/breakpilot-compliance`
|
- **Repos:** `benjamin_boenisch/certifai`, `benjamin_boenisch/breakpilot-compliance`
|
||||||
- **Deliverables:** MCP endpoints per `PRODUCT_INTEGRATION_SPEC.md §10` `mcp:` block; gated on `plan == enterprise`; routed via `mcp.yourplatform.com`.
|
- **Deliverables:** MCP endpoints per `PRODUCT_INTEGRATION_SPEC.md §10` `mcp:` block; gated on `plan == enterprise`; routed via `mcp.breakpilot.com`.
|
||||||
- **Acceptance:** Claude Code can connect to `mcp.yourplatform.com/certifai` with a service token and call `list_ai_agents`.
|
- **Acceptance:** Claude Code can connect to `mcp.breakpilot.com/certifai` with a service token and call `list_ai_agents`.
|
||||||
- **Tests:** MCP contract test using `mcp-cli`.
|
- **Tests:** MCP contract test using `mcp-cli`.
|
||||||
- **Gate:** standard + security checklist
|
- **Gate:** standard + security checklist
|
||||||
- **Effort:** L
|
- **Effort:** L
|
||||||
@@ -598,7 +632,7 @@ These ship only when a paying customer requires them.
|
|||||||
|
|
||||||
### M18.1 — Custom domains
|
### M18.1 — Custom domains
|
||||||
- **Depends on:** M0.3, M10.1
|
- **Depends on:** M0.3, M10.1
|
||||||
- **Repos:** `platform/orca-platform`, `platform/portal`
|
- **Repos:** `platform/breakpilot-edge` (Orca-Proxy ACME), `platform/portal` (UI)
|
||||||
- **Deliverables:** ACME on-demand TLS in Orca-Proxy; portal UI for customer to add domain; CNAME verification.
|
- **Deliverables:** ACME on-demand TLS in Orca-Proxy; portal UI for customer to add domain; CNAME verification.
|
||||||
- **Acceptance:** `compliance.acme.com` resolves and renders the Acme portal.
|
- **Acceptance:** `compliance.acme.com` resolves and renders the Acme portal.
|
||||||
- **Tests:** integration test with a synthetic domain.
|
- **Tests:** integration test with a synthetic domain.
|
||||||
@@ -607,7 +641,7 @@ These ship only when a paying customer requires them.
|
|||||||
|
|
||||||
### M18.2 — Physical data isolation
|
### M18.2 — Physical data isolation
|
||||||
- **Depends on:** M4.1, M6.1, M7.1
|
- **Depends on:** M4.1, M6.1, M7.1
|
||||||
- **Repos:** all data-plane products + `tenant-registry`
|
- **Repos:** all app-plane products + `platform/tenant-registry`
|
||||||
- **Deliverables:** option per tenant for a dedicated Postgres / Mongo schema or database; provisioning automation; migration path from logical → physical.
|
- **Deliverables:** option per tenant for a dedicated Postgres / Mongo schema or database; provisioning automation; migration path from logical → physical.
|
||||||
- **Acceptance:** an enterprise tenant runs on a dedicated schema; cross-tenant queries are physically impossible.
|
- **Acceptance:** an enterprise tenant runs on a dedicated schema; cross-tenant queries are physically impossible.
|
||||||
- **Tests:** isolation enforcement test.
|
- **Tests:** isolation enforcement test.
|
||||||
@@ -645,7 +679,7 @@ When starting work, the first sequence of PRs should be:
|
|||||||
2. **PR-2 to PR-7** (M0.1 continued): Bootstrap each of the other five repos with §1.2 scaffolding. Land in parallel.
|
2. **PR-2 to PR-7** (M0.1 continued): Bootstrap each of the other five repos with §1.2 scaffolding. Land in parallel.
|
||||||
3. **PR-8** (M0.2): CI templates + branch protection per repo.
|
3. **PR-8** (M0.2): CI templates + branch protection per repo.
|
||||||
4. **PR-9** (M1.1): `orca-platform` directory layout + first stub manifest.
|
4. **PR-9** (M1.1): `orca-platform` directory layout + first stub manifest.
|
||||||
5. **PR-10** (M1.2): VM provisioning (vm-edge, vm-identity, vm-secrets, vm-control first — DNS and Keycloak depend on these).
|
5. **PR-10** (M1.2): VM provisioning + cluster-repo split. Per the 2026-06-30 decision: 4 VMs across 3 clusters (`vm-edge` in `breakpilot-edge`, `vm-control` in `breakpilot-control`, `vm-app-prod` + `vm-app-stage` in `breakpilot-app`). Spin up `vm-edge` first since DNS, Keycloak, and Infisical all live there. **Gated on legal entity** (cannot sign SysEleven / Hetzner business contracts before).
|
||||||
6. **PR-11** (M0.3): PowerDNS on vm-edge + zone file + registrar NS delegation + wildcard TLS via Let's Encrypt DNS-01.
|
6. **PR-11** (M0.3): PowerDNS on vm-edge + zone file + registrar NS delegation + wildcard TLS via Let's Encrypt DNS-01.
|
||||||
|
|
||||||
After PR-11, the dependency graph fans out and parallel work begins.
|
After PR-11, the dependency graph fans out and parallel work begins.
|
||||||
@@ -690,7 +724,7 @@ That's 18 milestones. With one full-time agent and standard human review pacing,
|
|||||||
|
|
||||||
**Parallelism opportunities:**
|
**Parallelism opportunities:**
|
||||||
- M6.x and M7.x can run fully in parallel (different repos, different stacks).
|
- M6.x and M7.x can run fully in parallel (different repos, different stacks).
|
||||||
- M8.x is independent of all data-plane work once M2.2 is done.
|
- M8.x is independent of all app-plane work once M2.2 is done.
|
||||||
- M15.x can begin as soon as M10.1 lands.
|
- M15.x can begin as soon as M10.1 lands.
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -703,7 +737,7 @@ That's 18 milestones. With one full-time agent and standard human review pacing,
|
|||||||
- ~~Cloudflare account ownership~~ → not used; DNS is self-hosted via PowerDNS on vm-edge (M0.3). Registrar account (Benjamin's) still needs documented 2FA recovery — see new DR item below.
|
- ~~Cloudflare account ownership~~ → not used; DNS is self-hosted via PowerDNS on vm-edge (M0.3). Registrar account (Benjamin's) still needs documented 2FA recovery — see new DR item below.
|
||||||
|
|
||||||
**Still open:**
|
**Still open:**
|
||||||
- **CDN host** for `cdn.yourplatform.com`: self-hosted MinIO + Caddy on vm-edge is the OSS-aligned default; alternative is BunnyCDN (cheap, EU). Decide before M6.3 (manifest bundles + hero images).
|
- **CDN host** for `cdn.breakpilot.com`: self-hosted MinIO + Caddy on vm-edge is the OSS-aligned default; alternative is BunnyCDN (cheap, EU). Decide before M6.3 (manifest bundles + hero images).
|
||||||
- **Cloud provider for port 25 outbound.** Stalwart needs unblocked port 25 to send mail. Hetzner blocks by default and requires a request to unblock with proof of intent + abuse contact; OVH and Scaleway unblock on request faster. Confirm with Benjamin which provider vm-control runs on. Block on M3.2 if port 25 is unblockable — fallback is sending via a different provider's IP with reverse DNS.
|
- **Cloud provider for port 25 outbound.** Stalwart needs unblocked port 25 to send mail. Hetzner blocks by default and requires a request to unblock with proof of intent + abuse contact; OVH and Scaleway unblock on request faster. Confirm with Benjamin which provider vm-control runs on. Block on M3.2 if port 25 is unblockable — fallback is sending via a different provider's IP with reverse DNS.
|
||||||
- **Test data privacy.** The demo tenant must contain ONLY synthetic data — confirm seed pipeline strips real PII even if our test orgs accidentally seed from prod.
|
- **Test data privacy.** The demo tenant must contain ONLY synthetic data — confirm seed pipeline strips real PII even if our test orgs accidentally seed from prod.
|
||||||
- **Registrar + DNS bus-factor.** Document who owns the registrar account, who has 2FA recovery codes, and the procedure to update NS records without that person available. Goes in `platform/docs/runbooks/dr.md` before M0.3 ships.
|
- **Registrar + DNS bus-factor.** Document who owns the registrar account, who has 2FA recovery codes, and the procedure to update NS records without that person available. Goes in `platform/docs/runbooks/dr.md` before M0.3 ships.
|
||||||
|
|||||||
+96
-75
@@ -1,55 +1,59 @@
|
|||||||
# Infrastructure Specification
|
# Infrastructure Specification
|
||||||
**Status:** Locked Topology
|
**Status:** Locked Topology
|
||||||
**Authors:** Sharang, Benjamin
|
**Authors:** Sharang, Benjamin
|
||||||
**Date:** 2026-05-11 (topology lock: 2026-05-18)
|
**Date:** 2026-05-11 (topology lock: 2026-05-18; cluster split: 2026-06-30)
|
||||||
**Companion docs:** PLATFORM_ARCHITECTURE.md, IMPLEMENTATION_PLAN.md, COST_PLAN.md
|
**Companion docs:** PLATFORM_ARCHITECTURE.md, IMPLEMENTATION_PLAN.md, COST_PLAN.md
|
||||||
**Cloud provider:** SysEleven Cloud Services (DUS2, OpenStack)
|
**Cloud provider:** SysEleven Cloud Services (DUS2, OpenStack) — multi-VM rollout gated on legal entity
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
> **2026-06-30 cluster-split decision.** The 4 VMs from the May 18 lock are unchanged in count and flavor. What changes is how they're organized: **one Orca cluster per plane**, three cluster repos (`platform/breakpilot-edge`, `platform/breakpilot-control`, `platform/breakpilot-app`). Identity and Infra are co-tenant on `vm-edge` inside `breakpilot-edge` (single-VM core; KC JVM heap pinned so it cannot starve PowerDNS / Infisical). The "Data plane" is renamed to **App plane** to reflect that it carries product workloads, not just data stores. `vm-data` is renamed `vm-app-prod`, and the standalone `stage` VM is renamed `vm-app-stage` — both live in the same `breakpilot-app` cluster. Multi-VM rollout is gated on the legal entity being established so we can sign SysEleven business contracts; until then, single-VM ops continues via `~/workspace/orca-infra`. See `IMPLEMENTATION_PLAN.md §1.1 / M1.2` and `platform/orca-platform/clusters/README.md` for the migration mechanics. **The SLA targets in §6 / §7 below are unaffected** — they apply at the plane level regardless of which cluster a plane lives in.
|
||||||
|
|
||||||
## 1. VM Inventory
|
## 1. VM Inventory
|
||||||
|
|
||||||
**Four billable VMs total.** Three in production (one per plane after collapsing Identity+Infra), one in stage. Dev runs entirely on developer laptops via docker-compose.
|
**Four billable VMs across three Orca clusters.** Dev runs entirely on developer laptops via docker-compose.
|
||||||
|
|
||||||
```
|
```
|
||||||
┌──────────────┬─────────────────┬────────────────────────┬───────────┬─────────────────┐
|
┌──────────────────┬────────────────────┬──────────┬────────────────────────┬───────────┬─────────────────┐
|
||||||
│ Name │ Env │ SysEleven flavor │ Public IP │ Planes owned │
|
│ Name │ Cluster │ Env │ SysEleven flavor │ Public IP │ Planes owned │
|
||||||
├──────────────┼─────────────────┼────────────────────────┼───────────┼─────────────────┤
|
├──────────────────┼────────────────────┼──────────┼────────────────────────┼───────────┼─────────────────┤
|
||||||
│ vm-edge │ prod │ m2.small (2v / 8 GB) │ YES (1) │ Identity + Infra│
|
│ vm-edge │ breakpilot-edge │ prod │ m2.small (2v / 8 GB) │ YES (1) │ Identity + Infra│
|
||||||
│ vm-control │ prod │ m2.medium (4v / 16 GB) │ No │ Control │
|
│ vm-control │ breakpilot-control │ prod │ m2.medium (4v / 16 GB) │ No │ Control │
|
||||||
│ vm-data │ prod │ m2.medium (4v / 16 GB) │ No │ Data │
|
│ vm-app-prod │ breakpilot-app │ prod │ m2.medium (4v / 16 GB) │ No │ App (prod) │
|
||||||
│ stage │ stage │ m2.small (2v / 8 GB) │ YES (1) │ App plane only │
|
│ vm-app-stage │ breakpilot-app │ stage │ m2.small (2v / 8 GB) │ YES (1) │ App (stage) │
|
||||||
│ (dev) │ dev │ local docker-compose │ n/a │ all (in-memory) │
|
│ (dev) │ — │ dev │ local docker-compose │ n/a │ all (in-memory) │
|
||||||
└──────────────┴─────────────────┴────────────────────────┴───────────┴─────────────────┘
|
└──────────────────┴────────────────────┴──────────┴────────────────────────┴───────────┴─────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
**Total compute:** 48 GiB-RAM, 12 vCPU. **Monthly compute net: €192 (36M upfront) / €295 (12M) / €435 (On-Demand).** See COST_PLAN.md for the full three-mode table.
|
**Total compute:** 48 GiB-RAM, 12 vCPU. **Monthly compute net: €192 (36M upfront) / €295 (12M) / €435 (On-Demand).** See COST_PLAN.md for the full three-mode table. The cluster split does not change the per-VM bill (same flavors); the additional cost is operational (3 cluster repos to keep CI green in) and is offset by the independent failure-domain wins.
|
||||||
|
|
||||||
### Why this topology and not the previous 7-VM layout
|
### Why this topology and not the previous 7-VM layout
|
||||||
|
|
||||||
The earlier draft proposed one VM per service group (vm-gateway, vm-identity, vm-secrets, vm-ops, vm-control, vm-certifai, vm-compliance). That gave maximum failure isolation but cost 132 GiB-RAM stage+prod. At 5 customers the isolation is unused — every VM ran at <10% utilisation. The locked topology buys back failure isolation incrementally as load grows (see §13 Growth Trajectory).
|
The earlier draft proposed one VM per service group (vm-gateway, vm-identity, vm-secrets, vm-ops, vm-control, vm-certifai, vm-compliance). That gave maximum failure isolation but cost 132 GiB-RAM stage+prod. At 5 customers the isolation is unused — every VM ran at <10% utilisation. The locked topology buys back failure isolation incrementally as load grows (see §13 Growth Trajectory).
|
||||||
|
|
||||||
Critical isolations preserved even at 4 VMs:
|
Critical isolations preserved even at 4 VMs:
|
||||||
- **vm-edge isolates identity from app workloads.** Keycloak JVM has its own page cache; ERPNext background jobs cannot starve token issuance.
|
- **vm-edge isolates identity from app workloads.** Keycloak JVM has its own page cache + heap pin (`-Xmx1500m`) so ERPNext or product background jobs on a different VM cannot starve token issuance — and PowerDNS / Infisical on the same VM cannot be starved by KC either.
|
||||||
- **vm-data isolates databases from stateless services.** All data-plane DBs share one host, but they're walled off from the portal + ERPNext + Stalwart competing on vm-control.
|
- **vm-app-prod isolates product databases from stateless services.** All app-plane DBs share one host, but they're walled off from the portal + ERPNext + Stalwart competing on vm-control.
|
||||||
- **stage runs the app plane only.** It calls prod Keycloak + prod Tenant Registry under `tenant.kind = stage` rather than mirroring those services.
|
- **vm-app-stage runs the app plane only.** It calls prod Keycloak (`breakpilot-edge`) + prod Tenant Registry (`breakpilot-control`) under `tenant.kind = stage` rather than mirroring those services. Stage and prod live in the **same** `breakpilot-app` Orca cluster on **different** VMs — same config, different physical workloads.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 2. Service-to-VM Mapping
|
## 2. Service-to-VM Mapping
|
||||||
|
|
||||||
|
Each VM is owned by one Orca cluster (see §1). Service manifests live in that cluster's repo at `platform/breakpilot-<name>/services/...`.
|
||||||
|
|
||||||
```
|
```
|
||||||
vm-edge (prod, m2.small 8 GB, public IP)
|
vm-edge (cluster: breakpilot-edge, prod, m2.small 8 GB, public IP)
|
||||||
├── orca-proxy (Orca-managed; wildcard TLS terminator)
|
├── orca-proxy (Orca-managed; wildcard TLS terminator)
|
||||||
├── powerdns-auth (Orca-managed; authoritative DNS for yourplatform.com)
|
├── powerdns-auth (Orca-managed; authoritative DNS for breakpilot.com)
|
||||||
├── keycloak-26 (Orca-managed; JVM, ~1.5 GB heap)
|
├── keycloak-26 (Orca-managed; JVM, ~1.5 GB heap; repurposed from CERTifAI KC at M2.1)
|
||||||
├── postgres-keycloak (Orca-managed; dedicated PG instance for Keycloak only)
|
├── postgres-keycloak (Orca-managed; dedicated PG instance for Keycloak only)
|
||||||
├── infisical (Orca-managed)
|
├── infisical (Orca-managed)
|
||||||
├── postgres-infisical (Orca-managed; dedicated PG instance for Infisical only)
|
├── postgres-infisical (Orca-managed; dedicated PG instance for Infisical only)
|
||||||
├── redis-infisical (Orca-managed; ephemeral)
|
├── redis-infisical (Orca-managed; ephemeral)
|
||||||
└── gitea (Orca-managed; SQLite backend to avoid a third PG)
|
└── gitea (Orca-managed; SQLite backend to avoid a third PG)
|
||||||
|
|
||||||
vm-control (prod, m2.medium 16 GB)
|
vm-control (cluster: breakpilot-control, prod, m2.medium 16 GB)
|
||||||
├── customer-portal (Orca-managed; Next.js)
|
├── customer-portal (Orca-managed; Next.js)
|
||||||
├── tenant-registry (Orca-managed; Go)
|
├── tenant-registry (Orca-managed; Go)
|
||||||
├── orca-controller (Orca core process; NOT a managed container)
|
├── orca-controller (Orca core process; NOT a managed container)
|
||||||
@@ -57,9 +61,9 @@ vm-control (prod, m2.medium 16 GB)
|
|||||||
├── frappe-hd (same bench as ERPNext)
|
├── frappe-hd (same bench as ERPNext)
|
||||||
├── mariadb (Orca-managed; for ERPNext)
|
├── mariadb (Orca-managed; for ERPNext)
|
||||||
├── redis-erpnext (Orca-managed)
|
├── redis-erpnext (Orca-managed)
|
||||||
└── stalwart-mail (Orca-managed; SMTP/IMAP/JMAP on mail.yourplatform.com)
|
└── stalwart-mail (Orca-managed; SMTP/IMAP/JMAP on mail.breakpilot.com)
|
||||||
|
|
||||||
vm-data (prod, m2.medium 16 GB)
|
vm-app-prod (cluster: breakpilot-app, prod, m2.medium 16 GB)
|
||||||
├── certifai-dashboard (Orca-managed)
|
├── certifai-dashboard (Orca-managed)
|
||||||
├── mongodb (Orca-managed)
|
├── mongodb (Orca-managed)
|
||||||
├── litellm (Orca-managed)
|
├── litellm (Orca-managed)
|
||||||
@@ -70,22 +74,23 @@ vm-data (prod, m2.medium 16 GB)
|
|||||||
├── qdrant (Orca-managed)
|
├── qdrant (Orca-managed)
|
||||||
└── minio (Orca-managed)
|
└── minio (Orca-managed)
|
||||||
|
|
||||||
stage (stage, m2.small 8 GB, public IP)
|
vm-app-stage (cluster: breakpilot-app — SAME cluster as vm-app-prod, stage, m2.small 8 GB, public IP)
|
||||||
├── orca-proxy (light; only routes to stage app)
|
├── orca-proxy (light; only routes to stage app)
|
||||||
├── customer-portal (NEW VERSION under test)
|
├── customer-portal (NEW VERSION under test)
|
||||||
├── tenant-registry (NEW VERSION under test, talks to ephemeral PG below)
|
├── tenant-registry (placeholder; stage actually calls PROD tenant-registry — manifest exists for parity, see §5 isolation rules)
|
||||||
├── certifai-dashboard (NEW VERSION under test)
|
├── certifai-dashboard (NEW VERSION under test)
|
||||||
├── backend-compliance (NEW VERSION under test)
|
├── backend-compliance (NEW VERSION under test)
|
||||||
├── ai-compliance-sdk (NEW VERSION under test)
|
├── ai-compliance-sdk (NEW VERSION under test)
|
||||||
├── admin-compliance (NEW VERSION under test)
|
├── admin-compliance (NEW VERSION under test)
|
||||||
├── litellm (light; same image as prod)
|
├── litellm (light; same image as prod)
|
||||||
├── postgres-app-stage (ephemeral; lives entirely on stage VM)
|
├── postgres-app-stage (ephemeral; lives entirely on vm-app-stage)
|
||||||
├── mongodb-stage (ephemeral)
|
├── mongodb-stage (ephemeral)
|
||||||
└── qdrant-stage (ephemeral, tiny corpus)
|
└── qdrant-stage (ephemeral, tiny corpus)
|
||||||
|
|
||||||
Calls OUT to prod:
|
Calls OUT to prod:
|
||||||
→ auth.yourplatform.com (Keycloak token issuance, under stage client_id)
|
→ auth.breakpilot.com (PROD Keycloak in breakpilot-edge, under stage client_id, tenant.kind = "stage")
|
||||||
→ mail.yourplatform.com (Stalwart SMTP, recipient filter forces +stage@ only)
|
→ registry.breakpilot.com (PROD tenant-registry in breakpilot-control, read-only for stage tenants)
|
||||||
|
→ mail.breakpilot.com (PROD Stalwart in breakpilot-control, recipient filter forces +stage@ only)
|
||||||
→ Polar SANDBOX webhook URL (NEVER prod Polar)
|
→ Polar SANDBOX webhook URL (NEVER prod Polar)
|
||||||
→ no calls to prod Postgres-app, MariaDB, MongoDB
|
→ no calls to prod Postgres-app, MariaDB, MongoDB
|
||||||
```
|
```
|
||||||
@@ -107,8 +112,8 @@ stage (stage, m2.small 8 GB, public IP)
|
|||||||
```
|
```
|
||||||
INTERNET
|
INTERNET
|
||||||
│
|
│
|
||||||
(yourplatform.com — authoritative on vm-edge PowerDNS;
|
(breakpilot.com — authoritative on vm-edge PowerDNS;
|
||||||
stage.yourplatform.com — authoritative same zone)
|
stage.breakpilot.com — authoritative same zone)
|
||||||
│
|
│
|
||||||
┌─────────────┴─────────────┐
|
┌─────────────┴─────────────┐
|
||||||
│ │
|
│ │
|
||||||
@@ -129,7 +134,7 @@ stage (stage, m2.small 8 GB, public IP)
|
|||||||
┌────────┴─────────┐
|
┌────────┴─────────┐
|
||||||
│ │
|
│ │
|
||||||
┌──────▼───────┐ ┌───────▼──────┐
|
┌──────▼───────┐ ┌───────▼──────┐
|
||||||
│ vm-control │ │ vm-data │
|
│ vm-control │ │ vm-app-prod │
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│ portal │ │ certifai │
|
│ portal │ │ certifai │
|
||||||
│ tenant-reg │ │ mongodb │
|
│ tenant-reg │ │ mongodb │
|
||||||
@@ -143,15 +148,15 @@ stage (stage, m2.small 8 GB, public IP)
|
|||||||
└──────────────┘
|
└──────────────┘
|
||||||
|
|
||||||
Orca-Proxy routing (vm-edge, by Host header):
|
Orca-Proxy routing (vm-edge, by Host header):
|
||||||
auth.yourplatform.com → 127.0.0.1:8443 (Keycloak, local on vm-edge)
|
auth.breakpilot.com → 127.0.0.1:8443 (Keycloak, local on vm-edge)
|
||||||
erp.yourplatform.com → vm-control:8000 (ERPNext) [allowlist: our IPs only]
|
erp.breakpilot.com → vm-control:8000 (ERPNext) [allowlist: our IPs only]
|
||||||
git.yourplatform.com → vm-edge:3000 (Gitea, local) [allowlist: our IPs only]
|
git.breakpilot.com → vm-edge:3000 (Gitea, local) [allowlist: our IPs only]
|
||||||
mail.yourplatform.com → vm-control:587 (Stalwart submission) [allowlist: VM internal only]
|
mail.breakpilot.com → vm-control:587 (Stalwart submission) [allowlist: VM internal only]
|
||||||
ns1.yourplatform.com → 127.0.0.1:53 (PowerDNS, local)
|
ns1.breakpilot.com → 127.0.0.1:53 (PowerDNS, local)
|
||||||
*.yourplatform.com → vm-control:3000 (customer portal)
|
*.breakpilot.com → vm-control:3000 (customer portal)
|
||||||
|
|
||||||
Orca-Proxy routing (stage, by Host header):
|
Orca-Proxy routing (stage, by Host header):
|
||||||
*.stage.yourplatform.com → 127.0.0.1:3000 (stage portal — all subdomains route here)
|
*.stage.breakpilot.com → 127.0.0.1:3000 (stage portal — all subdomains route here)
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -166,7 +171,7 @@ Block volumes (Ceph 3x replicated, €0.10/GiB/mo) mounted to each VM.
|
|||||||
├──────────────┼───────────────────────────────────────────┼─────────┼─────────────────────┤
|
├──────────────┼───────────────────────────────────────────┼─────────┼─────────────────────┤
|
||||||
│ vm-edge │ pg-keycloak + pg-infisical + Gitea repos │ +50 GB │ Slow │
|
│ vm-edge │ pg-keycloak + pg-infisical + Gitea repos │ +50 GB │ Slow │
|
||||||
│ vm-control │ MariaDB (ERPNext) + Stalwart mail spool │ +250 GB │ Medium │
|
│ vm-control │ MariaDB (ERPNext) + Stalwart mail spool │ +250 GB │ Medium │
|
||||||
│ vm-data │ MongoDB + pg-app + Qdrant + MinIO │ +500 GB │ Fast (scales w/ N) │
|
│ vm-app-prod │ MongoDB + pg-app + Qdrant + MinIO │ +500 GB │ Fast (scales w/ N) │
|
||||||
│ stage │ pg-stage + mongo-stage + qdrant-stage │ +50 GB │ Resets per release │
|
│ stage │ pg-stage + mongo-stage + qdrant-stage │ +50 GB │ Resets per release │
|
||||||
└──────────────┴───────────────────────────────────────────┴─────────┴─────────────────────┘
|
└──────────────┴───────────────────────────────────────────┴─────────┴─────────────────────┘
|
||||||
|
|
||||||
@@ -200,10 +205,10 @@ All backups ship to **SysEleven Object Storage** (S3-compatible, geo-redundant D
|
|||||||
│ Infisical store │ encrypted → S3 │ Daily │ 30 days │ Infra Plane │
|
│ Infisical store │ encrypted → S3 │ Daily │ 30 days │ Infra Plane │
|
||||||
│ MariaDB (vm-control) │ mysqldump → S3 │ Every 6h │ 30 days │ Control Plane │
|
│ MariaDB (vm-control) │ mysqldump → S3 │ Every 6h │ 30 days │ Control Plane │
|
||||||
│ Stalwart queue/store │ tar → S3 │ Daily │ 7 days │ Control Plane │
|
│ Stalwart queue/store │ tar → S3 │ Daily │ 7 days │ Control Plane │
|
||||||
│ pg-app (vm-data) │ pg_dump → S3-geo │ Every 6h │ 30 days │ Data Plane (owns RPO)│
|
│ pg-app (vm-app-prod) │ pg_dump → S3-geo │ Every 6h │ 30 days │ App Plane (owns RPO) │
|
||||||
│ MongoDB (vm-data) │ mongodump → S3 │ Daily │ 30 days │ Data Plane │
|
│ MongoDB (vm-app-prod) │ mongodump → S3 │ Daily │ 30 days │ App Plane │
|
||||||
│ MinIO (vm-data) │ mc mirror → S3 │ Daily │ 90 days │ Data Plane │
|
│ MinIO (vm-app-prod) │ mc mirror → S3 │ Daily │ 90 days │ App Plane │
|
||||||
│ Qdrant (vm-data) │ API snap → S3 │ Daily │ 14 days │ Data Plane (rebuild) │
|
│ Qdrant (vm-app-prod) │ API snap → S3 │ Daily │ 14 days │ App Plane (rebuild) │
|
||||||
│ stage * │ no backup │ — │ — │ — (ephemeral) │
|
│ stage * │ no backup │ — │ — │ — (ephemeral) │
|
||||||
│ Orca config (IaC) │ Gitea (VCS) │ On commit │ Forever │ Infra Plane │
|
│ Orca config (IaC) │ Gitea (VCS) │ On commit │ Forever │ Infra Plane │
|
||||||
└───────────────────────┴──────────────────┴────────────┴────────────┴──────────────────────┘
|
└───────────────────────┴──────────────────┴────────────┴────────────┴──────────────────────┘
|
||||||
@@ -254,11 +259,11 @@ STAGE_ISOLATION— stage tenant cannot mutate any prod data; reads-only against
|
|||||||
|
|
||||||
### Plane ownership of constraints
|
### Plane ownership of constraints
|
||||||
|
|
||||||
Even though planes now share VMs, the **ownership model is unchanged** — the plane that owns a constraint owns it regardless of which VM hosts the service. The Infra Plane (now collapsed onto vm-edge alongside the Identity plane) still mechanically enforces backup, IaC, secrets, and network constraints.
|
Planes are the **SLA + ownership abstraction**; clusters are the **operational unit**. The plane that owns a constraint owns it regardless of which cluster hosts the service. Identity and Infra are co-tenant in the `breakpilot-edge` cluster on `vm-edge`; they remain distinct planes for purposes of SLA + ownership accounting.
|
||||||
|
|
||||||
```
|
```
|
||||||
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
||||||
║ IDENTITY PLANE (on vm-edge) ║
|
║ IDENTITY PLANE (cluster: breakpilot-edge, on vm-edge) ║
|
||||||
║ ║
|
║ ║
|
||||||
║ Owns / defines: ║
|
║ Owns / defines: ║
|
||||||
║ AVAILABILITY — must be ≥ 99.5% (root dep for everything) ║
|
║ AVAILABILITY — must be ≥ 99.5% (root dep for everything) ║
|
||||||
@@ -268,11 +273,16 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
|||||||
║ STAGE_ISOLATION— rate-limits stage_client_id; rejects stage JWTs in prod audiences ║
|
║ STAGE_ISOLATION— rate-limits stage_client_id; rejects stage JWTs in prod audiences ║
|
||||||
║ ║
|
║ ║
|
||||||
║ Co-tenant note: shares vm-edge with Infra Plane services. JVM heap pinned to 1.5 GB ║
|
║ Co-tenant note: shares vm-edge with Infra Plane services. JVM heap pinned to 1.5 GB ║
|
||||||
║ in Orca manifest so it cannot starve PowerDNS / Infisical. ║
|
║ in Orca manifest (-Xmx1500m) so it cannot starve PowerDNS / Infisical. Escape hatch ║
|
||||||
|
║ if the heap fight shows up: flip placement.node to a new vm-edge-identity VM inside ║
|
||||||
|
║ the SAME breakpilot-edge cluster — no schema migration, just a container move. ║
|
||||||
|
║ ║
|
||||||
|
║ KC instance is REPURPOSED from the existing CERTifAI Keycloak at M2.1 (realm export ║
|
||||||
|
║ + user database move; hostname swap), not stood up fresh. ║
|
||||||
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
||||||
║ CONTROL PLANE (on vm-control) ║
|
║ CONTROL PLANE (cluster: breakpilot-control, on vm-control) ║
|
||||||
║ ║
|
║ ║
|
||||||
║ Owns / defines: ║
|
║ Owns / defines: ║
|
||||||
║ RPO (tenant) — tenant registry & compliance schemas RPO ≤ 6h ║
|
║ RPO (tenant) — tenant registry & compliance schemas RPO ≤ 6h ║
|
||||||
@@ -288,10 +298,13 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
|||||||
║ mariadb: 3 GB memory cap ║
|
║ mariadb: 3 GB memory cap ║
|
||||||
║ stalwart: 1 GB memory cap ║
|
║ stalwart: 1 GB memory cap ║
|
||||||
║ tenant-registry: 500 MB ║
|
║ tenant-registry: 500 MB ║
|
||||||
|
║ ║
|
||||||
|
║ Stage callers: vm-app-stage in breakpilot-app calls THIS tenant-registry under ║
|
||||||
|
║ tenant.kind = "stage" (read-only for stage tenants). No duplicated tenant-registry. ║
|
||||||
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
||||||
║ DATA PLANE (on vm-data) ║
|
║ APP PLANE (was "Data Plane"; cluster: breakpilot-app, on vm-app-prod + vm-app-stage) ║
|
||||||
║ ║
|
║ ║
|
||||||
║ Owns / defines: ║
|
║ Owns / defines: ║
|
||||||
║ DATA_RESIDENCY — all customer data (MongoDB, pg-app, MinIO) must stay EU ║
|
║ DATA_RESIDENCY — all customer data (MongoDB, pg-app, MinIO) must stay EU ║
|
||||||
@@ -300,12 +313,17 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
|||||||
║ AUDIT_TRAIL — product-level actions ║
|
║ AUDIT_TRAIL — product-level actions ║
|
||||||
║ AVAILABILITY — CERTifAI ≥ 99.5%; compliance ≥ 99.5% ║
|
║ AVAILABILITY — CERTifAI ≥ 99.5%; compliance ≥ 99.5% ║
|
||||||
║ ║
|
║ ║
|
||||||
║ Co-tenant note: this VM is the SCALE driver. When vm-data hits 80% RAM, bump flavor ║
|
║ Co-tenant note: vm-app-prod is the SCALE driver. When it hits 80% RAM, bump flavor ║
|
||||||
║ (m2.medium → m2.large → m2.xlarge). See §13 Growth Trajectory. ║
|
║ (m2.medium → m2.large → m2.xlarge). See §13 Growth Trajectory. ║
|
||||||
|
║ ║
|
||||||
|
║ Stage VM (vm-app-stage) lives in the SAME cluster — same config, different physical ║
|
||||||
|
║ workload. Stage authenticates against the prod Identity plane (breakpilot-edge KC) ║
|
||||||
|
║ and reads tenant config from the prod Control plane (breakpilot-control ║
|
||||||
|
║ tenant-registry) — never duplicates identity or control. ║
|
||||||
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
╔══════════════════════════════════════════════════════════════════════════════════════════╗
|
||||||
║ INFRA PLANE (on vm-edge, alongside Identity) ║
|
║ INFRA PLANE (cluster: breakpilot-edge, on vm-edge, alongside Identity) ║
|
||||||
║ ║
|
║ ║
|
||||||
║ Owns / enforces ALL of: ║
|
║ Owns / enforces ALL of: ║
|
||||||
║ BACKUP — executes all backup jobs (pg_dump, mongodump, mc mirror) ║
|
║ BACKUP — executes all backup jobs (pg_dump, mongodump, mc mirror) ║
|
||||||
@@ -317,6 +335,9 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
|||||||
║ AVAILABILITY — Orca restart policies, health checks ║
|
║ AVAILABILITY — Orca restart policies, health checks ║
|
||||||
║ COLD_START — enforces startup ordering (see §10 Scenario F) ║
|
║ COLD_START — enforces startup ordering (see §10 Scenario F) ║
|
||||||
║ STAGE_ISOLATION— Infisical secret-path scoping for stage_app identity ║
|
║ STAGE_ISOLATION— Infisical secret-path scoping for stage_app identity ║
|
||||||
|
║ ║
|
||||||
|
║ Per-cluster note: each of the three cluster repos has its own [backup] block + S3 ║
|
||||||
|
║ bucket. A runaway backup on breakpilot-app cannot fill breakpilot-edge's bucket. ║
|
||||||
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
╚══════════════════════════════════════════════════════════════════════════════════════════╝
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -339,14 +360,14 @@ Even though planes now share VMs, the **ownership model is unchanged** — the p
|
|||||||
│ Frappe HD │ 99% │ 60 min │ 24h │ vm-control │
|
│ Frappe HD │ 99% │ 60 min │ 24h │ vm-control │
|
||||||
│ MariaDB │ 99.5% │ 20 min │ 6h │ vm-control │
|
│ MariaDB │ 99.5% │ 20 min │ 6h │ vm-control │
|
||||||
│ Stalwart Mail │ 99% │ 60 min │ 24h │ vm-control │
|
│ Stalwart Mail │ 99% │ 60 min │ 24h │ vm-control │
|
||||||
│ CERTifAI │ 99.5% │ 10 min │ 24h │ vm-data │
|
│ CERTifAI │ 99.5% │ 10 min │ 24h │ vm-app-prod │
|
||||||
│ MongoDB │ 99.5% │ 20 min │ 24h │ vm-data │
|
│ MongoDB │ 99.5% │ 20 min │ 24h │ vm-app-prod │
|
||||||
│ LiteLLM │ 99% │ 5 min │ N/A │ vm-data │
|
│ LiteLLM │ 99% │ 5 min │ N/A │ vm-app-prod │
|
||||||
│ backend-compliance │ 99.5% │ 10 min │ 6h │ vm-data │
|
│ backend-compliance │ 99.5% │ 10 min │ 6h │ vm-app-prod │
|
||||||
│ ai-compliance-sdk │ 99.5% │ 10 min │ 6h │ vm-data │
|
│ ai-compliance-sdk │ 99.5% │ 10 min │ 6h │ vm-app-prod │
|
||||||
│ pg-app │ 99.9% │ 20 min │ 6h │ vm-data (SPOF — RISK-1) │
|
│ pg-app │ 99.9% │ 20 min │ 6h │ vm-app-prod (SPOF — RISK-1) │
|
||||||
│ MinIO │ 99.5% │ 30 min │ 24h │ vm-data │
|
│ MinIO │ 99.5% │ 30 min │ 24h │ vm-app-prod │
|
||||||
│ Qdrant │ 99% │ 2h │ 24h │ vm-data (rebuildable) │
|
│ Qdrant │ 99% │ 2h │ 24h │ vm-app-prod (rebuildable) │
|
||||||
│ stage (any service) │ 95% │ best ef.│ N/A │ stage (ephemeral; no SLA) │
|
│ stage (any service) │ 95% │ best ef.│ N/A │ stage (ephemeral; no SLA) │
|
||||||
└───────────────────────┴──────────────┴─────────┴─────────┴────────────────────────────────┘
|
└───────────────────────┴──────────────┴─────────┴─────────┴────────────────────────────────┘
|
||||||
```
|
```
|
||||||
@@ -433,13 +454,13 @@ Arrows = "requires to function." Dashed = soft (degrades, doesn't fail).
|
|||||||
│ │ mariadb + redis-erp ──► erpnext + frappe-hd │ │
|
│ │ mariadb + redis-erp ──► erpnext + frappe-hd │ │
|
||||||
│ │ (intra) ─────────────► stalwart │ │
|
│ │ (intra) ─────────────► stalwart │ │
|
||||||
│ │ ──────────────────────► customer-portal │ │
|
│ │ ──────────────────────► customer-portal │ │
|
||||||
│ │ ──────────────────────► tenant-registry ──► pg-app (vm-data)│ │
|
│ │ ──────────────────────► tenant-registry ──► pg-app (vm-app-prod)│ │
|
||||||
│ └────────────────────────────────────────────────────────────┘ │
|
│ └────────────────────────────────────────────────────────────┘ │
|
||||||
│ │ tenant-registry API │
|
│ │ tenant-registry API │
|
||||||
└────────────────────────────┼─────────────────────────────────────┘
|
└────────────────────────────┼─────────────────────────────────────┘
|
||||||
▼
|
▼
|
||||||
┌──────────────────────────────────────────────────────────────────┐
|
┌──────────────────────────────────────────────────────────────────┐
|
||||||
│ vm-data (Data) │
|
│ vm-app-prod (Data) │
|
||||||
│ ┌────────────────────────────────────────────────────────────┐ │
|
│ ┌────────────────────────────────────────────────────────────┐ │
|
||||||
│ │ mongodb ───► certifai ◄── (vm-edge JWKS, vm-edge secrets) │ │
|
│ │ mongodb ───► certifai ◄── (vm-edge JWKS, vm-edge secrets) │ │
|
||||||
│ │ litellm ───► certifai, ai-compliance-sdk │ │
|
│ │ litellm ───► certifai, ai-compliance-sdk │ │
|
||||||
@@ -470,9 +491,9 @@ Arrows = "requires to function." Dashed = soft (degrades, doesn't fail).
|
|||||||
├──► keycloak (vm-edge) ──► pg-keycloak (intra-VM)
|
├──► keycloak (vm-edge) ──► pg-keycloak (intra-VM)
|
||||||
│
|
│
|
||||||
└──► customer-portal (vm-control)
|
└──► customer-portal (vm-control)
|
||||||
├──► tenant-registry (vm-control) ──► pg-app (vm-data)
|
├──► tenant-registry (vm-control) ──► pg-app (vm-app-prod)
|
||||||
├──► certifai (vm-data) ──► mongodb (intra-VM)
|
├──► certifai (vm-app-prod) ──► mongodb (intra-VM)
|
||||||
└──► backend-compliance (vm-data) ──► pg-app (intra-VM)
|
└──► backend-compliance (vm-app-prod) ──► pg-app (intra-VM)
|
||||||
──► ai-sdk ──► qdrant + minio
|
──► ai-sdk ──► qdrant + minio
|
||||||
──► litellm ──► [external AI APIs]
|
──► litellm ──► [external AI APIs]
|
||||||
```
|
```
|
||||||
@@ -515,7 +536,7 @@ Impact: customer-portal: DOWN → /[slug]/* all return 503
|
|||||||
ERPNext + Frappe HD: DOWN → we cannot create sales orders or read tickets
|
ERPNext + Frappe HD: DOWN → we cannot create sales orders or read tickets
|
||||||
Stalwart: DOWN → no outbound emails (trial nudges, exports, ticket replies)
|
Stalwart: DOWN → no outbound emails (trial nudges, exports, ticket replies)
|
||||||
MariaDB: DOWN → ERPNext queries fail; backups paused
|
MariaDB: DOWN → ERPNext queries fail; backups paused
|
||||||
Products (CERTifAI, compliance): UNAFFECTED (on vm-data, JWTs still validate)
|
Products (CERTifAI, compliance): UNAFFECTED (on vm-app-prod, JWTs still validate)
|
||||||
Existing logged-in users: can use products directly via product subdomain
|
Existing logged-in users: can use products directly via product subdomain
|
||||||
IF they bookmark it; portal home is 503.
|
IF they bookmark it; portal home is 503.
|
||||||
Cascade: T+0: portal 503; new tenant onboarding blocked (registry down)
|
Cascade: T+0: portal 503; new tenant onboarding blocked (registry down)
|
||||||
@@ -532,7 +553,7 @@ Cost of fix at Tier B/C: split vm-control → vm-portal + vm-ops (ERPNext)
|
|||||||
— €64/mo extra at m2.small
|
— €64/mo extra at m2.small
|
||||||
```
|
```
|
||||||
|
|
||||||
### Scenario C — vm-data fails
|
### Scenario C — vm-app-prod fails
|
||||||
|
|
||||||
```
|
```
|
||||||
Impact: tenant-registry queries: FAIL (pg-app down) → portal returns 503 for tenant lookup
|
Impact: tenant-registry queries: FAIL (pg-app down) → portal returns 503 for tenant lookup
|
||||||
@@ -543,7 +564,7 @@ Impact: tenant-registry queries: FAIL (pg-app down) → portal returns 503 fo
|
|||||||
Cascade: T+0: products down; portal degraded
|
Cascade: T+0: products down; portal degraded
|
||||||
T+15m: support tickets pile up
|
T+15m: support tickets pile up
|
||||||
Note: prod is partial — users see error pages but ERPNext + auth still work
|
Note: prod is partial — users see error pages but ERPNext + auth still work
|
||||||
Recovery: Restart vm-data containers. If pg-app corrupt: restore from pg_dump (RPO 6h).
|
Recovery: Restart vm-app-prod containers. If pg-app corrupt: restore from pg_dump (RPO 6h).
|
||||||
RTO target: 20 min
|
RTO target: 20 min
|
||||||
Mitigation: This is the SCALE-event VM. RISK-1 below makes this the worst SPOF:
|
Mitigation: This is the SCALE-event VM. RISK-1 below makes this the worst SPOF:
|
||||||
one pg-app instance owns tenant_registry + compliance schemas.
|
one pg-app instance owns tenant_registry + compliance schemas.
|
||||||
@@ -561,11 +582,11 @@ Impact: CERTifAI: AI features fail (summarization, chat completion).
|
|||||||
Compliance CRUD: UNAFFECTED.
|
Compliance CRUD: UNAFFECTED.
|
||||||
Cascade: Soft degradation only. Products show "AI features temporarily unavailable" banner.
|
Cascade: Soft degradation only. Products show "AI features temporarily unavailable" banner.
|
||||||
Deadlock: None.
|
Deadlock: None.
|
||||||
Recovery: Restart LiteLLM on vm-data (stateless, ~30s).
|
Recovery: Restart LiteLLM on vm-app-prod (stateless, ~30s).
|
||||||
Severity: MEDIUM — graceful degradation by design
|
Severity: MEDIUM — graceful degradation by design
|
||||||
```
|
```
|
||||||
|
|
||||||
### Scenario E — Stage VM compromised or buggy
|
### Scenario E — vm-app-stage compromised or buggy
|
||||||
|
|
||||||
```
|
```
|
||||||
Impact: On stage itself: stage portal serves bad data; stage testers see errors.
|
Impact: On stage itself: stage portal serves bad data; stage testers see errors.
|
||||||
@@ -587,7 +608,7 @@ Severity: LOW (in prod) / HIGH (on stage, but stage SLA is 95%)
|
|||||||
Three VMs boot at once. Services must start in dependency order or services
|
Three VMs boot at once. Services must start in dependency order or services
|
||||||
crash-loop until their deps are ready.
|
crash-loop until their deps are ready.
|
||||||
|
|
||||||
DEADLOCK RISK: vm-control services (portal, tenant-registry) start before vm-data
|
DEADLOCK RISK: vm-control services (portal, tenant-registry) start before vm-app-prod
|
||||||
services (pg-app, certifai, compliance). They'll crash-loop ~2-5min
|
services (pg-app, certifai, compliance). They'll crash-loop ~2-5min
|
||||||
with backoff retries.
|
with backoff retries.
|
||||||
Same for ERPNext on vm-control trying to reach Keycloak on vm-edge.
|
Same for ERPNext on vm-control trying to reach Keycloak on vm-edge.
|
||||||
@@ -598,7 +619,7 @@ RESOLUTION: Orca enforces cross-VM startup ordering via health-check dependencie
|
|||||||
|
|
||||||
Required cold start sequence:
|
Required cold start sequence:
|
||||||
|
|
||||||
Phase 0 — Data roots on vm-data (parallel):
|
Phase 0 — Data roots on vm-app-prod (parallel):
|
||||||
pg-app, mongodb, qdrant, minio
|
pg-app, mongodb, qdrant, minio
|
||||||
Phase 0 — Data roots on vm-control (parallel):
|
Phase 0 — Data roots on vm-control (parallel):
|
||||||
mariadb, redis-erpnext
|
mariadb, redis-erpnext
|
||||||
@@ -613,7 +634,7 @@ Required cold start sequence:
|
|||||||
keycloak (needs: pg-keycloak [Phase 0], infisical [Phase 1])
|
keycloak (needs: pg-keycloak [Phase 0], infisical [Phase 1])
|
||||||
gitea (needs: sqlite; ready from Phase 0)
|
gitea (needs: sqlite; ready from Phase 0)
|
||||||
|
|
||||||
Phase 3 — Control on vm-control + Data services on vm-data (parallel):
|
Phase 3 — Control on vm-control + Data services on vm-app-prod (parallel):
|
||||||
tenant-registry (needs: keycloak [Phase 2], pg-app [Phase 0, remote])
|
tenant-registry (needs: keycloak [Phase 2], pg-app [Phase 0, remote])
|
||||||
erpnext + frappe-hd (needs: mariadb, redis-erpnext [Phase 0], keycloak [Phase 2])
|
erpnext + frappe-hd (needs: mariadb, redis-erpnext [Phase 0], keycloak [Phase 2])
|
||||||
stalwart (needs: infisical [Phase 1])
|
stalwart (needs: infisical [Phase 1])
|
||||||
@@ -642,7 +663,7 @@ Impact: Portal cannot resolve tenant from subdomain → /[slug]/* all 503
|
|||||||
Cascade: New logins degraded.
|
Cascade: New logins degraded.
|
||||||
Existing sessions continue.
|
Existing sessions continue.
|
||||||
Deadlock: None.
|
Deadlock: None.
|
||||||
Recovery: Restart tenant-registry on vm-control. pg-app on vm-data must be healthy.
|
Recovery: Restart tenant-registry on vm-control. pg-app on vm-app-prod must be healthy.
|
||||||
RTO target: ≤ 60s
|
RTO target: ≤ 60s
|
||||||
Mitigation: Portal caches slug → tenant mapping with 60s TTL.
|
Mitigation: Portal caches slug → tenant mapping with 60s TTL.
|
||||||
Short outage invisible to customers.
|
Short outage invisible to customers.
|
||||||
@@ -683,7 +704,7 @@ stage-app │ │ │ │ │ │ │
|
|||||||
## 12. Open Infrastructure Risks (Priority Order)
|
## 12. Open Infrastructure Risks (Priority Order)
|
||||||
|
|
||||||
```
|
```
|
||||||
RISK-1 pg-app (vm-data) is a single instance serving tenant_registry + compliance schemas.
|
RISK-1 pg-app (vm-app-prod) is a single instance serving tenant_registry + compliance schemas.
|
||||||
One crash blocks portal AND compliance product simultaneously.
|
One crash blocks portal AND compliance product simultaneously.
|
||||||
→ Mitigation: split into pg-registry + pg-compliance at Tier B (200 customers).
|
→ Mitigation: split into pg-registry + pg-compliance at Tier B (200 customers).
|
||||||
Move pg-registry to its own DBaaS PostgreSQL cluster (€213/mo).
|
Move pg-registry to its own DBaaS PostgreSQL cluster (€213/mo).
|
||||||
@@ -738,10 +759,10 @@ The locked 4-VM topology is right for 5–~200 customers. Past that, expect to a
|
|||||||
```
|
```
|
||||||
Tier A (5–200 cust): 4 VMs as locked €192/mo compute (36M upfront)
|
Tier A (5–200 cust): 4 VMs as locked €192/mo compute (36M upfront)
|
||||||
↓
|
↓
|
||||||
Tier B (200–500): Bump vm-data m2.med → m2.large +€64/mo
|
Tier B (200–500): Bump vm-app-prod m2.med → m2.large +€64/mo
|
||||||
Add cold-standby vm-edge-spare +€0 (idle, paid only on swap)
|
Add cold-standby vm-edge-spare +€0 (idle, paid only on swap)
|
||||||
↓
|
↓
|
||||||
Tier C (500–1000): Split vm-data: vm-data + vm-data-db +€64/mo
|
Tier C (500–1000): Split vm-app-prod: vm-app-prod + vm-app-prod-db +€64/mo
|
||||||
(postgres-app moves to its own VM, or DBaaS cluster +€213/mo)
|
(postgres-app moves to its own VM, or DBaaS cluster +€213/mo)
|
||||||
Split vm-control: vm-control + vm-ops +€64/mo
|
Split vm-control: vm-control + vm-ops +€64/mo
|
||||||
(ERPNext + MariaDB + Stalwart move to vm-ops)
|
(ERPNext + MariaDB + Stalwart move to vm-ops)
|
||||||
@@ -749,7 +770,7 @@ Tier C (500–1000): Split vm-data: vm-data + vm-data-db +€64/mo
|
|||||||
Tier D (1000–2000): Split vm-edge: vm-edge + vm-identity + vm-secrets +€96/mo
|
Tier D (1000–2000): Split vm-edge: vm-edge + vm-identity + vm-secrets +€96/mo
|
||||||
HA Keycloak active-passive on 2× vm-identity +€32/mo
|
HA Keycloak active-passive on 2× vm-identity +€32/mo
|
||||||
Octavia Load Balancer Double Instance +€58/mo
|
Octavia Load Balancer Double Instance +€58/mo
|
||||||
vm-data m2.large → m2.xlarge or 2× +€128–256/mo
|
vm-app-prod m2.large → m2.xlarge or 2× +€128–256/mo
|
||||||
↓
|
↓
|
||||||
Final topology ≈ 8 prod VMs + DBaaS
|
Final topology ≈ 8 prod VMs + DBaaS
|
||||||
```
|
```
|
||||||
|
|||||||
+47
-33
@@ -1,10 +1,12 @@
|
|||||||
# Platform Architecture — B2B Customer Portal
|
# Platform Architecture — B2B Customer Portal
|
||||||
**Status:** Design Draft
|
**Status:** Design Draft
|
||||||
**Authors:** Sharang, Benjamin
|
**Authors:** Sharang, Benjamin
|
||||||
**Date:** 2026-05-11
|
**Date:** 2026-05-11 (cluster split: 2026-06-30)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
> **2026-06-30 cluster-split decision.** The four-plane vocabulary in this document still applies as the **SLA + ownership abstraction**. What changed is the operational shape: planes are now hosted by **three Orca clusters** (one cluster per plane, with Identity and Infra co-tenant in the same cluster on a single VM). The "Data plane" is renamed **App plane** to reflect that it carries product workloads, not just data stores. Cluster repos: `platform/breakpilot-edge` (Identity + Infra), `platform/breakpilot-control` (Control), `platform/breakpilot-app` (App, with prod + stage VMs). Multi-VM rollout is gated on the legal entity being established. See `INFRASTRUCTURE.md §1`, `IMPLEMENTATION_PLAN.md §1.1 / M1.2`, and `platform/orca-platform/clusters/README.md` for the operational details.
|
||||||
|
|
||||||
## 1. Vision
|
## 1. Vision
|
||||||
|
|
||||||
We sell CERTifAI and breakpilot-compliance as modular B2B building blocks. Customers buy one or both and operate them inside a unified customer portal — without needing to understand that they are separate products under the hood.
|
We sell CERTifAI and breakpilot-compliance as modular B2B building blocks. Customers buy one or both and operate them inside a unified customer portal — without needing to understand that they are separate products under the hood.
|
||||||
@@ -28,24 +30,32 @@ Out of scope: breakpilot-dataroom, breakpilot-lehrer, breakpilot-pitch-deck.
|
|||||||
|
|
||||||
## 3. The Four Planes
|
## 3. The Four Planes
|
||||||
|
|
||||||
|
Four logical planes — but only **three Orca clusters**. Identity and Infra are co-tenant in the `breakpilot-edge` cluster on `vm-edge` per the 2026-06-30 cluster-split decision. The plane abstraction owns SLA and audit responsibilities; the cluster abstraction owns operational concerns (failure domain, backup policy, CI scope, IaC repo).
|
||||||
|
|
||||||
```
|
```
|
||||||
╔══════════════════════════════════════════════════════════════════╗
|
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||||
║ PLANE 1 — IDENTITY (logical root, all auth flows through here) ║
|
║ PLANE 1 — IDENTITY cluster: breakpilot-edge VM: vm-edge ║
|
||||||
╚══════════════════════════════════════════════════════════════════╝
|
║ (logical root, all auth flows through here) ║
|
||||||
|
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||||
↓ JWT
|
↓ JWT
|
||||||
╔══════════════════════════════════════════════════════════════════╗
|
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||||
║ PLANE 2 — CONTROL (portal + ERPNext + tenant registry) ║
|
║ PLANE 2 — CONTROL cluster: breakpilot-control VM: vm-control ║
|
||||||
╚══════════════════════════════════════════════════════════════════╝
|
║ (portal + ERPNext + tenant registry) ║
|
||||||
|
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||||
↓ tenant-scoped API calls
|
↓ tenant-scoped API calls
|
||||||
╔══════════════════════════════════════════════════════════════════╗
|
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||||
║ PLANE 3 — DATA (CERTifAI + breakpilot-compliance) ║
|
║ PLANE 3 — APP (was DATA) cluster: breakpilot-app VMs: vm-app-prod ║
|
||||||
╚══════════════════════════════════════════════════════════════════╝
|
║ (CERTifAI + breakpilot-compliance + data stores) + vm-app-stage║
|
||||||
|
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||||
↓ everything runs on
|
↓ everything runs on
|
||||||
╔══════════════════════════════════════════════════════════════════╗
|
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||||
║ PLANE 4 — INFRA (Orca + VMs + Gitea + Infisical + LiteLLM) ║
|
║ PLANE 4 — INFRA cluster: breakpilot-edge VM: vm-edge (co-tenant) ║
|
||||||
╚══════════════════════════════════════════════════════════════════╝
|
║ (Orca + Gitea + Infisical + PowerDNS + Orca-Proxy) ║
|
||||||
|
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Stage and prod share the same App cluster on different VMs (`vm-app-stage` and `vm-app-prod`). Stage authenticates via the prod Identity plane (`auth.breakpilot.com`) and reads tenant config from the prod Control plane (`tenant-registry`) under `tenant.kind = "stage"` — Identity and Control are **not** duplicated for stage.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. Plane 1 — Identity
|
## 4. Plane 1 — Identity
|
||||||
@@ -94,7 +104,7 @@ org_roles — [IT_ADMIN, USER, ...] roles within their org
|
|||||||
realm_roles — [customer] | [BREAKPILOT_ADMIN] | [SUPPORT_ENGINEER] | [SALES_REP]
|
realm_roles — [customer] | [BREAKPILOT_ADMIN] | [SUPPORT_ENGINEER] | [SALES_REP]
|
||||||
products — [certifai, compliance] entitlements (injected by protocol mapper)
|
products — [certifai, compliance] entitlements (injected by protocol mapper)
|
||||||
plan — starter | professional | enterprise
|
plan — starter | professional | enterprise
|
||||||
iss — https://auth.yourplatform.com/realms/breakpilot-prod
|
iss — https://auth.breakpilot.com/realms/breakpilot-prod
|
||||||
```
|
```
|
||||||
|
|
||||||
The `products` and `plan` claims are added by a Keycloak **protocol mapper** that reads live entitlements from the Tenant Registry at token issuance. Products do not need to call back to the registry on every request.
|
The `products` and `plan` claims are added by a Keycloak **protocol mapper** that reads live entitlements from the Tenant Registry at token issuance. Products do not need to call back to the registry on every request.
|
||||||
@@ -108,12 +118,12 @@ Three distinct services. Clear separation of responsibility.
|
|||||||
### 5a. Customer Portal
|
### 5a. Customer Portal
|
||||||
|
|
||||||
**Technology:** Next.js 15 (new service)
|
**Technology:** Next.js 15 (new service)
|
||||||
**Deployed at:** `*.yourplatform.com` via Orca-Proxy wildcard routing
|
**Deployed at:** `*.breakpilot.com` via Orca-Proxy wildcard routing
|
||||||
|
|
||||||
The front door for all customers and for us. Owns no business logic — it is a routing, auth, and UI layer.
|
The front door for all customers and for us. Owns no business logic — it is a routing, auth, and UI layer.
|
||||||
|
|
||||||
**Subdomain routing:**
|
**Subdomain routing:**
|
||||||
- DNS wildcard `*.yourplatform.com` → Orca-Proxy
|
- DNS wildcard `*.breakpilot.com` → Orca-Proxy
|
||||||
- Orca-Proxy reads `Host` header → routes all traffic to the portal container
|
- Orca-Proxy reads `Host` header → routes all traffic to the portal container
|
||||||
- Portal reads `Host` → extracts tenant slug → looks up Tenant Registry
|
- Portal reads `Host` → extracts tenant slug → looks up Tenant Registry
|
||||||
|
|
||||||
@@ -190,7 +200,7 @@ The front door for all customers and for us. Owns no business logic — it is a
|
|||||||
### 5b. ERPNext
|
### 5b. ERPNext
|
||||||
|
|
||||||
**Technology:** Frappe + ERPNext (self-hosted via Orca)
|
**Technology:** Frappe + ERPNext (self-hosted via Orca)
|
||||||
**Access:** `erp.yourplatform.com` — us only (IP-restricted at Orca-Proxy)
|
**Access:** `erp.breakpilot.com` — us only (IP-restricted at Orca-Proxy)
|
||||||
**Auth:** Keycloak OIDC — we log in with our existing accounts, no separate password
|
**Auth:** Keycloak OIDC — we log in with our existing accounts, no separate password
|
||||||
|
|
||||||
ERPNext is our **business operations backbone**. We do not build CRM, invoicing, or HR — we configure ERPNext for these.
|
ERPNext is our **business operations backbone**. We do not build CRM, invoicing, or HR — we configure ERPNext for these.
|
||||||
@@ -262,7 +272,7 @@ api_keys portal-owned. tenant_id, product, scopes, name,
|
|||||||
|
|
||||||
### 5d. Demo Tenant (Shared)
|
### 5d. Demo Tenant (Shared)
|
||||||
|
|
||||||
**Slug:** `demo` — reachable at `demo.yourplatform.com`
|
**Slug:** `demo` — reachable at `demo.breakpilot.com`
|
||||||
**Status:** `demo` (never transitions; never billed)
|
**Status:** `demo` (never transitions; never billed)
|
||||||
**Owner:** us (`BREAKPILOT_ADMIN` curates content; `SALES_REP` reads + logs in)
|
**Owner:** us (`BREAKPILOT_ADMIN` curates content; `SALES_REP` reads + logs in)
|
||||||
|
|
||||||
@@ -299,13 +309,15 @@ all real-tenant flows work otherwise same flows, same code paths
|
|||||||
|
|
||||||
**Support flow:**
|
**Support flow:**
|
||||||
- Customer submits ticket via `/[slug]/support/` (Frappe HD customer portal, embedded or linked)
|
- Customer submits ticket via `/[slug]/support/` (Frappe HD customer portal, embedded or linked)
|
||||||
- Agent (us) triages in Frappe HD agent UI at `erp.yourplatform.com`
|
- Agent (us) triages in Frappe HD agent UI at `erp.breakpilot.com`
|
||||||
- If technical: agent clicks "Escalate to Engineering" → Frappe server script creates a Gitea issue in the relevant repo via Gitea REST API → issue URL stored on ticket
|
- If technical: agent clicks "Escalate to Engineering" → Frappe server script creates a Gitea issue in the relevant repo via Gitea REST API → issue URL stored on ticket
|
||||||
- When Gitea issue is closed → Gitea webhook → Frappe HD → ticket marked "Resolved"
|
- When Gitea issue is closed → Gitea webhook → Frappe HD → ticket marked "Resolved"
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 6. Plane 3 — Data
|
## 6. Plane 3 — App (was "Data")
|
||||||
|
|
||||||
|
Renamed 2026-06-30. The plane carries product workloads end-to-end (dashboards, APIs, AI services) as well as the data stores those products own — "App" reflects the full surface, not just the storage tier. SLA targets are unchanged (CERTifAI ≥ 99.5%, compliance ≥ 99.5%, RPO ≤ 6h–24h per data class). Lives in the `breakpilot-app` cluster on `vm-app-prod` (prod) and `vm-app-stage` (stage).
|
||||||
|
|
||||||
### CERTifAI
|
### CERTifAI
|
||||||
|
|
||||||
@@ -334,18 +346,20 @@ GDPR and AI-Act compliance automation platform. After updates, tenant identity c
|
|||||||
|
|
||||||
## 7. Plane 4 — Infra
|
## 7. Plane 4 — Infra
|
||||||
|
|
||||||
**Orchestration:** Orca manages all containers on Hetzner VMs
|
Co-tenant with the **Identity plane** in the `breakpilot-edge` cluster on `vm-edge`. Single-VM core per the 2026-06-30 cluster-split decision. Keycloak JVM heap is pinned (`-Xmx1500m`) so it cannot starve PowerDNS / Infisical. Escape hatch if the heap fight shows up post-launch: peel Keycloak onto its own VM inside the same cluster — no schema migration, just a `placement.node` flip.
|
||||||
|
|
||||||
|
**Orchestration:** Orca manages all containers on SysEleven VMs. Per-node ingress (every node binds 80/443 and runs its own ACME), one Orca cluster per plane (3 clusters total), one IaC repo per cluster (`platform/breakpilot-{edge,control,app}`).
|
||||||
**Secrets:** Infisical — every service has a machine identity, pulls its own secrets at startup
|
**Secrets:** Infisical — every service has a machine identity, pulls its own secrets at startup
|
||||||
**CI/CD:** Gitea Actions → Docker build → push to private registry → Orca redeploy webhook
|
**CI/CD:** Gitea Actions → Docker build → push to private registry → Orca redeploy webhook (one CI/release cadence per cluster repo, plus per-product repos for the deployed services)
|
||||||
**Routing:** Orca-Proxy handles all TLS termination and subdomain routing
|
**Routing:** Orca-Proxy handles TLS termination and subdomain routing on each node (no central proxy SPOF)
|
||||||
|
|
||||||
```
|
```
|
||||||
Orca-Proxy routing table:
|
Orca-Proxy routing table:
|
||||||
auth.yourplatform.com → Keycloak
|
auth.breakpilot.com → Keycloak
|
||||||
erp.yourplatform.com → ERPNext + Frappe HD (IP-restricted)
|
erp.breakpilot.com → ERPNext + Frappe HD (IP-restricted)
|
||||||
git.yourplatform.com → Gitea
|
git.breakpilot.com → Gitea
|
||||||
secrets.yourplatform.com → Infisical (IP-restricted)
|
secrets.breakpilot.com → Infisical (IP-restricted)
|
||||||
*.yourplatform.com → customer-portal (wildcard, Host → tenant)
|
*.breakpilot.com → customer-portal (wildcard, Host → tenant)
|
||||||
```
|
```
|
||||||
|
|
||||||
**Services managed by Orca:**
|
**Services managed by Orca:**
|
||||||
@@ -436,7 +450,7 @@ Data Stores
|
|||||||
```
|
```
|
||||||
USER ORCA-PROXY PORTAL KEYCLOAK CUSTOMER IdP
|
USER ORCA-PROXY PORTAL KEYCLOAK CUSTOMER IdP
|
||||||
│ │ │ │ │
|
│ │ │ │ │
|
||||||
│ acme.yourplatform.com │ │ │ │
|
│ acme.breakpilot.com │ │ │ │
|
||||||
│───────────────────────►│ │ │ │
|
│───────────────────────►│ │ │ │
|
||||||
│ │ Host=acme.* │ │ │
|
│ │ Host=acme.* │ │ │
|
||||||
│ │───────────────►│ │ │
|
│ │───────────────►│ │ │
|
||||||
@@ -458,7 +472,7 @@ Data Stores
|
|||||||
```
|
```
|
||||||
USER PORTAL KEYCLOAK
|
USER PORTAL KEYCLOAK
|
||||||
│ │ │
|
│ │ │
|
||||||
│ acme.yourplatform│ │
|
│ acme.breakpilot │ │
|
||||||
│──────────────────►│ │
|
│──────────────────►│ │
|
||||||
│ │ redirect + PKCE │
|
│ │ redirect + PKCE │
|
||||||
│ │─────────────────►│
|
│ │─────────────────►│
|
||||||
@@ -671,7 +685,7 @@ Data Stores
|
|||||||
│ │ impersonated_by │ │
|
│ │ impersonated_by │ │
|
||||||
│ │ claim) │ │
|
│ │ claim) │ │
|
||||||
│ │ │
|
│ │ │
|
||||||
│ new tab: acme.yourplatform.com │ │
|
│ new tab: acme.breakpilot.com │ │
|
||||||
│──────────────────────────────────────────────────────────►│
|
│──────────────────────────────────────────────────────────►│
|
||||||
│ │ [orange banner] │
|
│ │ [orange banner] │
|
||||||
│ │ Impersonating │
|
│ │ Impersonating │
|
||||||
@@ -749,7 +763,7 @@ Data Stores
|
|||||||
│ │ │ │
|
│ │ │ │
|
||||||
│ open Zoom with prospect, share screen │
|
│ open Zoom with prospect, share screen │
|
||||||
│ │
|
│ │
|
||||||
│ demo.yourplatform.com │
|
│ demo.breakpilot.com │
|
||||||
│────────────────────────────────►│ │
|
│────────────────────────────────►│ │
|
||||||
│ │ │ Host: demo │
|
│ │ │ Host: demo │
|
||||||
│ │ │ → slug = demo │
|
│ │ │ → slug = demo │
|
||||||
@@ -796,7 +810,7 @@ Data Stores
|
|||||||
```
|
```
|
||||||
PROSPECT PORTAL TENANT REGISTRY ERPNEXT KEYCLOAK
|
PROSPECT PORTAL TENANT REGISTRY ERPNEXT KEYCLOAK
|
||||||
│ │ │ │ │
|
│ │ │ │ │
|
||||||
│ yourplatform.com/start │ │ │
|
│ breakpilot.com/start │ │ │
|
||||||
│──────────────►│ │ │ │
|
│──────────────►│ │ │ │
|
||||||
│ form: email, company, password │ │ │
|
│ form: email, company, password │ │ │
|
||||||
│──────────────►│ │ │ │
|
│──────────────►│ │ │ │
|
||||||
|
|||||||
+19
-19
@@ -279,7 +279,7 @@ Products that ship custom styling must respect the `theme` attribute and the pre
|
|||||||
|
|
||||||
```
|
```
|
||||||
Product publishes a bundle at:
|
Product publishes a bundle at:
|
||||||
https://cdn.yourplatform.com/products/{name}/{version}/element.js
|
https://cdn.breakpilot.com/products/{name}/{version}/element.js
|
||||||
|
|
||||||
Portal loads it lazily via dynamic import when the user navigates to /[tenant]/products/{name}.
|
Portal loads it lazily via dynamic import when the user navigates to /[tenant]/products/{name}.
|
||||||
Portal caches the bundle URL per product version (declared in tenant_products.config).
|
Portal caches the bundle URL per product version (declared in tenant_products.config).
|
||||||
@@ -343,12 +343,12 @@ The product ships NO frontend code. The portal renders a generic management UI f
|
|||||||
│ CODE SAMPLES │
|
│ CODE SAMPLES │
|
||||||
│ ──────────────────────────────────────────────────── │
|
│ ──────────────────────────────────────────────────── │
|
||||||
│ [curl] [JS] [Python] │
|
│ [curl] [JS] [Python] │
|
||||||
│ curl -X POST https://notetaker-api.yourplatform.com/v1 │
|
│ curl -X POST https://notetaker-api.breakpilot.com/v1 │
|
||||||
│ -H "Authorization: ApiKey k_xxx" │
|
│ -H "Authorization: ApiKey k_xxx" │
|
||||||
│ -H "X-Tenant: acme" │
|
│ -H "X-Tenant: acme" │
|
||||||
│ -d '{...}' │
|
│ -d '{...}' │
|
||||||
│ │
|
│ │
|
||||||
│ DOCS ► developers.yourplatform.com/products/notetaker │
|
│ DOCS ► developers.breakpilot.com/products/notetaker │
|
||||||
└──────────────────────────────────────────────────────────┘
|
└──────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -394,7 +394,7 @@ An MCP (Model Context Protocol) server exposes the product's capabilities as too
|
|||||||
|
|
||||||
```
|
```
|
||||||
1. ONE MCP server per product
|
1. ONE MCP server per product
|
||||||
Endpoint: https://mcp.{product}.yourplatform.com (or unified mcp.yourplatform.com/{product})
|
Endpoint: https://mcp.{product}.breakpilot.com (or unified mcp.breakpilot.com/{product})
|
||||||
|
|
||||||
2. Authentication via SCOPED API KEY
|
2. Authentication via SCOPED API KEY
|
||||||
Customer IT Admin generates API key in /[tenant]/settings/api-keys.
|
Customer IT Admin generates API key in /[tenant]/settings/api-keys.
|
||||||
@@ -438,7 +438,7 @@ Enterprise customers automatically get MCP enabled. Starter/Pro customers see "A
|
|||||||
|
|
||||||
## 7. Documentation Contract
|
## 7. Documentation Contract
|
||||||
|
|
||||||
A product ships five required documents. They are published at `developers.yourplatform.com/products/{name}/`.
|
A product ships five required documents. They are published at `developers.breakpilot.com/products/{name}/`.
|
||||||
|
|
||||||
```
|
```
|
||||||
1. README What does it do? Value prop in 200 words.
|
1. README What does it do? Value prop in 200 words.
|
||||||
@@ -763,16 +763,16 @@ product:
|
|||||||
vendor: breakpilot # we; future third-parties will use their slug
|
vendor: breakpilot # we; future third-parties will use their slug
|
||||||
contract_version: "1.0"
|
contract_version: "1.0"
|
||||||
product_version: "1.4.2"
|
product_version: "1.4.2"
|
||||||
repo: git.yourplatform.com/sharang/certifai
|
repo: git.breakpilot.com/sharang/certifai
|
||||||
|
|
||||||
catalog:
|
catalog:
|
||||||
# Renders in /[tenant]/catalog and /backstage/products
|
# Renders in /[tenant]/catalog and /backstage/products
|
||||||
category: "AI Infrastructure" # AI Infrastructure | Compliance | Productivity | Security | Data
|
category: "AI Infrastructure" # AI Infrastructure | Compliance | Productivity | Security | Data
|
||||||
tagline: "GDPR-compliant LLMs without leaving the EU"
|
tagline: "GDPR-compliant LLMs without leaving the EU"
|
||||||
hero_image: https://cdn.yourplatform.com/products/certifai/hero.png
|
hero_image: https://cdn.breakpilot.com/products/certifai/hero.png
|
||||||
screenshots:
|
screenshots:
|
||||||
- https://cdn.yourplatform.com/products/certifai/dashboard.png
|
- https://cdn.breakpilot.com/products/certifai/dashboard.png
|
||||||
- https://cdn.yourplatform.com/products/certifai/agents.png
|
- https://cdn.breakpilot.com/products/certifai/agents.png
|
||||||
pricing_summary: "From €X/seat/month — included on Professional and Enterprise plans"
|
pricing_summary: "From €X/seat/month — included on Professional and Enterprise plans"
|
||||||
available_on_plans: [trial, professional, enterprise] # 'trial' opt-in for self-serve
|
available_on_plans: [trial, professional, enterprise] # 'trial' opt-in for self-serve
|
||||||
trial_days: 14
|
trial_days: 14
|
||||||
@@ -784,7 +784,7 @@ catalog:
|
|||||||
|
|
||||||
demo:
|
demo:
|
||||||
supported: true # MUST be true unless explicitly waived
|
supported: true # MUST be true unless explicitly waived
|
||||||
seed_data_url: https://cdn.yourplatform.com/products/certifai/demo/seed-v3.tar.gz
|
seed_data_url: https://cdn.breakpilot.com/products/certifai/demo/seed-v3.tar.gz
|
||||||
reset_endpoint: /v1/tenants/demo/reset # called nightly by portal cron
|
reset_endpoint: /v1/tenants/demo/reset # called nightly by portal cron
|
||||||
persona_hints: # for sales rep talk track
|
persona_hints: # for sales rep talk track
|
||||||
- "GDPR officer at a 200-person SaaS"
|
- "GDPR officer at a 200-person SaaS"
|
||||||
@@ -807,7 +807,7 @@ identity:
|
|||||||
frontend:
|
frontend:
|
||||||
type: interactive # interactive | widget | headless
|
type: interactive # interactive | widget | headless
|
||||||
tag: certifai-dashboard
|
tag: certifai-dashboard
|
||||||
bundle_url: https://cdn.yourplatform.com/products/certifai/{version}/element.js
|
bundle_url: https://cdn.breakpilot.com/products/certifai/{version}/element.js
|
||||||
bundle_size_kb: 380
|
bundle_size_kb: 380
|
||||||
routes:
|
routes:
|
||||||
- path: /
|
- path: /
|
||||||
@@ -828,7 +828,7 @@ backend:
|
|||||||
mcp:
|
mcp:
|
||||||
enabled: true
|
enabled: true
|
||||||
required_plan: enterprise
|
required_plan: enterprise
|
||||||
endpoint: https://mcp.yourplatform.com/certifai
|
endpoint: https://mcp.breakpilot.com/certifai
|
||||||
tools:
|
tools:
|
||||||
- name: list_ai_agents
|
- name: list_ai_agents
|
||||||
description: "Returns AI agents configured for the tenant"
|
description: "Returns AI agents configured for the tenant"
|
||||||
@@ -864,7 +864,7 @@ backup:
|
|||||||
retention_days: 30
|
retention_days: 30
|
||||||
|
|
||||||
infra:
|
infra:
|
||||||
image: registry.yourplatform.com/certifai-dashboard
|
image: registry.breakpilot.com/certifai-dashboard
|
||||||
vm: vm-certifai
|
vm: vm-certifai
|
||||||
replicas: 1
|
replicas: 1
|
||||||
resource_limits:
|
resource_limits:
|
||||||
@@ -909,7 +909,7 @@ The example above shows an `interactive` product. Headless and widget products d
|
|||||||
frontend:
|
frontend:
|
||||||
type: widget
|
type: widget
|
||||||
tag: status-monitor-widget
|
tag: status-monitor-widget
|
||||||
bundle_url: https://cdn.yourplatform.com/products/status/{version}/widget.js
|
bundle_url: https://cdn.breakpilot.com/products/status/{version}/widget.js
|
||||||
bundle_size_kb: 38
|
bundle_size_kb: 38
|
||||||
dimensions:
|
dimensions:
|
||||||
width: 400
|
width: 400
|
||||||
@@ -954,7 +954,7 @@ frontend:
|
|||||||
- language: curl
|
- language: curl
|
||||||
title: "Create a session"
|
title: "Create a session"
|
||||||
snippet: |
|
snippet: |
|
||||||
curl -X POST https://notetaker-api.yourplatform.com/v1/sessions \
|
curl -X POST https://notetaker-api.breakpilot.com/v1/sessions \
|
||||||
-H "Authorization: ApiKey k_xxx" \
|
-H "Authorization: ApiKey k_xxx" \
|
||||||
-H "X-Tenant: acme" \
|
-H "X-Tenant: acme" \
|
||||||
-d '{"audio_url": "...", "language": "en"}'
|
-d '{"audio_url": "...", "language": "en"}'
|
||||||
@@ -963,7 +963,7 @@ frontend:
|
|||||||
snippet: |
|
snippet: |
|
||||||
import requests
|
import requests
|
||||||
requests.post(
|
requests.post(
|
||||||
"https://notetaker-api.yourplatform.com/v1/sessions",
|
"https://notetaker-api.breakpilot.com/v1/sessions",
|
||||||
headers={"Authorization": "ApiKey k_xxx", "X-Tenant": "acme"},
|
headers={"Authorization": "ApiKey k_xxx", "X-Tenant": "acme"},
|
||||||
json={"audio_url": "...", "language": "en"},
|
json={"audio_url": "...", "language": "en"},
|
||||||
)
|
)
|
||||||
@@ -986,7 +986,7 @@ Products can call each other directly. Auth is via short-lived service tokens is
|
|||||||
1. Compliance product needs to list AI agents for an AI Act assessment.
|
1. Compliance product needs to list AI agents for an AI Act assessment.
|
||||||
|
|
||||||
2. Compliance backend requests a service token:
|
2. Compliance backend requests a service token:
|
||||||
POST https://auth.yourplatform.com/realms/breakpilot-prod/protocol/openid-connect/token
|
POST https://auth.breakpilot.com/realms/breakpilot-prod/protocol/openid-connect/token
|
||||||
Body: grant_type=client_credentials
|
Body: grant_type=client_credentials
|
||||||
client_id=compliance-svc
|
client_id=compliance-svc
|
||||||
client_secret=<from Infisical>
|
client_secret=<from Infisical>
|
||||||
@@ -1135,7 +1135,7 @@ A product is "ready to ship to a customer" when all boxes are ticked.
|
|||||||
☐ All tools tenant-scoped and audited
|
☐ All tools tenant-scoped and audited
|
||||||
|
|
||||||
☐ Documentation
|
☐ Documentation
|
||||||
☐ README published at developers.yourplatform.com/products/{name}
|
☐ README published at developers.breakpilot.com/products/{name}
|
||||||
☐ API reference auto-generated and live
|
☐ API reference auto-generated and live
|
||||||
☐ Integration guide for customer IT
|
☐ Integration guide for customer IT
|
||||||
☐ Operational runbook for us
|
☐ Operational runbook for us
|
||||||
@@ -1220,7 +1220,7 @@ Effort estimate: 3-5 weeks of focused work
|
|||||||
```
|
```
|
||||||
- Design tokens package (@breakpilot/design-tokens) — needs to exist before web components ship
|
- Design tokens package (@breakpilot/design-tokens) — needs to exist before web components ship
|
||||||
- CDN for product bundles — pick provider (Hetzner Object Storage + Cloudflare?)
|
- CDN for product bundles — pick provider (Hetzner Object Storage + Cloudflare?)
|
||||||
- MCP gateway — single mcp.yourplatform.com vs. per-product subdomains
|
- MCP gateway — single mcp.breakpilot.com vs. per-product subdomains
|
||||||
- Third-party manifest signing — defer until first real third-party conversation
|
- Third-party manifest signing — defer until first real third-party conversation
|
||||||
- Inter-product event bus — explicitly deferred; service tokens cover the use cases for now
|
- Inter-product event bus — explicitly deferred; service tokens cover the use cases for now
|
||||||
- Contract testing — automate manifest + openapi validation in Gitea Actions
|
- Contract testing — automate manifest + openapi validation in Gitea Actions
|
||||||
|
|||||||
@@ -39,8 +39,8 @@ For IaC: list the make targets.}}
|
|||||||
| Env | URL | How |
|
| Env | URL | How |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| dev | `http://localhost:3000` | `make dev` |
|
| dev | `http://localhost:3000` | `make dev` |
|
||||||
| stage | `https://docs.stage.yourplatform.com` | auto on merge to `main` |
|
| stage | `https://docs.stage.breakpilot.com` | auto on merge to `main` |
|
||||||
| prod | `https://docs.yourplatform.com` | manual: tag `vX.Y.Z` + sign-off |
|
| prod | `https://docs.breakpilot.com` | manual: tag `vX.Y.Z` + sign-off |
|
||||||
|
|
||||||
Rollback: `orca rollout undo docs --env={{env}}`.
|
Rollback: `orca rollout undo docs --env={{env}}`.
|
||||||
|
|
||||||
@@ -48,7 +48,7 @@ Rollback: `orca rollout undo docs --env={{env}}`.
|
|||||||
|
|
||||||
- Traces, logs, metrics: [SigNoz](https://signoz.meghsakha.com) — service name `docs`
|
- Traces, logs, metrics: [SigNoz](https://signoz.meghsakha.com) — service name `docs`
|
||||||
- Audit events: Tenant Registry `/audit` (Retraced-shape schema)
|
- Audit events: Tenant Registry `/audit` (Retraced-shape schema)
|
||||||
- On-call: `oncall@yourplatform.com` · runbook at `platform/docs/runbooks/docs.md`
|
- On-call: `oncall@breakpilot.com` · runbook at `platform/docs/runbooks/docs.md`
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user