diff --git a/.github/workflows/examples-it-multiregion.yml b/.github/workflows/examples-it-multiregion.yml new file mode 100644 index 0000000..ccb0a9f --- /dev/null +++ b/.github/workflows/examples-it-multiregion.yml @@ -0,0 +1,67 @@ +name: examples-it-multiregion + +# Runs the Ideas Board multi-region integration test against a real 3-region +# CockroachDB cluster brought up with docker-compose (one node each in us-east1, +# us-west1, us-central1). This is the only way test_regional_it.py actually runs: +# it SKIPs unless CRDB_MULTIREGION=true and CRDB_URL point at a cluster whose +# database already has those regions added. +# +# Kept as a separate workflow from examples-it.yml so the single-node +# Testcontainers job is untouched. +on: + pull_request: + paths: + - 'examples/ideas-board/**' + - '.github/workflows/examples-it-multiregion.yml' + push: + branches: [main] + paths: + - 'examples/ideas-board/**' + - '.github/workflows/examples-it-multiregion.yml' + workflow_dispatch: + +jobs: + ideas-board-multiregion-it: + name: Ideas Board multi-region IT (docker-compose cluster) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Start 3-region CockroachDB cluster + working-directory: examples/ideas-board + run: docker compose -f multiregion-compose.yml up -d crdb-us-east1 crdb-us-west1 crdb-us-central1 + + - name: Initialize cluster and configure regions + working-directory: examples/ideas-board + # Hard cap so a cluster that never reaches quorum fails the step (and runs + # the diagnostics below) instead of blocking the job. + run: timeout 420 docker compose -f multiregion-compose.yml run --rm crdb-init + + - name: Install dependencies (published packages from the Pages feed) + working-directory: examples/ideas-board + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run multi-region integration test + working-directory: examples/ideas-board + env: + CRDB_MULTIREGION: "true" + CRDB_URL: "postgres://root@localhost:26257/defaultdb?sslmode=disable" + run: python -m pytest test_regional_it.py -v + + - name: Cluster diagnostics on failure + if: failure() + working-directory: examples/ideas-board + run: | + docker compose -f multiregion-compose.yml ps + docker compose -f multiregion-compose.yml logs --tail=300 + + - name: Tear down cluster + if: always() + working-directory: examples/ideas-board + run: docker compose -f multiregion-compose.yml down -v diff --git a/examples/ideas-board/multiregion-compose.yml b/examples/ideas-board/multiregion-compose.yml new file mode 100644 index 0000000..0a516b6 --- /dev/null +++ b/examples/ideas-board/multiregion-compose.yml @@ -0,0 +1,100 @@ +# 3-node insecure CockroachDB cluster for the Ideas Board multi-region IT. +# +# Each node declares a distinct --locality=region, so the cluster exposes three +# regions (us-east1, us-west1, us-central1). The one-shot `crdb-init` service runs +# `cockroach init` and then makes `defaultdb` multi-region, which is what +# test_regional_it.py expects (it SKIPs unless CRDB_MULTIREGION=true and CRDB_URL +# point at a database that already has those regions added). The test itself only +# uses us-east1 + us-west1; us-central1 is the third region a multi-region DB wants. +# +# Only node1's SQL port is published to the host: +# CRDB_URL=postgres://root@localhost:26257/defaultdb?sslmode=disable +# +# Local use: +# docker compose -f multiregion-compose.yml up -d crdb-us-east1 crdb-us-west1 crdb-us-central1 +# docker compose -f multiregion-compose.yml run --rm crdb-init +# CRDB_MULTIREGION=true CRDB_URL=postgres://root@localhost:26257/defaultdb?sslmode=disable \ +# python -m pytest test_regional_it.py -v +# +# Default on-disk stores (ephemeral in the container, wiped by `down -v`) + capped +# caches keep three nodes light enough for a CI runner. (In-memory stores need +# >640 MiB each on v25.1, so on-disk is both simpler and lighter here.) +# Every SQL call sets a statement_timeout and every loop is iteration-bounded, so a +# cluster that never reaches quorum FAILS the init (with node status dumped) instead +# of hanging; the workflow also wraps this service in `timeout 420` as a backstop. +# ponytail: single node per region (RF=3 across 3 nodes), not 3-per-region zone +# survival — enough to prove region-homed writes; scale up only if survival goals +# are what you're testing. + +name: crdb-multiregion-it + +x-crdb: &crdb + image: cockroachdb/cockroach:latest-v25.1 + restart: "no" + +services: + crdb-us-east1: + <<: *crdb + hostname: crdb-us-east1 + command: start --insecure --cache=256MiB --max-sql-memory=256MiB --advertise-addr=crdb-us-east1 --join=crdb-us-east1,crdb-us-west1,crdb-us-central1 --locality=region=us-east1 + ports: + - "26257:26257" + + crdb-us-west1: + <<: *crdb + hostname: crdb-us-west1 + command: start --insecure --cache=256MiB --max-sql-memory=256MiB --advertise-addr=crdb-us-west1 --join=crdb-us-east1,crdb-us-west1,crdb-us-central1 --locality=region=us-west1 + + crdb-us-central1: + <<: *crdb + hostname: crdb-us-central1 + command: start --insecure --cache=256MiB --max-sql-memory=256MiB --advertise-addr=crdb-us-central1 --join=crdb-us-east1,crdb-us-west1,crdb-us-central1 --locality=region=us-central1 + + # One-shot: initialize the cluster, wait for all three regions to join, then make + # defaultdb multi-region. Every SQL call sets a statement_timeout and every loop is + # iteration-bounded, so this service always terminates: exit 0 on success, exit 1 + # (after dumping node status) if the cluster never becomes a healthy 3-region cluster. + crdb-init: + <<: *crdb + depends_on: + - crdb-us-east1 + - crdb-us-west1 + - crdb-us-central1 + entrypoint: ["/bin/bash", "-c"] + command: + - | + set -u + host=crdb-us-east1 + # 1) initialize the cluster (first reachable node); returns fast if refused. + for i in $$(seq 1 30); do + cockroach init --insecure --host="$$host" 2>/tmp/init.err && { echo "cluster initialized"; break; } + grep -q "already been initialized" /tmp/init.err && { echo "cluster already initialized"; break; } + cat /tmp/init.err; echo "init attempt $$i failed, retrying"; sleep 2 + done + # 2) wait until all three region localities are visible cluster-wide. + for i in $$(seq 1 40); do + count=$$(cockroach sql --insecure --host="$$host" --format=csv \ + -e "SET statement_timeout='15s'; SELECT count(DISTINCT region) FROM [SHOW REGIONS FROM CLUSTER]" 2>/tmp/reg.err | tail -n1) + echo "attempt $$i distinct_regions=$$count" + [ "$$count" = "3" ] && { echo "all 3 regions present"; break; } + cat /tmp/reg.err 2>/dev/null || true + sleep 3 + done + # 3) make defaultdb multi-region (idempotent). statement_timeout stops a + # schema change that cannot make progress from hanging the loop. + for i in $$(seq 1 20); do + if cockroach sql --insecure --host="$$host" -e " + SET statement_timeout='30s'; + ALTER DATABASE defaultdb SET PRIMARY REGION 'us-east1'; + ALTER DATABASE defaultdb ADD REGION IF NOT EXISTS 'us-west1'; + ALTER DATABASE defaultdb ADD REGION IF NOT EXISTS 'us-central1'; + "; then + echo "defaultdb is multi-region:" + cockroach sql --insecure --host="$$host" -e "SHOW REGIONS FROM DATABASE defaultdb" + exit 0 + fi + echo "region config attempt $$i failed, retrying"; sleep 3 + done + echo "FAILED to configure multi-region defaultdb; node status:" + cockroach node status --insecure --host="$$host" || true + exit 1