Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions .github/workflows/examples-it-multiregion.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
name: examples-it-multiregion

# Runs the Ideas Board multi-region integration test against a real 3-region
# CockroachDB cluster brought up with docker-compose (one node each in us-east1,
# us-west1, us-central1). This is the only way test_regional_it.py actually runs:
# it SKIPs unless CRDB_MULTIREGION=true and CRDB_URL point at a cluster whose
# database already has those regions added.
#
# Kept as a separate workflow from examples-it.yml so the single-node
# Testcontainers job is untouched.
on:
pull_request:
paths:
- 'examples/ideas-board/**'
- '.github/workflows/examples-it-multiregion.yml'
push:
branches: [main]
paths:
- 'examples/ideas-board/**'
- '.github/workflows/examples-it-multiregion.yml'
workflow_dispatch:

jobs:
ideas-board-multiregion-it:
name: Ideas Board multi-region IT (docker-compose cluster)
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'

- name: Start 3-region CockroachDB cluster
working-directory: examples/ideas-board
run: docker compose -f multiregion-compose.yml up -d crdb-us-east1 crdb-us-west1 crdb-us-central1

- name: Initialize cluster and configure regions
working-directory: examples/ideas-board
# Hard cap so a cluster that never reaches quorum fails the step (and runs
# the diagnostics below) instead of blocking the job.
run: timeout 420 docker compose -f multiregion-compose.yml run --rm crdb-init

- name: Install dependencies (published packages from the Pages feed)
working-directory: examples/ideas-board
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt

- name: Run multi-region integration test
working-directory: examples/ideas-board
env:
CRDB_MULTIREGION: "true"
CRDB_URL: "postgres://root@localhost:26257/defaultdb?sslmode=disable"
run: python -m pytest test_regional_it.py -v

- name: Cluster diagnostics on failure
if: failure()
working-directory: examples/ideas-board
run: |
docker compose -f multiregion-compose.yml ps
docker compose -f multiregion-compose.yml logs --tail=300

- name: Tear down cluster
if: always()
working-directory: examples/ideas-board
run: docker compose -f multiregion-compose.yml down -v
100 changes: 100 additions & 0 deletions examples/ideas-board/multiregion-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
# 3-node insecure CockroachDB cluster for the Ideas Board multi-region IT.
#
# Each node declares a distinct --locality=region, so the cluster exposes three
# regions (us-east1, us-west1, us-central1). The one-shot `crdb-init` service runs
# `cockroach init` and then makes `defaultdb` multi-region, which is what
# test_regional_it.py expects (it SKIPs unless CRDB_MULTIREGION=true and CRDB_URL
# point at a database that already has those regions added). The test itself only
# uses us-east1 + us-west1; us-central1 is the third region a multi-region DB wants.
#
# Only node1's SQL port is published to the host:
# CRDB_URL=postgres://root@localhost:26257/defaultdb?sslmode=disable
#
# Local use:
# docker compose -f multiregion-compose.yml up -d crdb-us-east1 crdb-us-west1 crdb-us-central1
# docker compose -f multiregion-compose.yml run --rm crdb-init
# CRDB_MULTIREGION=true CRDB_URL=postgres://root@localhost:26257/defaultdb?sslmode=disable \
# python -m pytest test_regional_it.py -v
#
# Default on-disk stores (ephemeral in the container, wiped by `down -v`) + capped
# caches keep three nodes light enough for a CI runner. (In-memory stores need
# >640 MiB each on v25.1, so on-disk is both simpler and lighter here.)
# Every SQL call sets a statement_timeout and every loop is iteration-bounded, so a
# cluster that never reaches quorum FAILS the init (with node status dumped) instead
# of hanging; the workflow also wraps this service in `timeout 420` as a backstop.
# ponytail: single node per region (RF=3 across 3 nodes), not 3-per-region zone
# survival — enough to prove region-homed writes; scale up only if survival goals
# are what you're testing.

name: crdb-multiregion-it

x-crdb: &crdb
image: cockroachdb/cockroach:latest-v25.1
restart: "no"

services:
crdb-us-east1:
<<: *crdb
hostname: crdb-us-east1
command: start --insecure --cache=256MiB --max-sql-memory=256MiB --advertise-addr=crdb-us-east1 --join=crdb-us-east1,crdb-us-west1,crdb-us-central1 --locality=region=us-east1
ports:
- "26257:26257"

crdb-us-west1:
<<: *crdb
hostname: crdb-us-west1
command: start --insecure --cache=256MiB --max-sql-memory=256MiB --advertise-addr=crdb-us-west1 --join=crdb-us-east1,crdb-us-west1,crdb-us-central1 --locality=region=us-west1

crdb-us-central1:
<<: *crdb
hostname: crdb-us-central1
command: start --insecure --cache=256MiB --max-sql-memory=256MiB --advertise-addr=crdb-us-central1 --join=crdb-us-east1,crdb-us-west1,crdb-us-central1 --locality=region=us-central1

# One-shot: initialize the cluster, wait for all three regions to join, then make
# defaultdb multi-region. Every SQL call sets a statement_timeout and every loop is
# iteration-bounded, so this service always terminates: exit 0 on success, exit 1
# (after dumping node status) if the cluster never becomes a healthy 3-region cluster.
crdb-init:
<<: *crdb
depends_on:
- crdb-us-east1
- crdb-us-west1
- crdb-us-central1
entrypoint: ["/bin/bash", "-c"]
command:
- |
set -u
host=crdb-us-east1
# 1) initialize the cluster (first reachable node); returns fast if refused.
for i in $$(seq 1 30); do
cockroach init --insecure --host="$$host" 2>/tmp/init.err && { echo "cluster initialized"; break; }
grep -q "already been initialized" /tmp/init.err && { echo "cluster already initialized"; break; }
cat /tmp/init.err; echo "init attempt $$i failed, retrying"; sleep 2
done
# 2) wait until all three region localities are visible cluster-wide.
for i in $$(seq 1 40); do
count=$$(cockroach sql --insecure --host="$$host" --format=csv \
-e "SET statement_timeout='15s'; SELECT count(DISTINCT region) FROM [SHOW REGIONS FROM CLUSTER]" 2>/tmp/reg.err | tail -n1)
echo "attempt $$i distinct_regions=$$count"
[ "$$count" = "3" ] && { echo "all 3 regions present"; break; }
cat /tmp/reg.err 2>/dev/null || true
sleep 3
done
# 3) make defaultdb multi-region (idempotent). statement_timeout stops a
# schema change that cannot make progress from hanging the loop.
for i in $$(seq 1 20); do
if cockroach sql --insecure --host="$$host" -e "
SET statement_timeout='30s';
ALTER DATABASE defaultdb SET PRIMARY REGION 'us-east1';
ALTER DATABASE defaultdb ADD REGION IF NOT EXISTS 'us-west1';
ALTER DATABASE defaultdb ADD REGION IF NOT EXISTS 'us-central1';
"; then
echo "defaultdb is multi-region:"
cockroach sql --insecure --host="$$host" -e "SHOW REGIONS FROM DATABASE defaultdb"
exit 0
fi
echo "region config attempt $$i failed, retrying"; sleep 3
done
echo "FAILED to configure multi-region defaultdb; node status:"
cockroach node status --insecure --host="$$host" || true
exit 1
Loading