From f3fd9aebc83a09799a0d6ae68f61a9cab3abbbcd Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:00:05 +0000 Subject: [PATCH 01/24] Fix psql ON_ERROR_STOP in database restore commands The restore commands passed -v ERROR_ON_STOP=1, a variable psql does not read, so a failed statement was ignored and psql exited 0. Use ON_ERROR_STOP=1, and connect to a database the dump does not drop, since pg_dump -C --clean dumps start with DROP DATABASE. On Kubernetes, first end other sessions, such as the pgsql-exporter sidecar's, which block DROP DATABASE. Amp-Thread-ID: https://ampcode.com/threads/T-454efa43-334a-4f07-a372-203b3c78a204 Co-authored-by: Amp --- .../self-hosted/deploy/docker-compose/operations.mdx | 12 ++++++------ docs/self-hosted/deploy/kubernetes/operations.mdx | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index ba71790b8..73e328b60 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -116,9 +116,9 @@ docker cp codeinsights_db.out codeinsights-db:/tmp/codeinsights_db.out 4\. Restore the databases ```bash -docker exec pgsql sh -c 'psql -v ERROR_ON_STOP=1 --username sg -f /tmp/sourcegraph_db.out sg' -docker exec codeintel-db sh -c 'psql -v ERROR_ON_STOP=1 --username sg -f /tmp/codeintel_db.out sg' -docker exec codeinsights-db sh -c 'psql -v ERROR_ON_STOP=1 --username postgres -f /tmp/codeinsights_db.out postgres' +docker exec pgsql sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/sourcegraph_db.out postgres' +docker exec codeintel-db sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/codeintel_db.out postgres' +docker exec codeinsights-db sh -c 'psql -v ON_ERROR_STOP=1 --username postgres -f /tmp/codeinsights_db.out template1' ``` 5\. Start the remaining Sourcegraph services @@ -185,9 +185,9 @@ docker cp codeinsights_db.out codeinsights-db:/tmp/codeinsights_db.out 4\. Restore the databases ```bash -docker exec pgsql sh -c 'psql -v ERROR_ON_STOP=1 --username sg -f /tmp/sourcegraph_db.out sg' -docker exec codeintel-db sh -c 'psql -v ERROR_ON_STOP=1 --username sg -f /tmp/codeintel_db.out sg' -docker exec codeinsights-db sh -c 'psql -v ERROR_ON_STOP=1 --username postgres -f /tmp/codeinsights_db.out postgres' +docker exec pgsql sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/sourcegraph_db.out postgres' +docker exec codeintel-db sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/codeintel_db.out postgres' +docker exec codeinsights-db sh -c 'psql -v ON_ERROR_STOP=1 --username postgres -f /tmp/codeinsights_db.out template1' ``` 5\. Start the remaining Sourcegraph services diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index a91f4667c..7e7774390 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -311,9 +311,9 @@ $ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/tmp/codei D. Restore the databases ```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ERROR_ON_STOP=1 --username sg -f /tmp/sourcegraph_db.out sg' -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ERROR_ON_STOP=1 --username sg -f /tmp/codeintel_db.out sg' -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ERROR_ON_STOP=1 --username postgres -f /tmp/codeinsights_db.out postgres' +$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/sourcegraph_db.out postgres' +$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/codeintel_db.out postgres' +$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/codeinsights_db.out template1' ``` E. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) @@ -378,9 +378,9 @@ $ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/tmp/codei F. Restore the databases ```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ERROR_ON_STOP=1 --username sg -f /tmp/sourcegraph_db.out sg' -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ERROR_ON_STOP=1 --username sg -f /tmp/codeintel_db.out sg' -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ERROR_ON_STOP=1 --username postgres -f /tmp/codeinsights_db.out postgres' +$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/sourcegraph_db.out postgres' +$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/codeintel_db.out postgres' +$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/codeinsights_db.out template1' ``` G. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) From ddbff06898e2bac900d661dea69b79aae96c534c Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:07:59 -0600 Subject: [PATCH 02/24] Fix the rest of the database backup and restore steps Docker Compose: db-only-migrate.docker-compose.yaml does not start codeinsights-db, so the backup wrote an empty codeinsights_db.out and the restore lost the code insights data. Start codeinsights-db too, and note where the volume names come from. Kubernetes: the databases are StatefulSets, their root filesystems are read-only, and kubectl exec -it adds a carriage return to every line of a dump. Restart the StatefulSets, copy the dumps into the database volumes and delete them after a successful restore, run pg_dump without -it, and stop deleting the database volumes, which left the database pods Pending; the restore drops and recreates each database. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 Co-authored-by: Amp --- .../deploy/docker-compose/operations.mdx | 5 ++ .../deploy/kubernetes/operations.mdx | 75 ++++++++++--------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 73e328b60..74585690f 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -81,6 +81,7 @@ zoekt-webserver-0 /sbin/tini -- /bin/sh -c z ... Up (healthy) ```bash docker-compose down docker-compose -f db-only-migrate.docker-compose.yaml up -d +docker-compose up -d codeinsights-db ``` 5\. Generate the database dumps @@ -103,6 +104,7 @@ The following instructions apply **only if you are restoring your databases into ```bash docker-compose -f db-only-migrate.docker-compose.yaml up -d +docker-compose up -d codeinsights-db ``` 3\. Copy the database files into the containers @@ -168,10 +170,13 @@ docker volume rm docker-compose_codeintel-db docker volume rm docker-compose_codeinsights-db ``` +The volume names start with the Compose project name, which defaults to the directory name, `docker-compose`. If you set a different project name, find the volume names with `docker volume ls`. + 2\. Start the databases services only ```bash docker-compose -f db-only-migrate.docker-compose.yaml up -d +docker-compose up -d codeinsights-db ``` 3\. Copy the database files into the containers diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 7e7774390..010da8e2b 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -275,11 +275,13 @@ and i.indisready AND i.indisvalid; D. Generate the database dumps ```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > sourcegraph_db.out -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > codeintel_db.out -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username postgres postgres' > codeinsights_db.out +$ kubectl exec $pgsql_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > sourcegraph_db.out +$ kubectl exec $codeintel-db_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > codeintel_db.out +$ kubectl exec $codeinsights-db_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username postgres postgres' > codeinsights_db.out ``` +Run these commands without `-it`: with a terminal attached, every line of the dump ends with a carriage return. + Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` files are moved to a safe and secure location. ### Restore Sourcegraph databases @@ -295,25 +297,29 @@ A. Copy the database dump files (eg. `sourcegraph_db.out`, `codeintel_db.out` an B. Start the database services by running the following command from your deployment directory: ```bash -$ kubectl rollout restart deployment pgsql -$ kubectl rollout restart deployment codeintel-db -$ kubectl rollout restart deployment codeinsights-db +$ kubectl rollout restart statefulset pgsql +$ kubectl rollout restart statefulset codeintel-db +$ kubectl rollout restart statefulset codeinsights-db ``` C. Copy the database files into the pods by running the following commands: ```bash -$ kubectl cp sourcegraph_db.out $NAMESPACE/$pgsql_POD_NAME:/tmp/sourcegraph_db.out -$ kubectl cp codeintel_db.out $NAMESPACE/$codeintel-db_POD_NAME:/tmp/codeintel_db.out -$ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/tmp/codeinsights_db.out +$ kubectl cp sourcegraph_db.out $NAMESPACE/$pgsql_POD_NAME:/data/sourcegraph_db.out +$ kubectl cp codeintel_db.out $NAMESPACE/$codeintel-db_POD_NAME:/data/codeintel_db.out +$ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/var/lib/postgresql/data/codeinsights_db.out ``` +The database containers have a read-only root filesystem, so copy the files into the database volumes. The restore commands delete them after a successful restore. + D. Restore the databases +The `pg_terminate_backend` query disconnects the other sessions on each database server, such as the `pgsql-exporter` sidecar and the `worker`, so the dump can drop and recreate the database. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. + ```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/sourcegraph_db.out postgres' -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/codeintel_db.out postgres' -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/codeinsights_db.out template1' +$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' +$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' +$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' ``` E. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) @@ -346,44 +352,39 @@ $ kubectl scale --replicas=0 deployment/sourcegraph-frontend $ kubectl delete deployment sourcegraph-frontend ``` -B. Remove any existing volumes for the databases in the existing deployment - -```bash -$ kubectl delete pvc pgsql -$ kubectl delete pvc codeintel-db -$ kubectl delete pvc codeinsights-db -$ kubectl delete pv $pgsql_PV_NAME --force -$ kubectl delete pv $codeintel-db_PV_NAME --force -$ kubectl delete pv $codeinsights-db_PV_NAME --force -``` +You don't need to delete the database volumes: the restore drops and recreates each database. -C. Copy the database dump files (eg. `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out`) into your deployment directory +B. Copy the database dump files (eg. `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out`) into your deployment directory -D. Start the database services only +C. Start the database services only ```bash -$ kubectl rollout restart deployment pgsql -$ kubectl rollout restart deployment codeintel-db -$ kubectl rollout restart deployment codeinsights-db +$ kubectl rollout restart statefulset pgsql +$ kubectl rollout restart statefulset codeintel-db +$ kubectl rollout restart statefulset codeinsights-db ``` -E. Copy the database files into the pods by running the following commands: +D. Copy the database files into the pods by running the following commands: ```bash -$ kubectl cp sourcegraph_db.out $NAMESPACE/$pgsql_POD_NAME:/tmp/sourcegraph_db.out -$ kubectl cp codeintel_db.out $NAMESPACE/$codeintel-db_POD_NAME:/tmp/codeintel_db.out -$ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/tmp/codeinsights_db.out +$ kubectl cp sourcegraph_db.out $NAMESPACE/$pgsql_POD_NAME:/data/sourcegraph_db.out +$ kubectl cp codeintel_db.out $NAMESPACE/$codeintel-db_POD_NAME:/data/codeintel_db.out +$ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/var/lib/postgresql/data/codeinsights_db.out ``` -F. Restore the databases +The database containers have a read-only root filesystem, so copy the files into the database volumes. The restore commands delete them after a successful restore. + +E. Restore the databases + +The `pg_terminate_backend` query disconnects the other sessions on each database server, such as the `pgsql-exporter` sidecar and the `worker`, so the dump can drop and recreate the database. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. ```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/sourcegraph_db.out postgres' -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/codeintel_db.out postgres' -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /tmp/codeinsights_db.out template1' +$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' +$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' +$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' ``` -G. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) +F. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) ```sql create extension amcheck; @@ -401,7 +402,7 @@ and c.relpersistence != 't' and i.indisready AND i.indisvalid; ``` -H. Start the remaining Sourcegraph services by following the steps in [applying manifests](#applying-manifests). +G. Start the remaining Sourcegraph services by following the steps in [applying manifests](#applying-manifests). ## List of ports From 65a055dfa28262cd40c8e91c1831337d8e2a3a68 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:11:50 -0600 Subject: [PATCH 03/24] Allow statefulset in the spell check Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 Co-authored-by: Amp --- cspell-allow-list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cspell-allow-list.txt b/cspell-allow-list.txt index 5cb75f48b..5db8eeb76 100644 --- a/cspell-allow-list.txt +++ b/cspell-allow-list.txt @@ -519,6 +519,7 @@ starcoder Starlark startedat startswith +statefulset statefulsets stepscontainer stepsenv From 2a6957b094900284f721c30237d6980a5c9f621e Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:16 -0600 Subject: [PATCH 04/24] Leave starting codeinsights-db to db-only-migrate.docker-compose.yaml sourcegraph/deploy-sourcegraph-docker#1485 adds codeinsights-db to db-only-migrate.docker-compose.yaml, so the docs page does not need to start it separately. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 Co-authored-by: Amp --- docs/self-hosted/deploy/docker-compose/operations.mdx | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 74585690f..20e083f2a 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -81,7 +81,6 @@ zoekt-webserver-0 /sbin/tini -- /bin/sh -c z ... Up (healthy) ```bash docker-compose down docker-compose -f db-only-migrate.docker-compose.yaml up -d -docker-compose up -d codeinsights-db ``` 5\. Generate the database dumps @@ -104,7 +103,6 @@ The following instructions apply **only if you are restoring your databases into ```bash docker-compose -f db-only-migrate.docker-compose.yaml up -d -docker-compose up -d codeinsights-db ``` 3\. Copy the database files into the containers @@ -176,7 +174,6 @@ The volume names start with the Compose project name, which defaults to the dire ```bash docker-compose -f db-only-migrate.docker-compose.yaml up -d -docker-compose up -d codeinsights-db ``` 3\. Copy the database files into the containers From 7745b1bdb1264c78c2a302d88d924fa57bbcc46c Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:53:00 -0600 Subject: [PATCH 05/24] Stop every service but the databases before a Kubernetes restore Scaling down only sourcegraph-frontend left the worker and other services connected. They reconnect right after pg_terminate_backend, and can write into the new database while the dump loads: one test restore failed with a duplicate key on critical_and_site_config_pkey. Scale every Sourcegraph Deployment and StatefulSet except the three databases to 0 instead, which also replaces restarting the database StatefulSets. Drop each database with DROP DATABASE ... WITH (FORCE), which disconnects its sessions and drops it in one statement, and say on the machine where kubectl runs where the dump files go. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 Co-authored-by: Amp --- .../deploy/kubernetes/operations.mdx | 54 ++++++++----------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 010da8e2b..203566b79 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -292,17 +292,17 @@ The following instructions apply only if you are restoring your databases into a If you are restoring a previously running environment, see the instructions for [restoring a previously running deployment](#restoring-sourcegraph-databases-into-an-existing-environment) -A. Copy the database dump files (eg. `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out`) into your deployment directory +A. On the machine where you run `kubectl`, `cd` to the directory with the database dump files: `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` -B. Start the database services by running the following command from your deployment directory: +B. Stop every Sourcegraph service except the databases, so nothing writes to the databases during the restore ```bash -$ kubectl rollout restart statefulset pgsql -$ kubectl rollout restart statefulset codeintel-db -$ kubectl rollout restart statefulset codeinsights-db +$ kubectl scale deployment,statefulset -l 'deploy=sourcegraph,app.kubernetes.io/component notin (pgsql,codeintel-db,codeinsights-db)' --replicas=0 ``` -C. Copy the database files into the pods by running the following commands: +Wait until `kubectl get pods` shows no Sourcegraph pods other than `pgsql`, `codeintel-db`, and `codeinsights-db`. + +C. Copy the database files into the pods ```bash $ kubectl cp sourcegraph_db.out $NAMESPACE/$pgsql_POD_NAME:/data/sourcegraph_db.out @@ -314,12 +314,12 @@ The database containers have a read-only root filesystem, so copy the files into D. Restore the databases -The `pg_terminate_backend` query disconnects the other sessions on each database server, such as the `pgsql-exporter` sidecar and the `worker`, so the dump can drop and recreate the database. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. +`DROP DATABASE ... WITH (FORCE)` disconnects the other sessions on each database, such as the `pgsql-exporter` sidecar, and drops it, so the dump can recreate it. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. ```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' +$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' +$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' +$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' ``` E. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) @@ -344,27 +344,19 @@ F. Start the remaining Sourcegraph services by following the steps in [applying #### Restoring Sourcegraph databases into an existing environment -A. Stop the existing deployment by removing the frontend deployment - -```bash -$ kubectl scale --replicas=0 deployment/sourcegraph-frontend -# or -$ kubectl delete deployment sourcegraph-frontend -``` - You don't need to delete the database volumes: the restore drops and recreates each database. -B. Copy the database dump files (eg. `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out`) into your deployment directory +A. On the machine where you run `kubectl`, `cd` to the directory with the database dump files: `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` -C. Start the database services only +B. Stop every Sourcegraph service except the databases, so nothing writes to the databases during the restore ```bash -$ kubectl rollout restart statefulset pgsql -$ kubectl rollout restart statefulset codeintel-db -$ kubectl rollout restart statefulset codeinsights-db +$ kubectl scale deployment,statefulset -l 'deploy=sourcegraph,app.kubernetes.io/component notin (pgsql,codeintel-db,codeinsights-db)' --replicas=0 ``` -D. Copy the database files into the pods by running the following commands: +Wait until `kubectl get pods` shows no Sourcegraph pods other than `pgsql`, `codeintel-db`, and `codeinsights-db`. + +C. Copy the database files into the pods ```bash $ kubectl cp sourcegraph_db.out $NAMESPACE/$pgsql_POD_NAME:/data/sourcegraph_db.out @@ -374,17 +366,17 @@ $ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/var/lib/p The database containers have a read-only root filesystem, so copy the files into the database volumes. The restore commands delete them after a successful restore. -E. Restore the databases +D. Restore the databases -The `pg_terminate_backend` query disconnects the other sessions on each database server, such as the `pgsql-exporter` sidecar and the `worker`, so the dump can drop and recreate the database. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. +`DROP DATABASE ... WITH (FORCE)` disconnects the other sessions on each database, such as the `pgsql-exporter` sidecar, and drops it, so the dump can recreate it. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. ```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname <> current_database()" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' +$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' +$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' +$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' ``` -F. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) +E. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) ```sql create extension amcheck; @@ -402,7 +394,7 @@ and c.relpersistence != 't' and i.indisready AND i.indisvalid; ``` -G. Start the remaining Sourcegraph services by following the steps in [applying manifests](#applying-manifests). +F. Start the remaining Sourcegraph services by following the steps in [applying manifests](#applying-manifests). ## List of ports From 3e4c7db3b955eef95dfb32e9fd5c96aca27eed83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:20:38 +0000 Subject: [PATCH 06/24] Add PGDATA, vacuumdb to cspell-allow-list.txt --- cspell-allow-list.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cspell-allow-list.txt b/cspell-allow-list.txt index 5db8eeb76..ba469f385 100644 --- a/cspell-allow-list.txt +++ b/cspell-allow-list.txt @@ -405,6 +405,7 @@ pekko persistentvolumeclaims pgbouncer pgcrypto +PGDATA PGDATABASE PGDATASOURCE pgdump @@ -605,6 +606,7 @@ urlencode urlsafe userprincipalname USERTRUST +vacuumdb Valsorda vegeta vercel From fdcae074cdd91a78b1f943575b9c0f6d001a7482 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:39:18 -0600 Subject: [PATCH 07/24] Merge the duplicate database restore sections The new-environment and existing-environment restore procedures were the same steps written twice. On Kubernetes they are now identical, since the existing-environment steps no longer delete the database volumes. On Docker Compose only the first step differed, so it becomes a conditional step: stop the deployment and remove the volumes when restoring into an existing one. Also drop the stale `docker-compose ps` sample output from the restore steps; it listed services that no longer exist. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/docker-compose/operations.mdx | 102 ++---------------- .../deploy/kubernetes/operations.mdx | 60 +---------- 2 files changed, 11 insertions(+), 151 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 20e083f2a..e6876e6ea 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -93,73 +93,13 @@ docker exec codeinsights-db sh -c 'pg_dump -C --clean --if-exists --username pos 6\. Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` files are moved to a safe and secure location. -### Restore Sourcegraph databases into a new environment +### Restore Sourcegraph databases -The following instructions apply **only if you are restoring your databases into a new deployment** of Sourcegraph ie: a new virtual machine. If you are restoring a previously running environment, see the instructions for [restoring a previously running deployment](#restoring-sourcegraph-databases-into-an-existing-environment) +These instructions apply to both a new deployment and an existing one. -1\. Copy the database dump files into the `deploy-sourcegraph-docker/docker-compose` directory. +1\. Copy the database dump files into the `deploy-sourcegraph-docker/docker-compose` directory, and `cd` to it. -2\. Start the database services - -```bash -docker-compose -f db-only-migrate.docker-compose.yaml up -d -``` - -3\. Copy the database files into the containers - -```bash -docker cp sourcegraph_db.out pgsql:/tmp/sourcegraph_db.out -docker cp codeintel_db.out codeintel-db:/tmp/codeintel_db.out -docker cp codeinsights_db.out codeinsights-db:/tmp/codeinsights_db.out -``` - -4\. Restore the databases - -```bash -docker exec pgsql sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/sourcegraph_db.out postgres' -docker exec codeintel-db sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/codeintel_db.out postgres' -docker exec codeinsights-db sh -c 'psql -v ON_ERROR_STOP=1 --username postgres -f /tmp/codeinsights_db.out template1' -``` - -5\. Start the remaining Sourcegraph services - -```bash -docker-compose up -d -``` - -6\. Verify the deployment has started - -```bash -docker-compose ps - Name Command State Ports -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -caddy caddy run --config /etc/ca ... Up 2019/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:80->80/tcp -cadvisor /usr/bin/cadvisor -logtost ... Up (health: starting) 8080/tcp -codeinsights-db docker-entrypoint.sh postgres Up 5432/tcp -codeintel-db /postgres.sh Up (healthy) 5432/tcp -gitserver-0 /sbin/tini -- /usr/local/b ... Up -grafana /entry.sh Up 3000/tcp, 0.0.0.0:3370->3370/tcp -jaeger /go/bin/all-in-one-linux - ... Up 0.0.0.0:14250->14250/tcp, 14268/tcp, 0.0.0.0:16686->16686/tcp, 5775/udp, 0.0.0.0:5778->5778/tcp, - 0.0.0.0:6831->6831/tcp, 6831/udp, 0.0.0.0:6832->6832/tcp, 6832/udp -blobstore /usr/bin/docker-entrypoint ... Up (healthy) 9000/tcp -pgsql /postgres.sh Up (healthy) 5432/tcp -precise-code-intel-worker /sbin/tini -- /usr/local/b ... Up (health: starting) 3188/tcp -prometheus /bin/prom-wrapper Up 0.0.0.0:9090->9090/tcp -query-runner /sbin/tini -- /usr/local/b ... Up -redis-cache /sbin/tini -- redis-server ... Up 6379/tcp -redis-store /sbin/tini -- redis-server ... Up 6379/tcp -searcher-0 /sbin/tini -- /usr/local/b ... Up (healthy) -syntect-server sh -c /http-server-stabili ... Up (healthy) 9238/tcp -worker /sbin/tini -- /usr/local/b ... Up 3189/tcp -zoekt-indexserver-0 /sbin/tini -- zoekt-source ... Up -zoekt-webserver-0 /sbin/tini -- /bin/sh -c z ... Up (healthy)> docker-compose ps -``` - -7\. Browse to your Sourcegraph deployment, login and verify your existing configuration has been restored - -### Restore Sourcegraph databases into an existing environment - -1\. `cd` to the `deploy-sourcegraph-docker/docker-compose` and stop the previous deployment and remove any existing volumes +2\. If you are restoring into an existing deployment, stop it and remove the database volumes ```bash docker-compose down @@ -170,13 +110,13 @@ docker volume rm docker-compose_codeinsights-db The volume names start with the Compose project name, which defaults to the directory name, `docker-compose`. If you set a different project name, find the volume names with `docker volume ls`. -2\. Start the databases services only +3\. Start the database services ```bash docker-compose -f db-only-migrate.docker-compose.yaml up -d ``` -3\. Copy the database files into the containers +4\. Copy the database files into the containers ```bash docker cp sourcegraph_db.out pgsql:/tmp/sourcegraph_db.out @@ -184,7 +124,7 @@ docker cp codeintel_db.out codeintel-db:/tmp/codeintel_db.out docker cp codeinsights_db.out codeinsights-db:/tmp/codeinsights_db.out ``` -4\. Restore the databases +5\. Restore the databases ```bash docker exec pgsql sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/sourcegraph_db.out postgres' @@ -192,41 +132,19 @@ docker exec codeintel-db sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/co docker exec codeinsights-db sh -c 'psql -v ON_ERROR_STOP=1 --username postgres -f /tmp/codeinsights_db.out template1' ``` -5\. Start the remaining Sourcegraph services +6\. Start the remaining Sourcegraph services ```bash docker-compose up -d ``` -6\. Verify the deployment has started +7\. Verify the deployment has started ```bash docker-compose ps - Name Command State Ports -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -caddy caddy run --config /etc/ca ... Up 2019/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:80->80/tcp -cadvisor /usr/bin/cadvisor -logtost ... Up (health: starting) 8080/tcp -codeinsights-db docker-entrypoint.sh postgres Up 5432/tcp -codeintel-db /postgres.sh Up (healthy) 5432/tcp -gitserver-0 /sbin/tini -- /usr/local/b ... Up -grafana /entry.sh Up 3000/tcp, 0.0.0.0:3370->3370/tcp -jaeger /go/bin/all-in-one-linux - ... Up 0.0.0.0:14250->14250/tcp, 14268/tcp, 0.0.0.0:16686->16686/tcp, 5775/udp, 0.0.0.0:5778->5778/tcp, - 0.0.0.0:6831->6831/tcp, 6831/udp, 0.0.0.0:6832->6832/tcp, 6832/udp -blobstore /usr/bin/docker-entrypoint ... Up (healthy) 9000/tcp -pgsql /postgres.sh Up (healthy) 5432/tcp -precise-code-intel-worker /sbin/tini -- /usr/local/b ... Up (health: starting) 3188/tcp -prometheus /bin/prom-wrapper Up 0.0.0.0:9090->9090/tcp -query-runner /sbin/tini -- /usr/local/b ... Up -redis-cache /sbin/tini -- redis-server ... Up 6379/tcp -redis-store /sbin/tini -- redis-server ... Up 6379/tcp -searcher-0 /sbin/tini -- /usr/local/b ... Up (healthy) -syntect-server sh -c /http-server-stabili ... Up (healthy) 9238/tcp -worker /sbin/tini -- /usr/local/b ... Up 3189/tcp -zoekt-indexserver-0 /sbin/tini -- zoekt-source ... Up -zoekt-webserver-0 /sbin/tini -- /bin/sh -c z ... Up (healthy)> docker-compose ps ``` -7\. Browse to your Sourcegraph deployment, login and verify your existing configuration has been restored +8\. Browse to your Sourcegraph deployment, login and verify your existing configuration has been restored ## Monitoring diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 203566b79..cf3f71230 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -286,65 +286,7 @@ Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` fi ### Restore Sourcegraph databases -#### Restoring Sourcegraph databases into a new environment - -The following instructions apply only if you are restoring your databases into a new deployment of Sourcegraph ie: a new virtual machine - -If you are restoring a previously running environment, see the instructions for [restoring a previously running deployment](#restoring-sourcegraph-databases-into-an-existing-environment) - -A. On the machine where you run `kubectl`, `cd` to the directory with the database dump files: `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` - -B. Stop every Sourcegraph service except the databases, so nothing writes to the databases during the restore - -```bash -$ kubectl scale deployment,statefulset -l 'deploy=sourcegraph,app.kubernetes.io/component notin (pgsql,codeintel-db,codeinsights-db)' --replicas=0 -``` - -Wait until `kubectl get pods` shows no Sourcegraph pods other than `pgsql`, `codeintel-db`, and `codeinsights-db`. - -C. Copy the database files into the pods - -```bash -$ kubectl cp sourcegraph_db.out $NAMESPACE/$pgsql_POD_NAME:/data/sourcegraph_db.out -$ kubectl cp codeintel_db.out $NAMESPACE/$codeintel-db_POD_NAME:/data/codeintel_db.out -$ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/var/lib/postgresql/data/codeinsights_db.out -``` - -The database containers have a read-only root filesystem, so copy the files into the database volumes. The restore commands delete them after a successful restore. - -D. Restore the databases - -`DROP DATABASE ... WITH (FORCE)` disconnects the other sessions on each database, such as the `pgsql-exporter` sidecar, and drops it, so the dump can recreate it. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. - -```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' -``` - -E. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) - -```sql -create extension amcheck; - -select bt_index_parent_check(c.oid, true), c.relname, c.relpages -from pg_index i -join pg_opclass op ON i.indclass[0] = op.oid -join pg_am am ON op.opcmethod = am.oid -join pg_class c ON i.indexrelid = c.oid -join pg_namespace n ON c.relnamespace = n.oid -where am.amname = 'btree' --- Don't check temp tables, which may be from another session: -and c.relpersistence != 't' --- Function may throw an error when this is omitted: -and i.indisready AND i.indisvalid; -``` - -F. Start the remaining Sourcegraph services by following the steps in [applying manifests](#applying-manifests). - -#### Restoring Sourcegraph databases into an existing environment - -You don't need to delete the database volumes: the restore drops and recreates each database. +These instructions apply to both a new deployment and an existing one. You don't need to delete the database volumes: the restore drops and recreates each database. A. On the machine where you run `kubectl`, `cd` to the directory with the database dump files: `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` From a9e83a5b7dfa11f68175fe1c6f1a04b2b09d0cf3 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:58:59 -0600 Subject: [PATCH 08/24] Use the database pod names in the Kubernetes backup and restore steps The steps used `$pgsql_POD_NAME`, `$codeintel-db_POD_NAME`, and `$codeinsights-db_POD_NAME`, which the page never defines. The last two are not valid shell variables either: `$codeintel-db_POD_NAME` expands `$codeintel` and leaves `-db_POD_NAME`. The databases are StatefulSets with one replica in both the Helm chart and deploy-sourcegraph-k8s, so their pods are always `pgsql-0`, `codeintel-db-0`, and `codeinsights-db-0`. `kubectl cp` also drops the `$NAMESPACE/` prefix, so every command takes `-n $NAMESPACE` the same way, as the note says. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/kubernetes/operations.mdx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index cf3f71230..184526596 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -233,7 +233,7 @@ The following instructions are specific to backing up and restoring the sourcegr > > The above may take a while if you have a lot of repositories. In the meantime, searches may be slow or return incomplete results. This process rarely takes longer than 6 hours and is usually **much** faster. -> NOTE: In some places you will see `$NAMESPACE` used. Add `-n $NAMESPACE` to commands if you are not using the default namespace +> NOTE: Add `-n $NAMESPACE` to commands if you are not using the default namespace. The database pods are `pgsql-0`, `codeintel-db-0`, and `codeinsights-db-0`. > More kubectl configuration options can be found here: [kubectl Cheat Sheet](https://kubernetes.io/docs/reference/kubectl/cheatsheet/) ### Back up Sourcegraph databases @@ -275,9 +275,9 @@ and i.indisready AND i.indisvalid; D. Generate the database dumps ```bash -$ kubectl exec $pgsql_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > sourcegraph_db.out -$ kubectl exec $codeintel-db_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > codeintel_db.out -$ kubectl exec $codeinsights-db_POD_NAME -- bash -c 'pg_dump -C --clean --if-exists --username postgres postgres' > codeinsights_db.out +$ kubectl exec pgsql-0 -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > sourcegraph_db.out +$ kubectl exec codeintel-db-0 -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > codeintel_db.out +$ kubectl exec codeinsights-db-0 -- bash -c 'pg_dump -C --clean --if-exists --username postgres postgres' > codeinsights_db.out ``` Run these commands without `-it`: with a terminal attached, every line of the dump ends with a carriage return. @@ -301,9 +301,9 @@ Wait until `kubectl get pods` shows no Sourcegraph pods other than `pgsql`, `cod C. Copy the database files into the pods ```bash -$ kubectl cp sourcegraph_db.out $NAMESPACE/$pgsql_POD_NAME:/data/sourcegraph_db.out -$ kubectl cp codeintel_db.out $NAMESPACE/$codeintel-db_POD_NAME:/data/codeintel_db.out -$ kubectl cp codeinsights_db.out $NAMESPACE/$codeinsights-db_POD_NAME:/var/lib/postgresql/data/codeinsights_db.out +$ kubectl cp sourcegraph_db.out pgsql-0:/data/sourcegraph_db.out +$ kubectl cp codeintel_db.out codeintel-db-0:/data/codeintel_db.out +$ kubectl cp codeinsights_db.out codeinsights-db-0:/var/lib/postgresql/data/codeinsights_db.out ``` The database containers have a read-only root filesystem, so copy the files into the database volumes. The restore commands delete them after a successful restore. @@ -313,9 +313,9 @@ D. Restore the databases `DROP DATABASE ... WITH (FORCE)` disconnects the other sessions on each database, such as the `pgsql-exporter` sidecar, and drops it, so the dump can recreate it. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. ```bash -$ kubectl exec -it $pgsql_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' -$ kubectl exec -it $codeintel-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' -$ kubectl exec -it $codeinsights-db_POD_NAME -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' +$ kubectl exec -it pgsql-0 -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' +$ kubectl exec -it codeintel-db-0 -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' +$ kubectl exec -it codeinsights-db-0 -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' ``` E. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) From 43bbf6c2adf29eb70d566eaa2344479e45b37ec7 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:00:03 -0600 Subject: [PATCH 09/24] Drop deleting the frontend deployment from the Kubernetes backup steps Scaling it to zero stops its connections just as well, and the final step of applying the manifests scales it back up. Deleting it leaves Helm and Kustomize users to recreate it. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- docs/self-hosted/deploy/kubernetes/operations.mdx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 184526596..cbf32dee7 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -246,12 +246,10 @@ A. Verify deployment running $ kubectl get pods -A ``` -B. Stop all connections to the database by removing the frontend deployment +B. Stop all connections to the database by scaling down the frontend deployment ```bash $ kubectl scale --replicas=0 deployment/sourcegraph-frontend -# or -$ kubectl delete deployment sourcegraph-frontend ``` C. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) From 1046cd4e2cc5c1758ff224aab1f78130f8eacab1 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:21:08 -0600 Subject: [PATCH 10/24] Break the long database backup and restore commands into short lines The docs site doesn't wrap code blocks, so these commands scrolled sideways: up to 253 characters, in a block that fits 75 at desktop width. Each line is now at most 67 characters. The commands pass the same arguments as before. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d140-e6de-7173-8b4f-187543b7833c Co-authored-by: Amp --- .../deploy/docker-compose/operations.mdx | 28 ++++++++++--- .../deploy/kubernetes/operations.mdx | 42 +++++++++++++++---- 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index e6876e6ea..7b47c831c 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -86,9 +86,17 @@ docker-compose -f db-only-migrate.docker-compose.yaml up -d 5\. Generate the database dumps ```bash -docker exec pgsql sh -c 'pg_dump -C --clean --if-exists --username sg sg' > sourcegraph_db.out -docker exec codeintel-db sh -c 'pg_dump -C --clean --if-exists --username sg sg' > codeintel_db.out -docker exec codeinsights-db sh -c 'pg_dump -C --clean --if-exists --username postgres postgres' > codeinsights_db.out +docker exec pgsql sh -c \ + 'pg_dump -C --clean --if-exists --username sg sg' \ + > sourcegraph_db.out + +docker exec codeintel-db sh -c \ + 'pg_dump -C --clean --if-exists --username sg sg' \ + > codeintel_db.out + +docker exec codeinsights-db sh -c \ + 'pg_dump -C --clean --if-exists --username postgres postgres' \ + > codeinsights_db.out ``` 6\. Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` files are moved to a safe and secure location. @@ -127,9 +135,17 @@ docker cp codeinsights_db.out codeinsights-db:/tmp/codeinsights_db.out 5\. Restore the databases ```bash -docker exec pgsql sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/sourcegraph_db.out postgres' -docker exec codeintel-db sh -c 'psql -v ON_ERROR_STOP=1 --username sg -f /tmp/codeintel_db.out postgres' -docker exec codeinsights-db sh -c 'psql -v ON_ERROR_STOP=1 --username postgres -f /tmp/codeinsights_db.out template1' +docker exec pgsql sh -c ' + psql -v ON_ERROR_STOP=1 --username sg \ + -f /tmp/sourcegraph_db.out postgres' + +docker exec codeintel-db sh -c ' + psql -v ON_ERROR_STOP=1 --username sg \ + -f /tmp/codeintel_db.out postgres' + +docker exec codeinsights-db sh -c ' + psql -v ON_ERROR_STOP=1 --username postgres \ + -f /tmp/codeinsights_db.out template1' ``` 6\. Start the remaining Sourcegraph services diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index cbf32dee7..34a22b09b 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -273,9 +273,17 @@ and i.indisready AND i.indisvalid; D. Generate the database dumps ```bash -$ kubectl exec pgsql-0 -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > sourcegraph_db.out -$ kubectl exec codeintel-db-0 -- bash -c 'pg_dump -C --clean --if-exists --username sg sg' > codeintel_db.out -$ kubectl exec codeinsights-db-0 -- bash -c 'pg_dump -C --clean --if-exists --username postgres postgres' > codeinsights_db.out +$ kubectl exec pgsql-0 -- bash -c \ + 'pg_dump -C --clean --if-exists --username sg sg' \ + > sourcegraph_db.out + +$ kubectl exec codeintel-db-0 -- bash -c \ + 'pg_dump -C --clean --if-exists --username sg sg' \ + > codeintel_db.out + +$ kubectl exec codeinsights-db-0 -- bash -c \ + 'pg_dump -C --clean --if-exists --username postgres postgres' \ + > codeinsights_db.out ``` Run these commands without `-it`: with a terminal attached, every line of the dump ends with a carriage return. @@ -291,7 +299,10 @@ A. On the machine where you run `kubectl`, `cd` to the directory with the databa B. Stop every Sourcegraph service except the databases, so nothing writes to the databases during the restore ```bash -$ kubectl scale deployment,statefulset -l 'deploy=sourcegraph,app.kubernetes.io/component notin (pgsql,codeintel-db,codeinsights-db)' --replicas=0 +$ kubectl scale deployment,statefulset --replicas=0 -l ' + deploy=sourcegraph, + app.kubernetes.io/component notin + (pgsql,codeintel-db,codeinsights-db)' ``` Wait until `kubectl get pods` shows no Sourcegraph pods other than `pgsql`, `codeintel-db`, and `codeinsights-db`. @@ -301,7 +312,8 @@ C. Copy the database files into the pods ```bash $ kubectl cp sourcegraph_db.out pgsql-0:/data/sourcegraph_db.out $ kubectl cp codeintel_db.out codeintel-db-0:/data/codeintel_db.out -$ kubectl cp codeinsights_db.out codeinsights-db-0:/var/lib/postgresql/data/codeinsights_db.out +$ kubectl cp codeinsights_db.out \ + codeinsights-db-0:/var/lib/postgresql/data/codeinsights_db.out ``` The database containers have a read-only root filesystem, so copy the files into the database volumes. The restore commands delete them after a successful restore. @@ -311,9 +323,23 @@ D. Restore the databases `DROP DATABASE ... WITH (FORCE)` disconnects the other sessions on each database, such as the `pgsql-exporter` sidecar, and drops it, so the dump can recreate it. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. ```bash -$ kubectl exec -it pgsql-0 -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/sourcegraph_db.out postgres && rm /data/sourcegraph_db.out' -$ kubectl exec -it codeintel-db-0 -- bash -c 'psql -v ON_ERROR_STOP=1 --username sg -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" -f /data/codeintel_db.out postgres && rm /data/codeintel_db.out' -$ kubectl exec -it codeinsights-db-0 -- bash -c 'psql -v ON_ERROR_STOP=1 --username postgres -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" -f /var/lib/postgresql/data/codeinsights_db.out template1 && rm /var/lib/postgresql/data/codeinsights_db.out' +$ kubectl exec -it pgsql-0 -- bash -c ' + psql -v ON_ERROR_STOP=1 --username sg \ + -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" \ + -f /data/sourcegraph_db.out postgres \ + && rm /data/sourcegraph_db.out' + +$ kubectl exec -it codeintel-db-0 -- bash -c ' + psql -v ON_ERROR_STOP=1 --username sg \ + -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" \ + -f /data/codeintel_db.out postgres \ + && rm /data/codeintel_db.out' + +$ kubectl exec -it codeinsights-db-0 -- bash -c ' + psql -v ON_ERROR_STOP=1 --username postgres \ + -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" \ + -f /var/lib/postgresql/data/codeinsights_db.out template1 \ + && rm /var/lib/postgresql/data/codeinsights_db.out' ``` E. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) From 9bb4268b37097e07276cc760c237857ca2632a3f Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:36:04 -0600 Subject: [PATCH 11/24] Back up Kubernetes databases live, and restart them with Helm Backup: - Drop scaling down the frontend. pg_dump takes a consistent snapshot of each database while services stay connected, and scaling down only the frontend left the worker and others connected anyway. The backup also never scaled the frontend back up. - Step A asks for a baseline of the running pods and their replicas. - The amcheck step says it blocks writes to each table while it checks that table's indexes, since services now keep running. Restore: - Run pg_dump and psql directly with kubectl exec, without bash -c, so no quotes nest. - Deleting the dump files from the volumes is its own step, after all three restores succeed. - Restart the services with helm upgrade, which sets the replicas that the scale-down step zeroed back to the values; Kustomize users keep the applying manifests link. Run as written on m6 (Helm 8.2.0, Helm 4.3.1), with the backup taken while every service ran: all exit codes 0, no ERROR lines, the three re-dumps identical to the backup, GraphQL data matches the seeded data, and every workload back at 1/1. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/kubernetes/operations.mdx | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 34a22b09b..5691fd0a6 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -240,19 +240,13 @@ The following instructions are specific to backing up and restoring the sourcegr These instructions will back up the primary `sourcegraph` database, the [codeintel](/code-navigation/) database, and the [codeinsights](/code-insights/) database. -A. Verify deployment running +A. Get a baseline understanding of which Sourcegraph pods are running / healthy, and how many replicas of each ```bash -$ kubectl get pods -A +$ kubectl get pods ``` -B. Stop all connections to the database by scaling down the frontend deployment - -```bash -$ kubectl scale --replicas=0 deployment/sourcegraph-frontend -``` - -C. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) +B. Check for corrupt database indexes. The check blocks writes to each table while it checks that table's indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) ```sql create extension amcheck; @@ -270,19 +264,19 @@ and c.relpersistence != 't' and i.indisready AND i.indisvalid; ``` -D. Generate the database dumps +C. Generate the database dumps ```bash -$ kubectl exec pgsql-0 -- bash -c \ - 'pg_dump -C --clean --if-exists --username sg sg' \ +$ kubectl exec pgsql-0 -- \ + pg_dump -C --clean --if-exists --username sg sg \ > sourcegraph_db.out -$ kubectl exec codeintel-db-0 -- bash -c \ - 'pg_dump -C --clean --if-exists --username sg sg' \ +$ kubectl exec codeintel-db-0 -- \ + pg_dump -C --clean --if-exists --username sg sg \ > codeintel_db.out -$ kubectl exec codeinsights-db-0 -- bash -c \ - 'pg_dump -C --clean --if-exists --username postgres postgres' \ +$ kubectl exec codeinsights-db-0 -- \ + pg_dump -C --clean --if-exists --username postgres postgres \ > codeinsights_db.out ``` @@ -316,33 +310,39 @@ $ kubectl cp codeinsights_db.out \ codeinsights-db-0:/var/lib/postgresql/data/codeinsights_db.out ``` -The database containers have a read-only root filesystem, so copy the files into the database volumes. The restore commands delete them after a successful restore. +The database containers have a read-only root filesystem, so copy the files into the database volumes. Step E deletes them after the restore. D. Restore the databases `DROP DATABASE ... WITH (FORCE)` disconnects the other sessions on each database, such as the `pgsql-exporter` sidecar, and drops it, so the dump can recreate it. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. ```bash -$ kubectl exec -it pgsql-0 -- bash -c ' +$ kubectl exec -it pgsql-0 -- \ psql -v ON_ERROR_STOP=1 --username sg \ - -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" \ - -f /data/sourcegraph_db.out postgres \ - && rm /data/sourcegraph_db.out' + -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" \ + -f /data/sourcegraph_db.out postgres -$ kubectl exec -it codeintel-db-0 -- bash -c ' +$ kubectl exec -it codeintel-db-0 -- \ psql -v ON_ERROR_STOP=1 --username sg \ - -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" \ - -f /data/codeintel_db.out postgres \ - && rm /data/codeintel_db.out' + -c "DROP DATABASE IF EXISTS sg WITH (FORCE)" \ + -f /data/codeintel_db.out postgres -$ kubectl exec -it codeinsights-db-0 -- bash -c ' +$ kubectl exec -it codeinsights-db-0 -- \ psql -v ON_ERROR_STOP=1 --username postgres \ - -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" \ - -f /var/lib/postgresql/data/codeinsights_db.out template1 \ - && rm /var/lib/postgresql/data/codeinsights_db.out' + -c "DROP DATABASE IF EXISTS postgres WITH (FORCE)" \ + -f /var/lib/postgresql/data/codeinsights_db.out template1 +``` + +E. After all three restores succeed, delete the database files from the pods + +```bash +$ kubectl exec pgsql-0 -- rm /data/sourcegraph_db.out +$ kubectl exec codeintel-db-0 -- rm /data/codeintel_db.out +$ kubectl exec codeinsights-db-0 -- \ + rm /var/lib/postgresql/data/codeinsights_db.out ``` -E. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) +F. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) ```sql create extension amcheck; @@ -360,7 +360,15 @@ and c.relpersistence != 't' and i.indisready AND i.indisvalid; ``` -F. Start the remaining Sourcegraph services by following the steps in [applying manifests](#applying-manifests). +G. Start the remaining Sourcegraph services, by running `helm upgrade` with the chart version and values file of your deployment (`helm list` shows the chart version). It sets every service back to the number of replicas in your values. + +```bash +$ helm upgrade --install --values ./override.yaml \ + --version \ + sourcegraph sourcegraph/sourcegraph +``` + +If you deployed with Kustomize, follow the steps in [applying manifests](#applying-manifests) instead. ## List of ports From 7ece69e8f60896ab8ab78aa8f24f8a826b757a82 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:52:21 -0600 Subject: [PATCH 12/24] Call the dumps backup files in the restore cleanup step Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- docs/self-hosted/deploy/kubernetes/operations.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 5691fd0a6..2b92d8071 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -333,7 +333,7 @@ $ kubectl exec -it codeinsights-db-0 -- \ -f /var/lib/postgresql/data/codeinsights_db.out template1 ``` -E. After all three restores succeed, delete the database files from the pods +E. After all three restores succeed, delete the backup files from the pods ```bash $ kubectl exec pgsql-0 -- rm /data/sourcegraph_db.out From 4dfe5b1e67125cdde09b63d43ca9faf438671027 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:53:42 -0600 Subject: [PATCH 13/24] Correct the Kubernetes pod names, drop the Kustomize and upgrade links "Access the database" showed `pgsql-76a4bfcd64-rt4cn`, a pod name from when pgsql was a Deployment; it is a StatefulSet, so its pod is always `pgsql-0`. The restore's wait step now names the database pods too. Drop the "Migrate to Kustomize" and "Upgrade" sections, which only linked elsewhere; Kustomize is being deprecated. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/kubernetes/operations.mdx | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 2b92d8071..6670ab279 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -196,12 +196,12 @@ $ kubectl rollout restart deployment sourcegraph-frontend ## Access the database -Get the id of one `pgsql` Pod: +Check that the `pgsql` pod is running: ```bash $ kubectl get pods -l app=pgsql -NAME READY STATUS RESTARTS AGE -pgsql-76a4bfcd64-rt4cn 2/2 Running 0 19m +NAME READY STATUS RESTARTS AGE +pgsql-0 2/2 Running 0 19m ``` Make sure you are operating under the correct namespace (i.e. add `-n prod` if your pod is under the `prod` namespace). @@ -209,7 +209,7 @@ Make sure you are operating under the correct namespace (i.e. add `-n prod` if y Open a PostgreSQL interactive terminal: ```bash -$ kubectl exec -it pgsql-76a4bfcd64-rt4cn -- psql -U sg +$ kubectl exec -it pgsql-0 -- psql -U sg ``` Run your SQL query: @@ -299,7 +299,7 @@ $ kubectl scale deployment,statefulset --replicas=0 -l ' (pgsql,codeintel-db,codeinsights-db)' ``` -Wait until `kubectl get pods` shows no Sourcegraph pods other than `pgsql`, `codeintel-db`, and `codeinsights-db`. +Wait until `kubectl get pods` shows no Sourcegraph pods other than `pgsql-0`, `codeintel-db-0`, and `codeinsights-db-0`. C. Copy the database files into the pods @@ -405,15 +405,6 @@ syntect-server ClusterIP 10.72.14.49 9238/TC worker ClusterIP 10.72.7.72 3189/TCP,6060/TCP 25h ``` -## Migrate to Kustomize - -See the [migration docs for Kustomize](/self-hosted/deploy/kubernetes/kustomize/migrate) for more information. - -## Upgrade - -- See the [Updating Sourcegraph docs](/self-hosted/updates/) on how to upgrade.
-- See the [Updating a Kubernetes Sourcegraph instance docs](/self-hosted/deploy/kubernetes/upgrade) for details on changes in each version to determine if manual migration steps are necessary. - ## Troubleshoot See the [Troubleshooting docs](/self-hosted/deploy/kubernetes/troubleshoot). From 0949357bc2eb3cfdc16c5aa7539eae3b35eb0106 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:03:11 -0600 Subject: [PATCH 14/24] Run the Kubernetes amcheck query with kubectl exec The amcheck SQL had no command to run it. Save it as amcheck.sql and pipe it into psql on each database pod, with ON_ERROR_STOP=1 so a failed check exits 3 instead of 0, and stdout discarded so only errors show. The restore points at the backup's commands instead of repeating the query. - `create extension if not exists`: amcheck already exists in all three databases, so the plain form failed every time. - `c.relkind = 'i'`: on codeintel-db the check failed with "only B-Tree indexes are supported as targets for verification" on the partitioned index codeintel_documents_pkey. Step A now asks to verify the pods, especially the three databases. Tested on m6: exit 0 on all three databases; without the relkind filter, codeintel-db prints the error and exits 3. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/kubernetes/operations.mdx | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 6670ab279..0ada75f85 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -240,7 +240,7 @@ The following instructions are specific to backing up and restoring the sourcegr These instructions will back up the primary `sourcegraph` database, the [codeintel](/code-navigation/) database, and the [codeinsights](/code-insights/) database. -A. Get a baseline understanding of which Sourcegraph pods are running / healthy, and how many replicas of each +A. Verify your Sourcegraph pods are running / healthy, especially the 3 database pods ```bash $ kubectl get pods @@ -248,8 +248,10 @@ $ kubectl get pods B. Check for corrupt database indexes. The check blocks writes to each table while it checks that table's indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) +Save this query in a file named `amcheck.sql`: + ```sql -create extension amcheck; +create extension if not exists amcheck; select bt_index_parent_check(c.oid, true), c.relname, c.relpages from pg_index i @@ -260,10 +262,28 @@ join pg_namespace n ON c.relnamespace = n.oid where am.amname = 'btree' -- Don't check temp tables, which may be from another session: and c.relpersistence != 't' +-- Skip partitioned indexes; the indexes of their partitions are checked: +and c.relkind = 'i' -- Function may throw an error when this is omitted: and i.indisready AND i.indisvalid; ``` +Run it on each database. Each command prints any errors, and exits 0 when every index is intact: + +```bash +$ kubectl exec -i pgsql-0 -- \ + psql -v ON_ERROR_STOP=1 --username sg sg \ + < amcheck.sql > /dev/null + +$ kubectl exec -i codeintel-db-0 -- \ + psql -v ON_ERROR_STOP=1 --username sg sg \ + < amcheck.sql > /dev/null + +$ kubectl exec -i codeinsights-db-0 -- \ + psql -v ON_ERROR_STOP=1 --username postgres postgres \ + < amcheck.sql > /dev/null +``` + C. Generate the database dumps ```bash @@ -342,23 +362,7 @@ $ kubectl exec codeinsights-db-0 -- \ rm /var/lib/postgresql/data/codeinsights_db.out ``` -F. Check for corrupt database indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) - -```sql -create extension amcheck; - -select bt_index_parent_check(c.oid, true), c.relname, c.relpages -from pg_index i -join pg_opclass op ON i.indclass[0] = op.oid -join pg_am am ON op.opcmethod = am.oid -join pg_class c ON i.indexrelid = c.oid -join pg_namespace n ON c.relnamespace = n.oid -where am.amname = 'btree' --- Don't check temp tables, which may be from another session: -and c.relpersistence != 't' --- Function may throw an error when this is omitted: -and i.indisready AND i.indisvalid; -``` +F. Check for corrupt database indexes, with the commands in step B of [Back up Sourcegraph databases](#back-up-sourcegraph-databases). If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) G. Start the remaining Sourcegraph services, by running `helm upgrade` with the chart version and values file of your deployment (`helm list` shows the chart version). It sets every service back to the number of replicas in your values. From 2948d4fcd7d6407ff069c5a273377ae132a83255 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:14:32 -0600 Subject: [PATCH 15/24] Back up Docker Compose databases live, and use docker compose Drop the backup step that stopped the deployment and started only the databases. pg_dump takes a consistent snapshot of each database while services stay connected, and the backup never started the deployment again, so following it left Sourcegraph down. This matches the Kubernetes page. Use the `docker compose` plugin instead of the old standalone `docker-compose` command in the examples. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/docker-compose/operations.mdx | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 7b47c831c..5889a08e8 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -25,7 +25,7 @@ docker exec -it codeinsights-db psql -U postgres #access codeinsights-db contain The `frontend` container in the `docker-compose.yaml` file will automatically run on startup and migrate the databases if any changes are required, however administrators may wish to migrate their databases before upgrading the rest of the system when working with large databases. Sourcegraph guarantees database backward compatibility to the most recent minor point release so the database can safely be upgraded before the application code. -To execute the database migrations independently, follow the [docker-compose instructions on how to manually run database migrations](/self-hosted/updates/migrator/migrator-operations#docker-compose). Running the `up` (default) command on the `migrator` of the _version you are upgrading to_ will apply all migrations required by the next version of Sourcegraph. +To execute the database migrations independently, follow the [Docker Compose instructions on how to manually run database migrations](/self-hosted/updates/migrator/migrator-operations#docker-compose). Running the `up` (default) command on the `migrator` of the _version you are upgrading to_ will apply all migrations required by the next version of Sourcegraph. ## Backup and restore @@ -51,7 +51,7 @@ These instructions will back up the primary `sourcegraph` database, the [codeint 3\. Verify the deployment is running: ```bash -docker-compose ps +docker compose ps Name Command State Ports ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- caddy caddy run --config /etc/ca ... Up 2019/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:80->80/tcp @@ -76,14 +76,7 @@ zoekt-indexserver-0 /sbin/tini -- zoekt-source ... Up zoekt-webserver-0 /sbin/tini -- /bin/sh -c z ... Up (healthy) ``` -4\. Stop the deployment, and restart the databases service only to ensure there are no other connections during backup and restore. - -```bash -docker-compose down -docker-compose -f db-only-migrate.docker-compose.yaml up -d -``` - -5\. Generate the database dumps +4\. Generate the database dumps ```bash docker exec pgsql sh -c \ @@ -99,7 +92,7 @@ docker exec codeinsights-db sh -c \ > codeinsights_db.out ``` -6\. Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` files are moved to a safe and secure location. +5\. Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` files are moved to a safe and secure location. ### Restore Sourcegraph databases @@ -110,7 +103,7 @@ These instructions apply to both a new deployment and an existing one. 2\. If you are restoring into an existing deployment, stop it and remove the database volumes ```bash -docker-compose down +docker compose down docker volume rm docker-compose_pgsql docker volume rm docker-compose_codeintel-db docker volume rm docker-compose_codeinsights-db @@ -121,7 +114,7 @@ The volume names start with the Compose project name, which defaults to the dire 3\. Start the database services ```bash -docker-compose -f db-only-migrate.docker-compose.yaml up -d +docker compose -f db-only-migrate.docker-compose.yaml up -d ``` 4\. Copy the database files into the containers @@ -151,13 +144,13 @@ docker exec codeinsights-db sh -c ' 6\. Start the remaining Sourcegraph services ```bash -docker-compose up -d +docker compose up -d ``` 7\. Verify the deployment has started ```bash -docker-compose ps +docker compose ps ``` 8\. Browse to your Sourcegraph deployment, login and verify your existing configuration has been restored From 5c8401266cde8d0881cdb2a1599b0c5402cb4d10 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:16:40 -0600 Subject: [PATCH 16/24] Number the backup and restore steps as Markdown lists AGENTS.md asks for sequentially numbered ordered Markdown lists. The Kubernetes page lettered its steps, the only page in the repo to do so, and the Docker Compose page escaped its numbers (`1\.`) so they were not lists. Both now use `1.`, `2.`, `3.`. The Kubernetes restore also ends with the Docker Compose page's last step: browse to Sourcegraph and check the configuration came back. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/docker-compose/operations.mdx | 26 +++++++++---------- .../deploy/kubernetes/operations.mdx | 24 +++++++++-------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 5889a08e8..4aff6a5df 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -44,11 +44,11 @@ The following instructions are specific to backing up and restoring the Sourcegr These instructions will back up the primary `sourcegraph` database, the [codeintel](/code-navigation/) database, and the [codeinsights](/code-insights/) database. -1\. `ssh` from your local machine into the machine hosting the `sourcegraph` deployment +1. `ssh` from your local machine into the machine hosting the `sourcegraph` deployment -2\. `cd` to the `deploy-sourcegraph-docker/docker-compose` directory on the host +2. `cd` to the `deploy-sourcegraph-docker/docker-compose` directory on the host -3\. Verify the deployment is running: +3. Verify the deployment is running: ```bash docker compose ps @@ -76,7 +76,7 @@ zoekt-indexserver-0 /sbin/tini -- zoekt-source ... Up zoekt-webserver-0 /sbin/tini -- /bin/sh -c z ... Up (healthy) ``` -4\. Generate the database dumps +4. Generate the database dumps ```bash docker exec pgsql sh -c \ @@ -92,15 +92,15 @@ docker exec codeinsights-db sh -c \ > codeinsights_db.out ``` -5\. Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` files are moved to a safe and secure location. +5. Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` files are moved to a safe and secure location. ### Restore Sourcegraph databases These instructions apply to both a new deployment and an existing one. -1\. Copy the database dump files into the `deploy-sourcegraph-docker/docker-compose` directory, and `cd` to it. +1. Copy the database dump files into the `deploy-sourcegraph-docker/docker-compose` directory, and `cd` to it. -2\. If you are restoring into an existing deployment, stop it and remove the database volumes +2. If you are restoring into an existing deployment, stop it and remove the database volumes ```bash docker compose down @@ -111,13 +111,13 @@ docker volume rm docker-compose_codeinsights-db The volume names start with the Compose project name, which defaults to the directory name, `docker-compose`. If you set a different project name, find the volume names with `docker volume ls`. -3\. Start the database services +3. Start the database services ```bash docker compose -f db-only-migrate.docker-compose.yaml up -d ``` -4\. Copy the database files into the containers +4. Copy the database files into the containers ```bash docker cp sourcegraph_db.out pgsql:/tmp/sourcegraph_db.out @@ -125,7 +125,7 @@ docker cp codeintel_db.out codeintel-db:/tmp/codeintel_db.out docker cp codeinsights_db.out codeinsights-db:/tmp/codeinsights_db.out ``` -5\. Restore the databases +5. Restore the databases ```bash docker exec pgsql sh -c ' @@ -141,19 +141,19 @@ docker exec codeinsights-db sh -c ' -f /tmp/codeinsights_db.out template1' ``` -6\. Start the remaining Sourcegraph services +6. Start the remaining Sourcegraph services ```bash docker compose up -d ``` -7\. Verify the deployment has started +7. Verify the deployment has started ```bash docker compose ps ``` -8\. Browse to your Sourcegraph deployment, login and verify your existing configuration has been restored +8. Browse to your Sourcegraph deployment, login and verify your existing configuration has been restored ## Monitoring diff --git a/docs/self-hosted/deploy/kubernetes/operations.mdx b/docs/self-hosted/deploy/kubernetes/operations.mdx index 0ada75f85..1f9ab6bcf 100644 --- a/docs/self-hosted/deploy/kubernetes/operations.mdx +++ b/docs/self-hosted/deploy/kubernetes/operations.mdx @@ -240,13 +240,13 @@ The following instructions are specific to backing up and restoring the sourcegr These instructions will back up the primary `sourcegraph` database, the [codeintel](/code-navigation/) database, and the [codeinsights](/code-insights/) database. -A. Verify your Sourcegraph pods are running / healthy, especially the 3 database pods +1. Verify your Sourcegraph pods are running / healthy, especially the 3 database pods ```bash $ kubectl get pods ``` -B. Check for corrupt database indexes. The check blocks writes to each table while it checks that table's indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) +2. Check for corrupt database indexes. The check blocks writes to each table while it checks that table's indexes. If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) Save this query in a file named `amcheck.sql`: @@ -284,7 +284,7 @@ $ kubectl exec -i codeinsights-db-0 -- \ < amcheck.sql > /dev/null ``` -C. Generate the database dumps +3. Generate the database dumps ```bash $ kubectl exec pgsql-0 -- \ @@ -308,9 +308,9 @@ Ensure the `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` fi These instructions apply to both a new deployment and an existing one. You don't need to delete the database volumes: the restore drops and recreates each database. -A. On the machine where you run `kubectl`, `cd` to the directory with the database dump files: `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` +1. On the machine where you run `kubectl`, `cd` to the directory with the database dump files: `sourcegraph_db.out`, `codeintel_db.out` and `codeinsights_db.out` -B. Stop every Sourcegraph service except the databases, so nothing writes to the databases during the restore +2. Stop every Sourcegraph service except the databases, so nothing writes to the databases during the restore ```bash $ kubectl scale deployment,statefulset --replicas=0 -l ' @@ -321,7 +321,7 @@ $ kubectl scale deployment,statefulset --replicas=0 -l ' Wait until `kubectl get pods` shows no Sourcegraph pods other than `pgsql-0`, `codeintel-db-0`, and `codeinsights-db-0`. -C. Copy the database files into the pods +3. Copy the database files into the pods ```bash $ kubectl cp sourcegraph_db.out pgsql-0:/data/sourcegraph_db.out @@ -330,9 +330,9 @@ $ kubectl cp codeinsights_db.out \ codeinsights-db-0:/var/lib/postgresql/data/codeinsights_db.out ``` -The database containers have a read-only root filesystem, so copy the files into the database volumes. Step E deletes them after the restore. +The database containers have a read-only root filesystem, so copy the files into the database volumes. Step 5 deletes them after the restore. -D. Restore the databases +4. Restore the databases `DROP DATABASE ... WITH (FORCE)` disconnects the other sessions on each database, such as the `pgsql-exporter` sidecar, and drops it, so the dump can recreate it. `ON_ERROR_STOP=1` stops the restore at the first error; if a restore fails, fix the cause and run it again. @@ -353,7 +353,7 @@ $ kubectl exec -it codeinsights-db-0 -- \ -f /var/lib/postgresql/data/codeinsights_db.out template1 ``` -E. After all three restores succeed, delete the backup files from the pods +5. After all three restores succeed, delete the backup files from the pods ```bash $ kubectl exec pgsql-0 -- rm /data/sourcegraph_db.out @@ -362,9 +362,9 @@ $ kubectl exec codeinsights-db-0 -- \ rm /var/lib/postgresql/data/codeinsights_db.out ``` -F. Check for corrupt database indexes, with the commands in step B of [Back up Sourcegraph databases](#back-up-sourcegraph-databases). If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) +6. Check for corrupt database indexes, with the commands in step 2 of [Back up Sourcegraph databases](#back-up-sourcegraph-databases). If amcheck returns errors, please reach out to [support@sourcegraph.com](mailto:support@sourcegraph.com) -G. Start the remaining Sourcegraph services, by running `helm upgrade` with the chart version and values file of your deployment (`helm list` shows the chart version). It sets every service back to the number of replicas in your values. +7. Start the remaining Sourcegraph services, by running `helm upgrade` with the chart version and values file of your deployment (`helm list` shows the chart version). It sets every service back to the number of replicas in your values. ```bash $ helm upgrade --install --values ./override.yaml \ @@ -374,6 +374,8 @@ $ helm upgrade --install --values ./override.yaml \ If you deployed with Kustomize, follow the steps in [applying manifests](#applying-manifests) instead. +8. Browse to your Sourcegraph deployment, log in, and verify your existing configuration has been restored + ## List of ports To see a list of ports that are currently being used by your Sourcegraph instance: From 46281e7ce5b5be597a385173236dc8a01cb8ce72 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:20:57 -0600 Subject: [PATCH 17/24] Show current docker compose ps output in the Compose backup steps The sample was docker-compose v1 output from an old release, listing jaeger and query-runner, which 8.2.0 no longer runs. Replace it with output captured from a fresh 8.2.0 stack. The command prints only names and status, since plain `docker compose ps` prints pinned image digests and runs past 200 characters. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/docker-compose/operations.mdx | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 4aff6a5df..42823bbb1 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -51,29 +51,33 @@ These instructions will back up the primary `sourcegraph` database, the [codeint 3. Verify the deployment is running: ```bash -docker compose ps - Name Command State Ports -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -caddy caddy run --config /etc/ca ... Up 2019/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:80->80/tcp -cadvisor /usr/bin/cadvisor -logtost ... Up (health: starting) 8080/tcp -codeinsights-db docker-entrypoint.sh postgres Up 5432/tcp -codeintel-db /postgres.sh Up (healthy) 5432/tcp -gitserver-0 /sbin/tini -- /usr/local/b ... Up -grafana /entry.sh Up 3000/tcp, 0.0.0.0:3370->3370/tcp -jaeger /go/bin/all-in-one-linux - ... Up 0.0.0.0:14250->14250/tcp, 14268/tcp, 0.0.0.0:16686->16686/tcp, 5775/udp, 0.0.0.0:5778->5778/tcp, - 0.0.0.0:6831->6831/tcp, 6831/udp, 0.0.0.0:6832->6832/tcp, 6832/udp -blobstore /usr/bin/docker-entrypoint ... Up (healthy) 9000/tcp -pgsql /postgres.sh Up (healthy) 5432/tcp -precise-code-intel-worker /sbin/tini -- /usr/local/b ... Up (health: starting) 3188/tcp -prometheus /bin/prom-wrapper Up 0.0.0.0:9090->9090/tcp -query-runner /sbin/tini -- /usr/local/b ... Up -redis-cache /sbin/tini -- redis-server ... Up 6379/tcp -redis-store /sbin/tini -- redis-server ... Up 6379/tcp -searcher-0 /sbin/tini -- /usr/local/b ... Up (healthy) -syntect-server sh -c /http-server-stabili ... Up (healthy) 9238/tcp -worker /sbin/tini -- /usr/local/b ... Up 3189/tcp -zoekt-indexserver-0 /sbin/tini -- zoekt-source ... Up -zoekt-webserver-0 /sbin/tini -- /bin/sh -c z ... Up (healthy) +docker compose ps --format 'table {{.Name}}\t{{.Status}}' +NAME STATUS +blobstore Up 49 seconds (healthy) +caddy Up 46 seconds +cadvisor Up 48 seconds +codeinsights-db Up 44 seconds (healthy) +codeinsights-db-exporter Up 44 seconds +codeintel-db Up 46 seconds (healthy) +codeintel-db-exporter Up 46 seconds +gitserver-0 Up 48 seconds +grafana Up 48 seconds +node-exporter Up 48 seconds +otel-collector Up 48 seconds +pgsql Up 47 seconds (healthy) +pgsql-exporter Up 47 seconds +precise-code-intel-worker Up 45 seconds (healthy) +prometheus Up 47 seconds +redis-cache Up 45 seconds +redis-store Up 44 seconds +searcher-0 Up 47 seconds (healthy) +sourcegraph-frontend-0 Up 20 seconds (healthy) +sourcegraph-frontend-internal Up 30 seconds (healthy) +syntactic-code-intel-worker Up 46 seconds (healthy) +syntect-server Up 45 seconds (healthy) +worker Up 45 seconds +zoekt-indexserver-0 Up 45 seconds +zoekt-webserver-0 Up 49 seconds (healthy) ``` 4. Generate the database dumps From 488c5ab2e932c06e22542148ef9939e34e1a0d30 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:25:59 -0600 Subject: [PATCH 18/24] Fix the amcheck query on the index corruption how-to pages The same two fixes as the Kubernetes operations page: - `create extension if not exists amcheck`: the plain form errors on databases that already have amcheck. - `c.relkind = 'i'`: without it the check stops on codeintel-db with "only B-Tree indexes are supported as targets for verification" at the partitioned index codeintel_documents_pkey, which reads like corruption and leaves the remaining indexes unchecked. Tested on m6 (8.2.0): both pages' queries exit 0 with no errors on all three databases. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- docs/self-hosted/how-to/postgres14-index-corruption.mdx | 4 +++- docs/self-hosted/how-to/rebuild-corrupt-postgres-indexes.mdx | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/self-hosted/how-to/postgres14-index-corruption.mdx b/docs/self-hosted/how-to/postgres14-index-corruption.mdx index 027867f81..7c9c064c8 100644 --- a/docs/self-hosted/how-to/postgres14-index-corruption.mdx +++ b/docs/self-hosted/how-to/postgres14-index-corruption.mdx @@ -17,7 +17,7 @@ SELECT version(); You may also check for index corruption in your database using the `amcheck` by running the following query in your database ``` -create extension amcheck; +create extension if not exists amcheck; select bt_index_parent_check(c.oid, true), c.relname, c.relpages from pg_index i @@ -28,6 +28,8 @@ join pg_namespace n ON c.relnamespace = n.oid where am.amname = 'btree' -- Don't check temp tables, which may be from another session: and c.relpersistence != 't' +-- Skip partitioned indexes; the indexes of their partitions are checked: +and c.relkind = 'i' -- Function may throw an error when this is omitted: and i.indisready AND i.indisvalid; ``` diff --git a/docs/self-hosted/how-to/rebuild-corrupt-postgres-indexes.mdx b/docs/self-hosted/how-to/rebuild-corrupt-postgres-indexes.mdx index 521646e55..07c5554d6 100644 --- a/docs/self-hosted/how-to/rebuild-corrupt-postgres-indexes.mdx +++ b/docs/self-hosted/how-to/rebuild-corrupt-postgres-indexes.mdx @@ -57,7 +57,7 @@ After deleting duplicates, just re-run the above statement. Repeat the process u At the end of the index rebuilding process, as a last sanity check, we use the amcheck extension to verify there are no corrupt indexes — an error is raised if there are (you should expect to see some output from this command). ```sql -create extension amcheck; +create extension if not exists amcheck; select bt_index_parent_check(c.oid, true), c.relname, c.relpages from pg_index i @@ -68,6 +68,8 @@ join pg_namespace n ON c.relnamespace = n.oid where am.amname = 'btree' -- Don't check temp tables, which may be from another session: and c.relpersistence != 't' +-- Skip partitioned indexes; the indexes of their partitions are checked: +and c.relkind = 'i' -- Function may throw an error when this is omitted: and i.indisready AND i.indisvalid; ``` From 552485ebd94e303ffde68bd4fe7b058e6a248f67 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:25:59 -0600 Subject: [PATCH 19/24] Run pg_dump and psql without sh -c on the Docker Compose page docker exec runs them directly, as the Kubernetes page does with kubectl exec, so no quotes nest. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/docker-compose/operations.mdx | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 42823bbb1..9e44096b3 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -83,16 +83,16 @@ zoekt-webserver-0 Up 49 seconds (healthy) 4. Generate the database dumps ```bash -docker exec pgsql sh -c \ - 'pg_dump -C --clean --if-exists --username sg sg' \ +docker exec pgsql \ + pg_dump -C --clean --if-exists --username sg sg \ > sourcegraph_db.out -docker exec codeintel-db sh -c \ - 'pg_dump -C --clean --if-exists --username sg sg' \ +docker exec codeintel-db \ + pg_dump -C --clean --if-exists --username sg sg \ > codeintel_db.out -docker exec codeinsights-db sh -c \ - 'pg_dump -C --clean --if-exists --username postgres postgres' \ +docker exec codeinsights-db \ + pg_dump -C --clean --if-exists --username postgres postgres \ > codeinsights_db.out ``` @@ -132,17 +132,17 @@ docker cp codeinsights_db.out codeinsights-db:/tmp/codeinsights_db.out 5. Restore the databases ```bash -docker exec pgsql sh -c ' +docker exec pgsql \ psql -v ON_ERROR_STOP=1 --username sg \ - -f /tmp/sourcegraph_db.out postgres' + -f /tmp/sourcegraph_db.out postgres -docker exec codeintel-db sh -c ' +docker exec codeintel-db \ psql -v ON_ERROR_STOP=1 --username sg \ - -f /tmp/codeintel_db.out postgres' + -f /tmp/codeintel_db.out postgres -docker exec codeinsights-db sh -c ' +docker exec codeinsights-db \ psql -v ON_ERROR_STOP=1 --username postgres \ - -f /tmp/codeinsights_db.out template1' + -f /tmp/codeinsights_db.out template1 ``` 6. Start the remaining Sourcegraph services From 930f5d221f0f34e7165d247a5620f43c28b1d24d Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:27:36 -0600 Subject: [PATCH 20/24] Delete the Postgres 14 index corruption page It covers a bug in PostgreSQL 14.0 to 14.3. Sourcegraph now requires PostgreSQL 16, so no supported deployment can hit it. Remove the two links to it, and redirect its old URLs to "How to rebuild corrupt Postgres indexes". Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- docs/self-hosted/how-to/index.mdx | 1 - .../how-to/postgres14-index-corruption.mdx | 59 ------------------- docs/self-hosted/index.mdx | 1 - src/data/redirects.ts | 6 +- 4 files changed, 5 insertions(+), 62 deletions(-) delete mode 100644 docs/self-hosted/how-to/postgres14-index-corruption.mdx diff --git a/docs/self-hosted/how-to/index.mdx b/docs/self-hosted/how-to/index.mdx index 63ada9327..29917151f 100644 --- a/docs/self-hosted/how-to/index.mdx +++ b/docs/self-hosted/how-to/index.mdx @@ -22,7 +22,6 @@ - [How to address common monorepo problems](/admin/how-to/monorepo-issues) - [How to Set a password for Redis using a ConfigMap](/self-hosted/how-to/redis-configmap) - [How to import a set of internal repositories to Sourcegraph](/admin/how-to/internal-github-repos) -- [How to identify and resolve index corruption in postgres 14](/self-hosted/how-to/postgres14-index-corruption) - [How to export search results](/admin/how-to/export-search-results) - [How to debug / confirm blobstore is healthy](/self-hosted/how-to/blobstore-debugging) - [How to handle postgresql 12 to 16 drift](/self-hosted/how-to/postgres-12-to-16-drift) diff --git a/docs/self-hosted/how-to/postgres14-index-corruption.mdx b/docs/self-hosted/how-to/postgres14-index-corruption.mdx deleted file mode 100644 index 7c9c064c8..000000000 --- a/docs/self-hosted/how-to/postgres14-index-corruption.mdx +++ /dev/null @@ -1,59 +0,0 @@ -# How to identify and resolve index corruption in Postgres v14.0 - v14.3 - -A bug has been identified in PostgreSQL 14.0-14.3 that can cause database index corruption in indexes created concurrently. Sourcegraph [utilizes](https://sourcegraph.com/search?q=context:global+repo:%5Egithub%5C.com/sourcegraph/sourcegraph%24+%22CREATE+INDEX%22+AND+CONCURRENTLY&patternType=keyword) concurrent index creation, and if you are on these versions, you may experience slow queries or database corruption. To learn more about this bug see PostgreSQL's [out-of-cycle release announcement](https://www.postgresql.org/message-id/165473835807.573551.1512237163040609764%40wrigleys.postgresql.org) or [migops article](https://www.migops.com/blog/important-postgresql-14-update-to-avoid-silent-corruption-of-indexes/) on the subject. - -> NOTE: If you are running a default Sourcegraph deployment with default Postgres image values, it is likely Postgres is running in version 12 and your instance will be unaffected by this bug. - -## Identify your database version - -To identify which version of Sourcegraph you are running in a default Sourcegraph deployment. You can access your database via the `docker` or `kubectl` cli tools and run the following command: - -``` -SELECT version(); -``` - -> NOTE: You can refer to the following instructions for accessing databases on your deployment type: [Docker Compose](/self-hosted/deploy/docker-compose/#access-the-database), [Kubernetes](/self-hosted/deploy/kubernetes/operations#access-the-database). - -You may also check for index corruption in your database using the `amcheck` by running the following query in your database - -``` -create extension if not exists amcheck; - -select bt_index_parent_check(c.oid, true), c.relname, c.relpages -from pg_index i -join pg_opclass op ON i.indclass[0] = op.oid -join pg_am am ON op.opcmethod = am.oid -join pg_class c ON i.indexrelid = c.oid -join pg_namespace n ON c.relnamespace = n.oid -where am.amname = 'btree' --- Don't check temp tables, which may be from another session: -and c.relpersistence != 't' --- Skip partitioned indexes; the indexes of their partitions are checked: -and c.relkind = 'i' --- Function may throw an error when this is omitted: -and i.indisready AND i.indisvalid; -``` - -This query will return error if there are corrupted indexes. - -## Resolve - -If you are impacted, you can remediate this by upgrading to a newer version of Postgres (14.4+) and running the following commands in your database - -Determine database name: - -``` -SELECT current_database(); -``` - -``` -REINDEX DATABASE ; -``` - -_You may want to use the amcheck query above to verify the reindex has resolved index corruption._ - -### Upgrading your database - -In default Sourcegraph deployments internal PostgreSQL instances are used and may be upgraded via the [pg_upgrade](https://www.postgresql.org/docs/11/pgupgrade.html). For external databases consult your service providers documentation. For a deeper look at database upgrade operations please consult our [PostgreSQL documentation](/self-hosted/postgres#upgrading-postgresql). - -If you have any questions, please reach out to support on Slack or email support@sourcegraph.com. diff --git a/docs/self-hosted/index.mdx b/docs/self-hosted/index.mdx index abb846d17..c71afe2e0 100644 --- a/docs/self-hosted/index.mdx +++ b/docs/self-hosted/index.mdx @@ -99,7 +99,6 @@ Get started running Sourcegraph on-prem. - [Clear code intelligence data](/self-hosted/how-to/clear-codeintel-data) - [Dirty database](/self-hosted/how-to/dirty-database) - [PostgreSQL 12 to 16 drift](/self-hosted/how-to/postgres-12-to-16-drift) -- [PostgreSQL 14 index corruption](/self-hosted/how-to/postgres14-index-corruption) - [Privileged migrations](/self-hosted/how-to/privileged-migrations) - [Rebuild corrupt PostgreSQL indexes](/self-hosted/how-to/rebuild-corrupt-postgres-indexes) - [Redis ConfigMap](/self-hosted/how-to/redis-configmap) diff --git a/src/data/redirects.ts b/src/data/redirects.ts index 8fcdb8c25..4f9c316e8 100644 --- a/src/data/redirects.ts +++ b/src/data/redirects.ts @@ -3435,7 +3435,11 @@ const redirectsData = [ }, { source: '/admin/how-to/postgres14-index-corruption', - destination: '/self-hosted/how-to/postgres14-index-corruption' + destination: '/self-hosted/how-to/rebuild-corrupt-postgres-indexes' + }, + { + source: '/self-hosted/how-to/postgres14-index-corruption', + destination: '/self-hosted/how-to/rebuild-corrupt-postgres-indexes' }, { source: '/admin/how-to/postgres_12_to_16_drift', From c418fb90307dfba5a704e19d7de42e598fd8c808 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:45:19 -0600 Subject: [PATCH 21/24] Drop the Database Migrations section from the Docker Compose operations page It only restated that the frontend migrates on startup and linked to the migrator docs. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- docs/self-hosted/deploy/docker-compose/operations.mdx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 9e44096b3..70dc19571 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -21,12 +21,6 @@ docker exec -it codeintel-db psql -U sg #access codeintel-db container and run p docker exec -it codeinsights-db psql -U postgres #access codeinsights-db container and run psql ``` -## Database Migrations - -The `frontend` container in the `docker-compose.yaml` file will automatically run on startup and migrate the databases if any changes are required, however administrators may wish to migrate their databases before upgrading the rest of the system when working with large databases. Sourcegraph guarantees database backward compatibility to the most recent minor point release so the database can safely be upgraded before the application code. - -To execute the database migrations independently, follow the [Docker Compose instructions on how to manually run database migrations](/self-hosted/updates/migrator/migrator-operations#docker-compose). Running the `up` (default) command on the `migrator` of the _version you are upgrading to_ will apply all migrations required by the next version of Sourcegraph. - ## Backup and restore The following instructions are specific to backing up and restoring the Sourcegraph databases in a Docker Compose deployment. These do not apply to other deployment types. From 63fe5c0f6743b0cc69c40cb56fa60d752a6f693b Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:46:50 -0600 Subject: [PATCH 22/24] Shorten the Docker Compose psql commands Drop the trailing comments, which only repeated the container name and pushed the lines past the width of a code block. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- docs/self-hosted/deploy/docker-compose/operations.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 70dc19571..1a6e05974 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -13,12 +13,12 @@ Guides for managing cloud storage and backups are available in our cloud-specifi ## Access the database -The following command allows a user to shell into the Sourcegraph database container and run `psql` to interact with the container's postgres database: +To open a `psql` session on one of the databases, run the command for its container: ```bash -docker exec -it pgsql psql -U sg #access pgsql container and run psql -docker exec -it codeintel-db psql -U sg #access codeintel-db container and run psql -docker exec -it codeinsights-db psql -U postgres #access codeinsights-db container and run psql +docker exec -it pgsql psql -U sg +docker exec -it codeintel-db psql -U sg +docker exec -it codeinsights-db psql -U postgres ``` ## Backup and restore From 1c95b389fbf435615c59ab0fa4dd3bf8a0276925 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:50:22 -0600 Subject: [PATCH 23/24] Keep the database volumes in the Docker Compose restore The restore drops and recreates each database, and db-only-migrate.docker-compose.yaml starts only the three databases, so nothing else is connected to block the dump's DROP DATABASE. Deleting the volumes is unnecessary, as on the Kubernetes page; stopping the deployment is still needed so nothing writes during the restore. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- docs/self-hosted/deploy/docker-compose/operations.mdx | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 1a6e05974..504955602 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -94,21 +94,16 @@ docker exec codeinsights-db \ ### Restore Sourcegraph databases -These instructions apply to both a new deployment and an existing one. +These instructions apply to both a new deployment and an existing one. You don't need to delete the database volumes: the restore drops and recreates each database. 1. Copy the database dump files into the `deploy-sourcegraph-docker/docker-compose` directory, and `cd` to it. -2. If you are restoring into an existing deployment, stop it and remove the database volumes +2. If you are restoring into an existing deployment, stop it, so nothing writes to the databases during the restore ```bash docker compose down -docker volume rm docker-compose_pgsql -docker volume rm docker-compose_codeintel-db -docker volume rm docker-compose_codeinsights-db ``` -The volume names start with the Compose project name, which defaults to the directory name, `docker-compose`. If you set a different project name, find the volume names with `docker volume ls`. - 3. Start the database services ```bash From 5698cb05a74c58347079fe2ac2c50d1b5d8349e1 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:55:01 -0600 Subject: [PATCH 24/24] Show Jaeger and the executor in the Compose ps sample, note -f files The docker compose ps sample now comes from an 8.2.0 stack started with the Jaeger and executor overrides, so it lists `jaeger` and `executor`. Plain `docker compose ps` lists them too. Plain `docker compose down` and `up -d` ignore those override files: jaeger and executor kept running through a test restore, and the final `up -d` recreated the frontend without `JAEGER_SERVER_URL`. The restore now says to pass the same `-f` options as when starting Sourcegraph. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1d197-7a9c-71fc-9998-6e55df9c47d0 --- .../deploy/docker-compose/operations.mdx | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/operations.mdx b/docs/self-hosted/deploy/docker-compose/operations.mdx index 504955602..5d5f54470 100644 --- a/docs/self-hosted/deploy/docker-compose/operations.mdx +++ b/docs/self-hosted/deploy/docker-compose/operations.mdx @@ -47,31 +47,33 @@ These instructions will back up the primary `sourcegraph` database, the [codeint ```bash docker compose ps --format 'table {{.Name}}\t{{.Status}}' NAME STATUS -blobstore Up 49 seconds (healthy) -caddy Up 46 seconds -cadvisor Up 48 seconds -codeinsights-db Up 44 seconds (healthy) -codeinsights-db-exporter Up 44 seconds -codeintel-db Up 46 seconds (healthy) -codeintel-db-exporter Up 46 seconds -gitserver-0 Up 48 seconds -grafana Up 48 seconds -node-exporter Up 48 seconds -otel-collector Up 48 seconds -pgsql Up 47 seconds (healthy) -pgsql-exporter Up 47 seconds -precise-code-intel-worker Up 45 seconds (healthy) -prometheus Up 47 seconds -redis-cache Up 45 seconds -redis-store Up 44 seconds -searcher-0 Up 47 seconds (healthy) -sourcegraph-frontend-0 Up 20 seconds (healthy) -sourcegraph-frontend-internal Up 30 seconds (healthy) -syntactic-code-intel-worker Up 46 seconds (healthy) -syntect-server Up 45 seconds (healthy) -worker Up 45 seconds -zoekt-indexserver-0 Up 45 seconds -zoekt-webserver-0 Up 49 seconds (healthy) +blobstore Up About a minute (healthy) +caddy Up About a minute +cadvisor Up About a minute +codeinsights-db Up About a minute (healthy) +codeinsights-db-exporter Up About a minute +codeintel-db Up About a minute (healthy) +codeintel-db-exporter Up About a minute +executor Up About a minute +gitserver-0 Up About a minute +grafana Up About a minute +jaeger Up About a minute +node-exporter Up About a minute +otel-collector Up About a minute +pgsql Up About a minute (healthy) +pgsql-exporter Up About a minute +precise-code-intel-worker Up About a minute (healthy) +prometheus Up About a minute +redis-cache Up About a minute +redis-store Up About a minute +searcher-0 Up About a minute (healthy) +sourcegraph-frontend-0 Up About a minute (healthy) +sourcegraph-frontend-internal Up About a minute (healthy) +syntactic-code-intel-worker Up About a minute (healthy) +syntect-server Up About a minute (healthy) +worker Up About a minute +zoekt-indexserver-0 Up About a minute +zoekt-webserver-0 Up About a minute (healthy) ``` 4. Generate the database dumps @@ -96,6 +98,8 @@ docker exec codeinsights-db \ These instructions apply to both a new deployment and an existing one. You don't need to delete the database volumes: the restore drops and recreates each database. +If you started Sourcegraph with more Compose files, such as `jaeger/docker-compose.yaml` or `executors/executor.docker-compose.yaml`, pass the same `-f` options to `docker compose down` and `docker compose up -d`. Otherwise those containers keep running during the restore, and Sourcegraph restarts without their settings. + 1. Copy the database dump files into the `deploy-sourcegraph-docker/docker-compose` directory, and `cd` to it. 2. If you are restoring into an existing deployment, stop it, so nothing writes to the databases during the restore