docker-compose: Remove unused settings, fix the default collector config, and correct the examples

caddy/builtins/https.custom-cert.Caddyfile

@@ -5,12 +5,6 @@
 
 {$SRC_SITE_ADDRESS}
 
-# Redirects all HTTP traffic to HTTPS (using custom SSL certificates disables the automatic HTTPS feature of Caddy, see https://caddyserver.com/docs/automatic-https)
-@http {
-	protocol http
-}
-redir @http https://{host}{uri}
-
 tls /sourcegraph.pem /sourcegraph.key
 
 # Add the reverse proxies IPs (or IP CIDR ranges) to the trusted_proxies list.

docker-compose/dev/docker-compose.yaml

@@ -1,5 +1,5 @@
-# Development overlay - has reduced resource requirements, and exposes the deployment on
-# 127.0.0.1:8080
+# Development overlay - has reduced resource requirements, and also publishes the deployment
+# on port 8080 of every host interface, in addition to ports 80 and 443 from the base file
 
 services:
   zoekt-webserver-0:

docker-compose/docker-compose.yaml

@@ -33,9 +33,8 @@ services:
       - CODEINTEL_PGDATABASE=sg
       - CODEINTEL_PGSSLMODE=disable
 
-      # If you are not running code insights, please see:
-      # https://sourcegraph.com/docs/self-hosted/deploy/docker-compose/operations#database-migrations for more information
-      # for information on how to configure Sourcegraph migrations
+      # For the migrator's environment variables and commands, see:
+      # https://sourcegraph.com/docs/self-hosted/updates/migrator/migrator-operations
       - CODEINSIGHTS_PGHOST=codeinsights-db
       - CODEINSIGHTS_PGPORT=5432
       - CODEINSIGHTS_PGUSER=postgres
@@ -117,14 +116,15 @@ services:
 
   # Description: Serves the frontend of Sourcegraph via HTTP(S).
   #
-  # Disk: 128GB / non-persistent SSD
-  # Ports exposed to other Sourcegraph services: 6060/TCP, 3080 (HTTP), and/or 3443 (HTTPS)
+  # Disk: none
+  # Ports exposed to other Sourcegraph services: 6060/TCP, 3080 (HTTP)
   # Ports exposed to the public internet: none
   #
-  # Note: SRC_GIT_SERVERS, and SEARCHER_URL are space-separated
-  # lists which each allow you to specify more container instances for scaling
-  # purposes. Be sure to also apply such a change here to the frontend-internal
-  # service.
+  # Note: SRC_GIT_SERVERS, SEARCHER_URL, INDEXED_SEARCH_SERVERS, and INDEXED_SEARCH_INDEXER
+  # each take either a space-separated list of addresses, or a number of replicas,
+  # ex. SRC_GIT_SERVERS=2 is the same as SRC_GIT_SERVERS=gitserver-0:3178 gitserver-1:3178.
+  # Only the two frontend services read them; be sure to apply the same change to the
+  # frontend-internal service.
   sourcegraph-frontend-0:
     container_name: sourcegraph-frontend-0
     image: 'index.docker.io/sourcegraph/frontend:8.2.0@sha256:00adad80d2a9e49c586d6735a1ddbc2e89fc06d6af0a3b668aefbcb7802f2e95'
@@ -154,6 +154,8 @@ services:
             { "Name": "frontend-internal-0", "Host": "sourcegraph-frontend-internal:6060" },
             { "Name": "gitserver-0", "Host": "gitserver-0:6060" },
             { "Name": "searcher-0", "Host": "searcher-0:6060" },
+            { "Name": "precise-code-intel-worker", "Host": "precise-code-intel-worker:6060" },
+            { "Name": "syntactic-code-intel-worker", "Host": "syntactic-code-intel-worker:6060" },
             { "Name": "worker", "Host": "worker:6060" },
             { "Name": "zoekt-indexserver-0", "Host": "zoekt-indexserver-0:6072" },
             { "Name": "zoekt-webserver-0", "Host": "zoekt-webserver-0:6070", "DefaultPath": "/debug/requests/" }
@@ -164,8 +166,6 @@ services:
       timeout: 10s
       retries: 10
       start_period: 300s
-    volumes:
-      - 'sourcegraph-frontend-0:/mnt/cache'
     networks:
       - sourcegraph
     restart: always
@@ -175,7 +175,7 @@ services:
 
   # Description: Serves the internal Sourcegraph frontend API.
   #
-  # Disk: 128GB / non-persistent SSD
+  # Disk: none
   # Ports exposed to other Sourcegraph services: 3090/TCP 6060/TCP
   # Ports exposed to the public internet: none
   #
@@ -201,8 +201,6 @@ services:
       - 'GRAFANA_SERVER_URL=http://grafana:3370'
       - 'PROMETHEUS_URL=http://prometheus:9090'
       - 'OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317'
-    volumes:
-      - 'sourcegraph-frontend-internal-0:/mnt/cache'
     networks:
       - sourcegraph
     restart: always
@@ -262,8 +260,6 @@ services:
     environment:
       - 'HOSTNAME=zoekt-webserver-0:6070'
       - 'SRC_FRONTEND_INTERNAL=http://sourcegraph-frontend-internal:3090'
-      - 'OPENTELEMETRY_DISABLED=false'
-      - 'OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317'
     volumes:
       - 'zoekt-0-shared:/data/index'
     networks:
@@ -356,8 +352,8 @@ services:
 
   # Description: Manages background processes.
   #
-  # Disk: 128GB / non-persistent SSD
-  # Ports exposed to other Sourcegraph services: 3189/TCP 6060/TCP
+  # Disk: none
+  # Ports exposed to other Sourcegraph services: 3189/TCP 6060/TCP 6996/TCP (executor metrics)
   # Ports exposed to the public internet: none
   #
   worker:
@@ -371,11 +367,6 @@ services:
     environment:
       - 'SRC_FRONTEND_INTERNAL=sourcegraph-frontend-internal:3090'
       - 'OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317'
-      - 'SRC_GIT_SERVERS=gitserver-0:3178'
-      - 'SEARCHER_URL=http://searcher-0:3181'
-      - 'INDEXED_SEARCH_SERVERS=zoekt-webserver-0:6070'
-    volumes:
-      - 'worker:/mnt/cache'
     networks:
       - sourcegraph
     restart: always
@@ -433,7 +424,7 @@ services:
   # Description: Prometheus collects metrics and aggregates them into graphs.
   #
   # Disk: 200GB / persistent SSD
-  # Ports exposed to other Sourcegraph services: none
+  # Ports exposed to other Sourcegraph services: 9090/TCP
   # Ports exposed to the public internet: none (HTTP 9090 should be exposed to admins only)
   #
   prometheus:
@@ -463,8 +454,17 @@ services:
   # Add the following environment variables if you wish to use an auth proxy with Grafana:
   #
   # 'GF_AUTH_PROXY_ENABLED=true'
-  # 'GF_AUTH_PROXY_HEADER_NAME='X-Forwarded-User'
-  # 'GF_SERVER_ROOT_URL='https://grafana.example.com'
+  # 'GF_AUTH_PROXY_HEADER_NAME=X-Forwarded-User'
+  #
+  # Add all of the following environment variables if you wish to use the `pgsql` datasource,
+  # to query a Postgres database from Grafana; the datasource does not work until all six are set:
+  #
+  # 'GRAFANA_PGSQL_HOST=pgsql'
+  # 'GRAFANA_PGSQL_PORT=5432'
+  # 'GRAFANA_PGSQL_USER=sg'
+  # 'GRAFANA_PGSQL_PASSWORD=sg'
+  # 'GRAFANA_PGSQL_DATABASE=sg'
+  # 'GRAFANA_PGSQL_SSLMODE=disable'
   grafana:
     container_name: grafana
     image: 'index.docker.io/sourcegraph/grafana:8.2.0@sha256:fd478ac1a5ce4552bfc388984a559548b5b7ea8001c2e65adcfb7e292bb54784'
@@ -493,7 +493,7 @@ services:
     mem_limit: '1g'
     security_opt:
       - no-new-privileges:true
-    # You may set `privileged` to `false and `cadvisor` will run with reduced privileges.
+    # You may set `privileged` to `false` and `cadvisor` will run with reduced privileges.
     # `cadvisor` requires root privileges in order to display provisioning metrics.
     # These metrics provide critical information to help you scale the Sourcegraph deployment.
     # If you would like to bring your own infrastructure monitoring & alerting solution,
@@ -512,8 +512,6 @@ services:
     networks:
       - sourcegraph
     restart: always
-    expose:
-      - 48080
 
   # Description: Publishes Prometheus metrics about the machine's hardware / operating system.
   #
@@ -540,16 +538,14 @@ services:
       - '--no-collector.wifi'
       - '--no-collector.hwmon'
       - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
-    expose:
-      - 9100
     networks:
       - sourcegraph
     restart: always
 
   # Description: PostgreSQL database for various data.
   #
   # Disk: 128GB / persistent SSD
-  # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP
+  # Ports exposed to other Sourcegraph services: 5432/TCP
   # Ports exposed to the public internet: none
   #
   pgsql:
@@ -600,7 +596,7 @@ services:
   #
   # Disk: 128GB / persistent SSD
   # Network: 1Gbps
-  # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP
+  # Ports exposed to other Sourcegraph services: 5432/TCP
   # Ports exposed to the public internet: none
   #
   codeintel-db:
@@ -651,7 +647,7 @@ services:
   #
   # Disk: 128GB / persistent SSD
   # Network: 1Gbps
-  # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP
+  # Ports exposed to other Sourcegraph services: 5432/TCP
   # Ports exposed to the public internet: none
   #
   codeinsights-db:
@@ -732,7 +728,7 @@ services:
   # Description: Redis for storing short-lived caches.
   #
   # Disk: 128GB / persistent SSD
-  # Ports exposed to other Sourcegraph services: 6379/TCP 9121/TCP
+  # Ports exposed to other Sourcegraph services: 6379/TCP
   # Ports exposed to the public internet: none
   #
   redis-cache:
@@ -750,7 +746,7 @@ services:
   # Description: Redis for storing semi-persistent data like user sessions.
   #
   # Disk: 128GB / persistent SSD
-  # Ports exposed to other Sourcegraph services: 6379/TCP 9121/TCP
+  # Ports exposed to other Sourcegraph services: 6379/TCP
   # Ports exposed to the public internet: none
   #
   redis-store:
@@ -768,7 +764,7 @@ services:
 
   # Description: Ingests and exports OpenTelemetry data.
   #
-  # Ports exposed to other Sourcegraph services: 4317 (receiver), 55679 (zpages)
+  # Ports exposed to other Sourcegraph services: 4317 (OTLP gRPC), 4318 (OTLP HTTP), 8888 (metrics)
   # Ports exposed to the public internet: none
   otel-collector:
     container_name: otel-collector
@@ -780,8 +776,8 @@ services:
     networks:
       - sourcegraph
     restart: always
-    # Default configurations are shipped in '/etc/otel-collector/configs/'
-    command: ['--config', '/etc/otel-collector/configs/logging.yaml']
+    # Other configurations are shipped in '/etc/otel-collector/configs/'
+    command: ['--config', '/etc/otel-collector/config.yaml']
     volumes:
       - '../otel-collector/config.yaml:/etc/otel-collector/config.yaml'
 
@@ -796,10 +792,7 @@ volumes:
   prometheus-v2:
   redis-cache:
   redis-store:
-  worker:
   searcher-0:
-  sourcegraph-frontend-0:
-  sourcegraph-frontend-internal-0:
   zoekt-0-shared:
 networks:
   sourcegraph:

docker-compose/examples/aws-alb/docker-compose.override.yaml

@@ -1,5 +1,7 @@
 # Docker Compose override snippet for customers using AWS ALBs to terminate TLS,
-# and do not need TLS between the ALB and their Sourcegraph intance
+# and do not need TLS between the ALB and their Sourcegraph instance
+# Apply with `docker compose up -d --remove-orphans`, to remove the running caddy container,
+# which otherwise holds port 80
 
 
 ## Overrides for services defined in the base docker-compose.yaml files

docker-compose/examples/aws-rds-databases/docker-compose.override.yaml

@@ -3,19 +3,15 @@
 
 
 
-## Reusable blocks of environment variables
+## Reusable block of environment variables
 
-# For all containers, which connect to any of the databases
-x-database-env-var-rds-iam-auth: &database-env-var-rds-iam-auth
+# For migrator and both frontends, the only services which read these
+# The other services get their database connection strings from frontend
+x-database-env-vars: &database-env-vars
 
   # Using AWS RDS IAM authentication, not a password, so instruct the psql library to refresh the token
   PG_CONNECTION_UPDATER: EC2_ROLE_CREDENTIALS
 
-# For both migrator and frontend
-x-database-env-vars: &database-env-vars
-
-  <<: *database-env-var-rds-iam-auth
-
   # NOTE: Must clear / override the PGDATASOURCE env vars, otherwise frontends don't start up
   PGHOST: example-database-pgsql.cluster-example.us-west-2.rds.amazonaws.com
   PGPORT: 5432
@@ -47,11 +43,6 @@ x-database-env-vars: &database-env-vars
 services:
 
 
-  gitserver-0:
-    # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
-    environment:
-      <<: *database-env-var-rds-iam-auth
-
 
   migrator:
 
@@ -86,18 +77,6 @@ services:
     # If migrator fails to start up due to drift detected, or other issues, please open a support ticket by emailing support@sourcegraph.com
 
 
-  precise-code-intel-worker:
-    # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
-    environment:
-      <<: *database-env-var-rds-iam-auth
-
-
-  searcher-0:
-    # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
-    environment:
-      <<: *database-env-var-rds-iam-auth
-
-
   sourcegraph-frontend-0:
     # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
     environment:
@@ -115,25 +94,10 @@ services:
       <<: *database-env-vars
 
 
-  syntactic-code-intel-worker:
-    # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
-    environment:
-      <<: *database-env-var-rds-iam-auth
-
-
-  worker:
-    # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact
-    environment:
-      <<: *database-env-var-rds-iam-auth
-      # Add additional env vars, as needed
-      # TELEMETRY_GATEWAY_EXPORTER_USERMETADATA_EXPORT_ENABLED: true
-      # TELEMETRY_GATEWAY_EXPORTER_USERMETADATA_EXPORT_ORGANIZATIONS: true
-      # TELEMETRY_GATEWAY_EXPORTER_USERMETADATA_EXPORT_USERS: true
-
-
 ## Disable services
 
   # Drop built-in database services and volumes from the base compose file
+  # Prometheus still lists the three removed exporters as targets, and shows them as down; this is expected
   pgsql: !reset {}
   pgsql-exporter: !reset {}
   codeintel-db: !reset {}

docker-compose/examples/external-databases/docker-compose.override.yaml

@@ -26,9 +26,7 @@ x-env-vars-database: &env-vars-database
   PGSSLMODE: require
   PGUSER: sg
 
-# Uncomment if using AWS RDS with IAM authentication
-# PG_CONNECTION_UPDATER: EC2_ROLE_CREDENTIALS
-# AWS_STS_REGIONAL_ENDPOINTS: regional
+# For AWS RDS with IAM authentication, use ../aws-rds-databases/docker-compose.override.yaml instead
 
 
 services:
@@ -51,6 +49,7 @@ services:
 
 
   # Disable built-in databases
+  # Prometheus still lists the three removed exporters as targets, and shows them as down; this is expected
   codeinsights-db-exporter: !reset {}
   codeinsights-db: !reset {}
   codeintel-db-exporter: !reset {}

docker-compose/examples/migrator-operations/docker-compose.override.yaml

@@ -22,8 +22,8 @@ services:
     # https://github.com/sourcegraph/artifacts/blob/main/migrations/codeinsights/1675347548_squashed_migrations_privileged/up.sql
 
     # After the extensions are successfully installed, manually mark these migrations as completed,
-    # by uncommenting these commands, one at a time, and running `docker compose up migrator` each time,
-    # then comment the last one out, and run the normal `docker compose up -d --remove-orphans` command, for a normal startup
+    # by replacing the `command: ["up"]` line above with each of these commands, one at a time, and running `docker compose up migrator` each time,
+    # then restore the `command: ["up"]` line, and run the normal `docker compose up -d --remove-orphans` command, for a normal startup
     # command: ["add-log", "-db=frontend", "-version=1680296731"]
     # command: ["add-log", "-db=codeintel", "-version=1679010276"]
     # command: ["add-log", "-db=codeinsights", "-version=1675347548"]
@@ -33,6 +33,7 @@ services:
     ################################################################################
     # Multi-version upgrades
     # See https://sourcegraph.com/docs/self-hosted/deploy/docker-compose/upgrade#multi-version-upgrades
+    # Replace the `command: ["up"]` line above with this command
     # command: ["upgrade", "--from=v6.4.0", "--to=v6.9.1277"]
 
     # Do not skip the drift check, or use other args, without instruction from Sourcegraph Customer Support Engineering

docker-compose/examples/outbound-http-proxy/docker-compose.override.yaml

@@ -56,6 +56,7 @@ services:
       <<: *proxy-env-vars
 
   # Uncomment if connections to code hosts outside your private network need to use the proxy
+  # Git ignores the uppercase HTTP_PROXY for http:// code host URLs, so also set lowercase http_proxy for those
   # gitserver-0:
   #   environment:
   #     <<: *proxy-env-vars