refactor: Clickhouse test setup and teardown

.gitignore

@@ -63,6 +63,7 @@ gha-creds-*.json
 
 !.template.env
 !.single_tenant_bq.env
+!.single_tenant_ch.env
 !.single_tenant_pg.env
 # rustler nif binaries
 /priv/native

.reach.baseline.json

@@ -781,10 +781,10 @@
     {
       "source": "smells",
       "kind": "redundant_computation",
-      "fingerprint": "sha256:def6b8334cb5e0de59e58e669a16f009f819b7a3f280e81d65b6cf75216d3646",
-      "message": "Logflare.Auth.list_valid_access_tokens called twice with same args (line 138 and 168)",
+      "fingerprint": "sha256:9a7130a7a6e263eec866c2d02c31a837cb2fc4f7ed89ceac5eda162e7d76570e",
+      "message": "Logflare.Auth.list_valid_access_tokens called twice with same args (line 187 and 217)",
       "file": "lib/logflare/single_tenant.ex",
-      "line": 168
+      "line": 217
     },
     {
       "source": "smells",

.single_tenant_ch.env

@@ -0,0 +1,19 @@
+PHX_HTTP_PORT=4000
+DB_DATABASE=logflare_dev
+DB_HOSTNAME=localhost
+DB_PORT=5432
+DB_PASSWORD=postgres
+DB_USERNAME=postgres
+DB_SCHEMA=public
+
+LOGFLARE_SINGLE_TENANT=true
+LOGFLARE_PRIVATE_ACCESS_TOKEN=my-cool-api-key-123
+LOGFLARE_PUBLIC_ACCESS_TOKEN=my-other-cool-api-key-123
+LOGFLARE_GRPC_PORT=50051
+
+CLICKHOUSE_BACKEND_URL=http://logflare:logflare@localhost:8123/logflare_test
+CLICKHOUSE_BACKEND_POOL_SIZE=3
+
+LOGFLARE_NODE_HOST=127.0.0.1
+LOGFLARE_HTTP_CONNECTION_POOLS=all
+LOGFLARE_FEATURE_FLAG_OVERRIDE=multibackend=true,key_values=true

DEVELOPMENT.md

@@ -53,7 +53,7 @@ docker-compose up -d db clickhouse telegraf
 # install dependencies
 make setup
 
-# start in single tenant postgres backend (logflare_dev)
+# start in single tenant postgres backend (logflare_dev). See [Developing for Single Tenant](#developing-for-single-tenant) for other backends.
 make start.st.pg
 
 # run tests
@@ -73,17 +73,50 @@ To configure the BigQuery backend, please follow the [BigQuery setup documentati
 
 ### Developing for Single Tenant
 
-Use the single tenant `make start.*` variations. This works by switching out the `LOGFLARE_SINGLE_TENANT` env var.
+Use a backend-specific single tenant target. Each target loads its corresponding
+`.single_tenant_*.env` file and enables `LOGFLARE_SINGLE_TENANT`.
 
 ```bash
 make start.st.pg
 make start.st.bq
+make start.st.ch
 ```
 
+#### ClickHouse
+
+Start the application PostgreSQL database and ClickHouse service, then start Logflare:
+
+```bash
+docker compose up -d db clickhouse telegraf
+make start.st.ch
+```
+
+`make start.st.ch` loads `.single_tenant_ch.env`. The main settings are:
+
+- `CLICKHOUSE_BACKEND_URL` - the ClickHouse HTTP connection URL, including the database name
+- `CLICKHOUSE_BACKEND_POOL_SIZE` - optional connection pool size
+- `LOGFLARE_CLICKHOUSE_TABLE_SUFFIX` - suffix for the single-tenant `otel_logs`, `otel_metrics`, and `otel_traces` tables (defaults to `default`)
+- `LOGFLARE_PUBLIC_ACCESS_TOKEN` and `LOGFLARE_PRIVATE_ACCESS_TOKEN` - local API tokens
+
+PostgreSQL is still required for Logflare application data. ClickHouse becomes the default
+log backend and uses one shared ingestion pipeline configured from `CLICKHOUSE_BACKEND_URL`; the
+default ClickHouse backend is not stored as a database record.
+
+The local URL is typically:
+
+```text
+http://logflare:logflare@localhost:8123/logflare_test
+```
+
+If both `POSTGRES_BACKEND_URL` and `CLICKHOUSE_BACKEND_URL` are present in the effective
+environment, PostgreSQL takes precedence. Unset `POSTGRES_BACKEND_URL` when using
+`make start.st.ch`.
+
 To develop with Supabase mode:
 
 ```bash
 make start.sb.bq
+make start.sb.ch
 make start.sb.pg
 ```
 
@@ -164,7 +197,8 @@ bin/compose down -v
 make setup
 make start
 make start.{orange|pink}
-make start.{st|sb}.{bq|pg}
+make start.st.{bq|ch|pg}
+make start.sb.{bq|ch|pg}
 make decrypt.{dev|staging|prod}
 make encrypt.{dev|staging|prod}
 make reset

Makefile

@@ -120,7 +120,7 @@ setup: check-tools check-version-manager setup.node
 setup.node:
 	@echo -e "$(BOLD)$(BLUE)📦 Installing Node.js dependencies...$(NC)"
 	@echo ""
-	npm --prefix ./assets ci 
+	npm --prefix ./assets ci
 	@echo ""
 
 reset:
@@ -157,6 +157,9 @@ start.green: __start__
 start.sb.bq: LOGFLARE_SUPABASE_MODE = true
 start.sb.bq: start.st.bq
 
+start.sb.ch: LOGFLARE_SUPABASE_MODE = true
+start.sb.ch: start.st.ch
+
 # SPOOL_MODE (producer|consumer|both), SPOOL_PROVIDER (aws|gcp), and related
 # SPOOL_* vars pass straight through to the app via __start__ (see config/runtime.exs).
 # To run a producer + consumer pair side by side, override PORT/ERL_NAME/LOGFLARE_GRPC_PORT
@@ -169,6 +172,12 @@ start.st.bq: ENV_FILE = .single_tenant_bq.env
 start.st.bq: LOGFLARE_GRPC_PORT ?= 50051
 start.st.bq: __start__
 
+start.st.ch: ERL_NAME ?= st_
+start.st.ch: PORT ?= 4000
+start.st.ch: ENV_FILE = .single_tenant_ch.env
+start.st.ch: LOGFLARE_GRPC_PORT ?= 50051
+start.st.ch: __start__
+
 start.sb.pg: LOGFLARE_SUPABASE_MODE = true
 start.sb.pg: start.st.pg
 
@@ -195,7 +204,7 @@ migrate:
 stripe:
 	stripe listen --forward-to localhost:4000/webhooks/stripe
 
-.PHONY: __start__ migrate stripe start.sb.pg start.sb.bq start.st.pg start.st.bq start.orange start.pink
+.PHONY: __start__ migrate stripe start.sb.pg start.sb.ch start.sb.bq start.st.pg start.st.ch start.st.bq start.orange start.pink
 
 # Encryption and decryption of secrets
 # Usage:
@@ -463,12 +472,12 @@ $(TELEGRAF_KEYS_SENTINEL):
 
 	# Server (Telegraf)
 	@echo "subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:telegraf" > $(TELEGRAF_KEYS_DIR)/server.ext
-	
+
 	@openssl genrsa -out $(TELEGRAF_KEYS_DIR)/server.key 2048
 	@openssl req -new -key $(TELEGRAF_KEYS_DIR)/server.key \
 		-subj "/CN=localhost" \
 		-out $(TELEGRAF_KEYS_DIR)/server.csr
-	
+
 	@openssl x509 -req -days 3650 -in $(TELEGRAF_KEYS_DIR)/server.csr \
 		-CA $(TELEGRAF_KEYS_DIR)/ca.crt -CAkey $(TELEGRAF_KEYS_DIR)/ca.key -set_serial 01 \
 		-out $(TELEGRAF_KEYS_DIR)/server.crt \

config/runtime.exs

@@ -1,5 +1,6 @@
 import Config
 
+alias Logflare.SingleTenant
 alias Logflare.Utils
 
 defmodule Env do
@@ -135,6 +136,7 @@ config :logflare,
          recaptcha_secret: System.get_env("LOGFLARE_RECAPTCHA_SECRET"),
          config_cat_sdk_key: System.get_env("LOGFLARE_CONFIG_CAT_SDK_KEY"),
          single_tenant: Env.get_boolean("LOGFLARE_SINGLE_TENANT"),
+         clickhouse_table_suffix: System.get_env("LOGFLARE_CLICKHOUSE_TABLE_SUFFIX", "default"),
          supabase_mode: Env.get_boolean("LOGFLARE_SUPABASE_MODE"),
          unsafe_disable_ssrf_s3_endpoint_check:
            Env.get_boolean("LOGFLARE_UNSAFE_DISABLE_SSRF_S3_ENDPOINT_CHECK"),
@@ -390,15 +392,54 @@ socket_options_for_url = fn
     nil
 end
 
+single_tenant? = Env.get_boolean("LOGFLARE_SINGLE_TENANT")
+bigquery_service_account = System.get_env("GOOGLE_SERVICE_ACCOUNT")
+postgres_backend_url = System.get_env("POSTGRES_BACKEND_URL")
+clickhouse_backend_url = System.get_env("CLICKHOUSE_BACKEND_URL")
+
+backend_url_set? = fn url -> is_binary(url) and String.trim(url) != "" end
+
+if Enum.count(
+     [bigquery_service_account, postgres_backend_url, clickhouse_backend_url],
+     backend_url_set?
+   ) > 1 do
+  raise "Only one of GOOGLE_SERVICE_ACCOUNT, POSTGRES_BACKEND_URL, or CLICKHOUSE_BACKEND_URL may be configured"
+end
+
+single_tenant_backend =
+  cond do
+    single_tenant? and backend_url_set?.(postgres_backend_url) -> :postgres
+    single_tenant? and backend_url_set?.(clickhouse_backend_url) -> :clickhouse
+    true -> :bigquery
+  end
+
+config :logflare, :single_tenant_backend, single_tenant_backend
+
 cond do
-  Env.get_boolean("LOGFLARE_SINGLE_TENANT") &&
-      not is_nil(System.get_env("POSTGRES_BACKEND_URL")) ->
+  single_tenant? and single_tenant_backend == :postgres ->
     config :logflare,
            :postgres_backend_adapter,
            filter_nil_kv_pairs.(
-             url: System.get_env("POSTGRES_BACKEND_URL"),
+             url: postgres_backend_url,
              schema: System.get_env("POSTGRES_BACKEND_SCHEMA"),
-             socket_options: socket_options_for_url.(System.get_env("POSTGRES_BACKEND_URL", ""))
+             socket_options: socket_options_for_url.(postgres_backend_url)
+           )
+
+  single_tenant? and single_tenant_backend == :clickhouse ->
+    clickhouse_backend_config =
+      SingleTenant.clickhouse_backend_adapter_opts_from_url!(clickhouse_backend_url)
+
+    config :logflare,
+           :clickhouse_backend_adapter,
+           filter_nil_kv_pairs.(
+             clickhouse_backend_config ++
+               [
+                 pool_size:
+                   case System.get_env("CLICKHOUSE_BACKEND_POOL_SIZE") do
+                     nil -> nil
+                     value -> String.to_integer(value)
+                   end
+               ]
            )
 
   config_env() != :test ->

lib/logflare/application.ex

@@ -9,6 +9,7 @@ defmodule Logflare.Application do
   alias Logflare.Backends.Adaptor.BigQueryAdaptor
   alias Logflare.Backends.Spool.Health, as: SpoolHealth
   alias Logflare.Backends.Spool.SpoolAck
+  alias Logflare.Backends.SystemBackend
   alias Logflare.Backends.UserMonitoring
   alias Logflare.ContextCache
   alias Logflare.Logs
@@ -234,15 +235,13 @@ defmodule Logflare.Application do
   def startup_tasks do
     Logger.info("Executing startup tasks")
 
-    if !SingleTenant.postgres_backend?() do
-      BigQueryAdaptor.create_managed_service_accounts()
-      BigQueryAdaptor.update_iam_policy()
-    end
+    SystemBackend.on_system_start()
 
     if SingleTenant.single_tenant?() do
       Logger.info("Ensuring single tenant user is seeded...")
       SingleTenant.create_default_plan()
       SingleTenant.create_default_user()
+      SingleTenant.ensure_default_backend()
       SingleTenant.create_access_tokens()
     end
 
@@ -251,13 +250,7 @@ defmodule Logflare.Application do
       SingleTenant.create_supabase_endpoints()
       SingleTenant.ensure_supabase_sources_started()
 
-      unless SingleTenant.postgres_backend?() do
-        # buffer time for all sources to init and create tables
-        # in case of latency.
-        :timer.sleep(3_000)
-
-        SingleTenant.update_supabase_source_schemas()
-      end
+      SystemBackend.on_supabase_start()
     end
 
     # Schedule all alerts immediately on startup