Support Influx V2 on Docker

.github/workflows/golangci-lint.yml

@@ -23,7 +23,7 @@ jobs:
         run: sudo apt update && sudo apt-get install -y libpcap-dev
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20
+        uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a
         with:
-          version: v2.10
+          version: v2.13
           only-new-issues: true

README.md

@@ -57,12 +57,17 @@ If you need to change dashboards or datasources, set `GF_AUTH_ANONYMOUS_ORG_ROLE
 #### Accessing InfluxDB
 
 You can access the InfluxDB CLI using `docker compose exec -it influxdb influx`. 
-For example, to export all receiver telemetry as a CSV, run:
+For example, to list buckets, run:
 ```shell
-$ docker compose exec influxdb influx \
-    -database "gsw" \
-    -format csv \
-    -execute "SELECT * FROM receiver"
+docker compose exec influxdb influx bucket list
+```
+
+To query the `gsw` bucket directly, run:
+```shell
+docker compose exec influxdb influx query \
+    --org gsw \
+    --token gsw-local-dev-token \
+    'from(bucket: "gsw") |> range(start: 0)'
 ```
 
 ### Attaching to the container
@@ -130,4 +135,4 @@ this will probably start with `ws://localhost:3000`, but this can be replaced wi
 The second part of the address should always be `/api/live/push/telemetry/`, unless you have a reason to change it.
 * `http_addr`: Address where data is pushed to Grafana Live over HTTP. Make sure the protocol is `http://` or `https://`.
 * `use_websocket`: Whether to use WebSocket for data streaming. If disabled, HTTP will be used. WebSocket is recommended for better performance.
-* `use_http`: Whether to use HTTP for data streaming. If `use_websocket` is also true, HTTP will be used as backup if the WebSocket fails.
+* `use_http`: Whether to use HTTP for data streaming. If `use_websocket` is also true, HTTP will be used as backup if the WebSocket fails.

cmd/Containerfile

@@ -43,8 +43,6 @@ USER appuser
 
 ENV GSW_LOGGER_OUTPUT_PATHS=stdout
 
-ENV GSW_DATABASE_HOST_NAME=influxdb
-ENV GSW_DATABASE_PORT_NUMBER=8089
 ENV GSW_LOGGING_CONFIG=data/config/logger.yaml
 ENV GSW_TELEMETRY_CONFIG=data/config/backplane.yaml
 

cmd/gsw_service.go

@@ -140,8 +140,16 @@ func dbInitialize(ctx context.Context, channelMap map[int]chan []byte, cfg resol
 }
 
 func resolveDBConfig(config *viper.Viper) (resolvedDBConfig, error) {
-	v2Map := config.GetStringMap("database_v2")
-	if len(v2Map) > 0 {
+	v2Configured := config.IsSet("database_v2") ||
+		config.IsSet("database_v2.url") ||
+		config.IsSet("database_v2.token") ||
+		config.IsSet("database_v2.org") ||
+		config.IsSet("database_v2.bucket") ||
+		config.IsSet("database_v2.batch_size") ||
+		config.IsSet("database_v2.flush_interval_ms") ||
+		config.IsSet("database_v2.precision")
+
+	if v2Configured {
 		precision, err := db.ParsePrecision(config.GetString("database_v2.precision"))
 		if err != nil {
 			return resolvedDBConfig{}, fmt.Errorf("invalid database_v2.precision: %w", err)

cmd/gsw_service_test.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/AarC10/GSW-V2/lib/db"
+	"github.com/spf13/viper"
+)
+
+func newTestConfig() *viper.Viper {
+	cfg := viper.New()
+	cfg.SetEnvPrefix("GSW")
+	cfg.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	cfg.AutomaticEnv()
+	return cfg
+}
+
+func TestResolveDBConfigFromEnvironmentUsesV2(t *testing.T) {
+	t.Setenv("GSW_DATABASE_V2_URL", "http://influxdb:8086")
+	t.Setenv("GSW_DATABASE_V2_TOKEN", "test-token")
+	t.Setenv("GSW_DATABASE_V2_ORG", "gsw")
+	t.Setenv("GSW_DATABASE_V2_BUCKET", "gsw")
+	t.Setenv("GSW_DATABASE_V2_BATCH_SIZE", "250")
+	t.Setenv("GSW_DATABASE_V2_FLUSH_INTERVAL_MS", "1500")
+	t.Setenv("GSW_DATABASE_V2_PRECISION", "ms")
+
+	cfg := newTestConfig()
+
+	got, err := resolveDBConfig(cfg)
+	if err != nil {
+		t.Fatalf("resolveDBConfig returned error: %v", err)
+	}
+	if got.v2 == nil {
+		t.Fatal("expected v2 config to be selected")
+	}
+	if got.v1 != nil {
+		t.Fatal("expected v1 config to remain unset")
+	}
+
+	want := db.InfluxDBV2Config{
+		URL:           "http://influxdb:8086",
+		Token:         "test-token",
+		Org:           "gsw",
+		Bucket:        "gsw",
+		BatchSize:     250,
+		FlushInterval: 1500,
+		Precision:     db.PrecisionMS,
+	}
+	if *got.v2 != want {
+		t.Fatalf("got %+v, want %+v", *got.v2, want)
+	}
+}
+
+func TestResolveDBConfigPrefersV2OverV1(t *testing.T) {
+	cfg := viper.New()
+	cfg.Set("database_host_name", "legacy")
+	cfg.Set("database_port_number", 8089)
+	cfg.Set("database_v2.url", "http://influxdb:8086")
+	cfg.Set("database_v2.token", "token")
+	cfg.Set("database_v2.org", "gsw")
+	cfg.Set("database_v2.bucket", "gsw")
+
+	got, err := resolveDBConfig(cfg)
+	if err != nil {
+		t.Fatalf("resolveDBConfig returned error: %v", err)
+	}
+	if got.v2 == nil {
+		t.Fatal("expected v2 config to be selected")
+	}
+	if got.v1 != nil {
+		t.Fatal("expected v1 config to remain unset when v2 is configured")
+	}
+}

compose-services.yaml

@@ -6,26 +6,40 @@ services:
     ports:
       - "127.0.0.1:3000:3000/tcp"
     depends_on:
-      influxdb:
-        condition: service_started
+      - influxdb
     environment:
       - GSW_GRAFANA_INFLUXDB_URL=http://influxdb:8086
-      - GSW_GRAFANA_INFLUXDB_USER=root
-      - GSW_GRAFANA_INFLUXDB_PASSWORD=root
+      - GSW_GRAFANA_INFLUXDB_DB_NAME=${GSW_INFLUXDB_BUCKET:-gsw}
+      - GSW_GRAFANA_INFLUXDB_TOKEN=${GSW_INFLUXDB_ADMIN_TOKEN:-gsw-local-dev-token}
       - GF_AUTH_ANONYMOUS_ENABLED=true
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
       - GF_AUTH_BASIC_ENABLED=false
     volumes:
       - grafana-data:/var/lib/grafana
   influxdb:
-    image: influxdb:1.11-alpine
+    build:
+      context: data/influxdb
+      dockerfile: Containerfile
     volumes:
-      - influxdb-data:/var/lib/influxdb
+      - influxdb-data:/var/lib/influxdb2
+      - influxdb-config:/etc/influxdb2
     environment:
-      - INFLUXDB_UDP_ENABLED=true
-      - INFLUXDB_UDP_DATABASE=gsw
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=${GSW_INFLUXDB_USERNAME:-gsw}
+      - DOCKER_INFLUXDB_INIT_PASSWORD=${GSW_INFLUXDB_PASSWORD:-gsw-password}
+      - DOCKER_INFLUXDB_INIT_ORG=${GSW_INFLUXDB_ORG:-gsw}
+      - DOCKER_INFLUXDB_INIT_BUCKET=${GSW_INFLUXDB_BUCKET:-gsw}
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${GSW_INFLUXDB_ADMIN_TOKEN:-gsw-local-dev-token}
+      - GSW_INFLUXDB_V1_DATABASE=${GSW_INFLUXDB_BUCKET:-gsw}
+      - GSW_INFLUXDB_V1_RETENTION_POLICY=autogen
     ports:
-      - "127.0.0.1:8089:8089/udp"
+      - "127.0.0.1:8086:8086/tcp"
+    healthcheck:
+      test: ["CMD", "influx", "ping", "--host", "http://localhost:8086"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
 
   mosquitto:
     build:
@@ -38,3 +52,4 @@ services:
 volumes:
   grafana-data:
   influxdb-data:
+  influxdb-config:

compose.yaml

@@ -8,11 +8,14 @@ services:
       dockerfile: cmd/Containerfile
     depends_on:
       influxdb:
-        condition: service_started
+        condition: service_healthy
       grafana:
         condition: service_started
     environment:
-      - GSW_DATABASE_HOST_NAME=influxdb
+      - GSW_DATABASE_V2_URL=http://influxdb:8086
+      - GSW_DATABASE_V2_TOKEN=${GSW_INFLUXDB_ADMIN_TOKEN:-gsw-local-dev-token}
+      - GSW_DATABASE_V2_ORG=${GSW_INFLUXDB_ORG:-gsw}
+      - GSW_DATABASE_V2_BUCKET=${GSW_INFLUXDB_BUCKET:-gsw}
       - GSW_TELEMETRY_CONFIG=data/config/backplane.yaml
     ports:
       - 11020:11020/udp

data/config/gsw_service.yaml.example

@@ -9,12 +9,12 @@ telemetry_config: data/config/backplane.yaml
 # If database_v2 is set, V2 will be used and V1 settings are ignored
 database_v2:
   url: http://localhost:8086
-  token: your-token-here
+  token: gsw-local-dev-token
   org: gsw
   bucket: gsw
   batch_size: 100          # points buffered before auto-flush
   flush_interval_ms: 1000  # max ms before flushing partial batch
   precision: ns            # ns | us | ms | s
 
 # Path to GSW service logging config
-logging_config: data/config/logger.yaml
+logging_config: data/config/logger.yaml

data/grafana/provisioning/datasources/gsw-influxdb-provider.yaml

@@ -5,8 +5,12 @@ datasources:
     uid: gsw-influxdb
     type: influxdb
     access: proxy
-
-    user: $GSW_GRAFANA_INFLUXDB_USER
+
     url: $GSW_GRAFANA_INFLUXDB_URL
+    jsonData:
+      dbName: $GSW_GRAFANA_INFLUXDB_DB_NAME
+      httpHeaderName1: Authorization
+      httpMode: POST
+      version: InfluxQL
     secureJsonData:
-      password: $GSW_GRAFANA_INFLUXDB_PASSWORD
+      httpHeaderValue1: Token $GSW_GRAFANA_INFLUXDB_TOKEN

data/influxdb/Containerfile

@@ -0,0 +1,4 @@
+FROM influxdb:2.9-alpine
+
+COPY ./init-v1-compat.sh /docker-entrypoint-initdb.d/init-v1-compat.sh
+RUN chmod +x /docker-entrypoint-initdb.d/init-v1-compat.sh

data/influxdb/init-v1-compat.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -eu
+
+host="${INFLUX_HOST:-http://localhost:8086}"
+org="${DOCKER_INFLUXDB_INIT_ORG:?DOCKER_INFLUXDB_INIT_ORG is required}"
+bucket="${DOCKER_INFLUXDB_INIT_BUCKET:?DOCKER_INFLUXDB_INIT_BUCKET is required}"
+token="${DOCKER_INFLUXDB_INIT_ADMIN_TOKEN:?DOCKER_INFLUXDB_INIT_ADMIN_TOKEN is required}"
+database="${GSW_INFLUXDB_V1_DATABASE:-$bucket}"
+retention_policy="${GSW_INFLUXDB_V1_RETENTION_POLICY:-autogen}"
+
+echo "Waiting for InfluxDB at ${host}"
+until influx ping --host "${host}" >/dev/null 2>&1; do
+	sleep 1
+done
+
+bucket_id="$(influx bucket list \
+	--host "${host}" \
+	--org "${org}" \
+	--token "${token}" \
+	--name "${bucket}" \
+	--hide-headers | awk 'NR == 1 { print $1 }')"
+
+if [ -z "${bucket_id}" ]; then
+	echo "Unable to resolve bucket ID for ${bucket}" >&2
+	exit 1
+fi
+
+if influx v1 dbrp list \
+	--host "${host}" \
+	--org "${org}" \
+	--token "${token}" \
+	--db "${database}" \
+	--hide-headers | grep -Eq "[[:space:]]${retention_policy}([[:space:]]|$)"; then
+	echo "DBRP mapping already exists for ${database}/${retention_policy}"
+	exit 0
+fi
+
+echo "Creating DBRP mapping for ${database}/${retention_policy}"
+influx v1 dbrp create \
+	--host "${host}" \
+	--org "${org}" \
+	--token "${token}" \
+	--bucket-id "${bucket_id}" \
+	--db "${database}" \
+	--rp "${retention_policy}" \
+	--default