feat: support redis sentinel

Makefile

@@ -20,6 +20,9 @@ format:
 build:
 	go build
 
+build-linux:
+	GOOS=linux GOARCH=amd64 go build
+
 test: build
 	go test -race $(pkgs)
 

clients/redis.go

@@ -10,11 +10,14 @@ import (
 
 func NewRedisClient(cfg config.RedisCacheConfig) (redis.UniversalClient, error) {
 	options := &redis.UniversalOptions{
-		Addrs:      cfg.Addresses,
-		Username:   cfg.Username,
-		Password:   cfg.Password,
-		PoolSize:   cfg.PoolSize,
-		MaxRetries: 7, // default value = 3, since MinRetryBackoff = 8 msec & MinRetryBackoff = 512 msec
+		Addrs:            cfg.Addresses, // single for standalone | list for redis cluster | sentinel addresses
+		Username:         cfg.Username,
+		Password:         cfg.Password,
+		PoolSize:         cfg.PoolSize,
+		SentinelUsername: cfg.SentinelUsername, // for sentinel only, not required
+		SentinelPassword: cfg.SentinelPassword, // for sentinel only, not required
+		MasterName:       cfg.MasterName,       // for sentinel only, required
+		MaxRetries:       7,                    // default value = 3, since MinRetryBackoff = 8 msec & MinRetryBackoff = 512 msec
 		// the redis client will wait up to 1016 msec btw the 7 tries
 	}
 

config/config.go

@@ -121,6 +121,9 @@ func withoutSensitiveInfo(config *Config) *Config {
 		if len(c.Caches[i].Redis.Password) > 0 {
 			c.Caches[i].Redis.Password = pswPlaceHolder
 		}
+		if len(c.Caches[i].Redis.SentinelPassword) > 0 {
+			c.Caches[i].Redis.SentinelPassword = pswPlaceHolder
+		}
 	}
 	return c
 }
@@ -966,12 +969,15 @@ type RedisCacheConfig struct {
 	TLS       `yaml:",inline"`
 	EnableTLS bool `yaml:"enable_tls,omitempty"`
 
-	Username  string                 `yaml:"username,omitempty"`
-	Password  string                 `yaml:"password,omitempty"`
-	Addresses []string               `yaml:"addresses"`
-	DBIndex   int                    `yaml:"db_index,omitempty"`
-	PoolSize  int                    `yaml:"pool_size,omitempty"`
-	XXX       map[string]interface{} `yaml:",inline"`
+	Username         string                 `yaml:"username,omitempty"`
+	Password         string                 `yaml:"password,omitempty"`
+	Addresses        []string               `yaml:"addresses"`
+	DBIndex          int                    `yaml:"db_index,omitempty"`
+	PoolSize         int                    `yaml:"pool_size,omitempty"`
+	SentinelUsername string                 `yaml:"sentinel_username,omitempty"`
+	SentinelPassword string                 `yaml:"sentinel_password,omitempty"`
+	MasterName       string                 `yaml:"master_name,omitempty"`
+	XXX              map[string]interface{} `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.

config/config_test.go

@@ -50,10 +50,11 @@ var fullConfig = Config{
 			MaxPayloadSize:     ByteSize(100 << 30),
 			SharedWithAllUsers: true,
 			Redis: RedisCacheConfig{
-				Username:  "chproxy",
-				Password:  "password",
-				Addresses: []string{"127.0.0.1:" + redisPort},
-				PoolSize:  10,
+				Username:         "chproxy",
+				Password:         "password",
+				SentinelPassword: "password",
+				Addresses:        []string{"127.0.0.1:" + redisPort},
+				PoolSize:         10,
 			},
 		},
 	},
@@ -560,6 +561,10 @@ func TestExamples(t *testing.T) {
 			"combined",
 			"examples/combined.yml",
 		},
+		{
+			"redis-sentinel",
+			"examples/redis-sentinel.yml",
+		},
 	}
 
 	for _, tc := range testCases {
@@ -758,6 +763,7 @@ func TestRemovalSensitiveData(t *testing.T) {
 	conf.Clusters[1].HeartBeat.Password = "XXX"
 	conf.Clusters[2].ClusterUsers[0].Password = "XXX"
 	conf.Caches[2].Redis.Password = "XXX"
+	conf.Caches[2].Redis.SentinelPassword = "XXX"
 
 	if !cmp.Equal(conf, confSafe, cmpopts.IgnoreUnexported(Config{})) {
 		t.Fatalf("confCopy should have sensitive data replaced with XXX values,\n the diff is: %s",
@@ -926,6 +932,7 @@ caches:
     addresses:
     - 127.0.0.1:%s
     pool_size: 10
+    sentinel_password: XXX
   max_payload_size: 107374182400
   shared_with_all_users: true
 param_groups:

config/examples/redis-sentinel.yml

@@ -0,0 +1,30 @@
+log_debug: true
+server:
+  http:
+    listen_addr: ":9090"
+    allowed_networks: ["127.0.0.1/24"]
+
+users:
+  - name: "default"
+    to_cluster: "default"
+    to_user: "default"
+    cache: "redis-cache"
+
+clusters:
+  - name: "default"
+    nodes: ["127.0.0.1:8123"]
+
+caches:
+  - name: "redis-cache"
+    mode: "redis"
+    redis:
+      addresses:
+        - "localhost:26379"
+        - "localhost:26380"
+      max_size: "10M"
+      username: redisuser
+      password: RedisPassword
+      sentinel_user: ""
+      sentinel_password: "SentinelPassword"
+      master_name: "mymaster"
+    expire: "1m"

config/testdata/full.yml

@@ -60,6 +60,7 @@ caches:
     redis:
       username: chproxy
       password: password
+      sentinel_password: password
       pool_size: 10
       addresses:
         - 127.0.0.1:16379

examples/quick_start/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   zookeeper:
     image: zookeeper:3.7
@@ -14,7 +13,7 @@ services:
       - ${PWD}/resources/clickhouse/config:/etc/clickhouse-server
       - ${PWD}/resources/clickhouse/data:/data
     ports:
-      - 8123:8123
+      - "8123:8123"
     networks:
       clickhouse-network:
         ipv4_address: 172.23.0.11
@@ -61,12 +60,12 @@ services:
     depends_on:
       - zookeeper
   chproxy:
-    image: contentsquareplatform/chproxy:v1.24.0
+    image: contentsquareplatform/chproxy:v1.31.0
     volumes:
       - ${PWD}/resources/chproxy/config:/config
       - ${PWD}/resources/chproxy/data:/data
     ports:
-      - 9001:9001
+      - "9001:9001"
     networks:
       clickhouse-network:
         ipv4_address: 172.23.0.14

examples/quick_start/resources/chproxy/config/config_redis_sentinel.yml

@@ -0,0 +1,40 @@
+log_debug: false
+hack_me_please: true
+
+server:
+  http:
+    listen_addr: "0.0.0.0:9001"
+
+users:
+  - name: "default"
+    password: "password"
+    to_cluster: "default"
+    to_user: "admin"
+    max_concurrent_queries: 1000
+    max_execution_time: 10m
+    cache: "default_cache"
+
+clusters:
+  - name: "default"
+    nodes: ["172.23.0.11:8123", "172.23.0.12:8123", "172.23.0.13:8123"]
+    users:
+      - name: "admin"
+        password: "123"
+
+caches:
+  - name: "default_cache"
+    mode: "redis"
+    redis:
+      addresses: # only sentinel addresses here
+        - 172.23.0.54:26379
+        - 172.23.0.55:26379
+      username: redisuser
+      password: RedisPassword
+      sentinel_user: ""
+      sentinel_password: "SentinelPassword"
+      master_name: "mymaster"
+      pool_size: 10
+      db_index: 0
+    expire: 60s
+    max_payload_size: 1024000
+    shared_with_all_users: false

examples/redis/docker-compose.yml

@@ -0,0 +1,76 @@
+services:
+  redis-master:
+    image: redis:latest
+    container_name: redis-master
+    restart: unless-stopped
+    ports:
+      - "6379:6379"
+    volumes:
+      - ${PWD}/redis/resources/redis-master.conf:/usr/local/etc/redis/redis.conf
+    command: redis-server /usr/local/etc/redis/redis.conf
+    networks:
+      clickhouse-network:
+        ipv4_address: 172.23.0.51
+  redis-replica-1:
+    image: redis:latest
+    container_name: redis-replica-1
+    restart: unless-stopped
+    ports:
+      - "6380:6379"
+    volumes:
+      - ${PWD}/redis/resources/redis-replica.conf:/usr/local/etc/redis/redis.conf
+    command: redis-server /usr/local/etc/redis/redis.conf
+    depends_on:
+      - redis-master
+    networks:
+      clickhouse-network:
+        ipv4_address: 172.23.0.52
+  redis-replica-2:
+    image: redis:latest
+    container_name: redis-replica-2
+    restart: unless-stopped
+    ports:
+      - "6381:6379"
+    volumes:
+      - ${PWD}/redis/resources/redis-replica.conf:/usr/local/etc/redis/redis.conf
+    command: redis-server /usr/local/etc/redis/redis.conf
+    depends_on:
+      - redis-master
+    networks:
+      clickhouse-network:
+        ipv4_address: 172.23.0.53
+  sentinel-1:
+    image: redis:latest
+    container_name: sentinel-1
+    restart: unless-stopped
+    ports:
+      - "26379:26379"
+    volumes:
+      - ${PWD}/redis/resources/sentinel1.conf:/usr/local/etc/redis/sentinel.conf
+    command: redis-sentinel /usr/local/etc/redis/sentinel.conf
+    depends_on:
+      - redis-master
+      - redis-replica-1
+      - redis-replica-2
+    networks:
+      clickhouse-network:
+        ipv4_address: 172.23.0.54
+  sentinel-2:
+    image: redis:latest
+    container_name: sentinel-2
+    restart: unless-stopped
+    ports:
+      - "26380:26379"
+    volumes:
+      - ${PWD}/redis/resources/sentinel2.conf:/usr/local/etc/redis/sentinel.conf
+    command: redis-sentinel /usr/local/etc/redis/sentinel.conf
+    depends_on:
+      - redis-master
+      - redis-replica-1
+      - redis-replica-2
+    networks:
+      clickhouse-network:
+        ipv4_address: 172.23.0.55
+networks:
+  clickhouse-network:
+    external: true

examples/redis/redis/resources/redis-master.conf

@@ -0,0 +1,15 @@
+bind 0.0.0.0
+port 6379
+daemonize no
+pidfile /var/run/redis/redis-server.pid
+loglevel notice
+save 900 1
+save 300 10
+save 60 10000
+protected-mode no
+
+#security checks
+user redisuser on >RedisPassword ~* &* +@all
+masteruser redisuser
+masterauth RedisPassword
+user default off

examples/redis/redis/resources/redis-replica.conf

@@ -0,0 +1,16 @@
+bind 0.0.0.0
+port 6379
+daemonize no
+pidfile /var/run/redis/redis-server.pid
+loglevel notice
+save 900 1
+save 300 10
+save 60 10000
+protected-mode no
+replicaof 172.23.0.51 6379
+
+#security checks
+user redisuser on >RedisPassword ~* &* +@all
+masteruser redisuser
+masterauth RedisPassword
+user default off

examples/redis/redis/resources/sentinel1.conf

@@ -0,0 +1,18 @@
+port 26379
+daemonize no
+pidfile /var/run/redis-sentinel.pid
+loglevel notice
+
+sentinel monitor mymaster 172.23.0.51 6379 2
+sentinel down-after-milliseconds mymaster 5000
+sentinel failover-timeout mymaster 60000
+sentinel parallel-syncs mymaster 1
+sentinel auth-user mymaster redisuser
+sentinel auth-pass mymaster RedisPassword
+bind 0.0.0.0
+protected-mode no
+
+requirepass SentinelPassword
+
+sentinel announce-ip 172.23.0.54
+sentinel announce-port 26379

examples/redis/redis/resources/sentinel2.conf

@@ -0,0 +1,18 @@
+port 26379
+daemonize no
+pidfile /var/run/redis-sentinel.pid
+loglevel notice
+
+sentinel monitor mymaster 172.23.0.51 6379 2
+sentinel down-after-milliseconds mymaster 5000
+sentinel failover-timeout mymaster 60000
+sentinel parallel-syncs mymaster 1
+sentinel auth-user mymaster redisuser
+sentinel auth-pass mymaster RedisPassword
+bind 0.0.0.0
+protected-mode no
+
+requirepass SentinelPassword
+
+sentinel announce-ip 172.23.0.55
+sentinel announce-port 26379