I want to prepare basic setup for Redis Sentinel on docker. Everything looks fine, replication is working as expected, however, failover doesn’t work. In order to test it I created some simple python app and kill redis-master from docker compose. I tried to find some solutions and one possible thing is I’m facing https://redis.io/docs/management/sentinel/#sentinel-docker-nat-and-possible-issues. I tried to use container IP or different announce options but still not sure how to solve it.
I try to test failover by simply pausing the container docker-compose pause redis-master. Sentinel detects that master is down sdown, but didn’t elect a new master after that.
Here is my app tree:
.
├── app
│ ├── Dockerfile
│ └── redis_app.py
├── compose.yaml
└── sentinel
├── Dockerfile
├── sentinel.conf
└── sentinel-entrypoint.sh
compose.yml
version: '3.2'
services:
redis-master:
image: redis:7.0.5
command: redis-server
ports:
- 16379:6379
redis-slave-1:
image: redis:7.0.5
command: redis-server --slaveof redis-master 6379
ports:
- 26379:6379
links:
- redis-master
redis-slave-2:
image: redis:7.0.5
command: redis-server --slaveof redis-master 6379
ports:
- 36379:6379
links:
- redis-master
sentinel-1:
build: sentinel
environment:
- SENTINEL_DOWN_AFTER=5000
- SENTINEL_FAILOVER=500
- SENTINEL_QUORUM=2
depends_on:
- redis-master
- redis-slave-1
- redis-slave-2
links:
- redis-master
redis_cluster_app:
build: app
environment:
- SENTINEL_DOWN_AFTER=5000
- SENTINEL_FAILOVER=500
- SENTINEL_QUORUM=2
depends_on:
- redis-master
- redis-slave-1
- redis-slave-2
command: /bin/bash -c "echo 'Waiting for redis to run..' && sleep 60 && python redis_app.py"
volumes:
- ./app:/usr/src/app/redis_app/
sentinel/Dockerfile
FROM redis:7.0.5
EXPOSE 26379
ADD sentinel.conf /etc/redis/sentinel.conf
RUN chown redis:redis /etc/redis/sentinel.conf
COPY sentinel-entrypoint.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/sentinel-entrypoint.sh
ENTRYPOINT ["sentinel-entrypoint.sh"]
sentinel/sentinel.conf
port 26379
dir /tmp
sentinel resolve-hostnames yes
sentinel monitor mymaster redis-master 6379 $SENTINEL_QUORUM
sentinel down-after-milliseconds mymaster $SENTINEL_DOWN_AFTER
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster $SENTINEL_FAILOVER
sentinel announce-port 26379
sentinel/sentinel-entrypoint.sh
#!/bin/sh
sed -i "s/\$SENTINEL_QUORUM/$SENTINEL_QUORUM/g" /etc/redis/sentinel.conf
sed -i "s/\$SENTINEL_DOWN_AFTER/$SENTINEL_DOWN_AFTER/g" /etc/redis/sentinel.conf
sed -i "s/\$SENTINEL_FAILOVER/$SENTINEL_FAILOVER/g" /etc/redis/sentinel.conf
app/Dockerfile
# basic python image
FROM python:3.11
# install redis to access redis APIs
RUN pip install redis
# Without this setting, Python never prints anything out.
ENV PYTHONUNBUFFERED=1
# declare the source directory
WORKDIR /usr/src/app/
# copy the file
COPY redis_app.py .
# start command
CMD [ "python", "redis_app.py" ]
app/redis_app.py
import time
from redis import RedisError, sentinel, ReadOnlyError
import sys
ERROR_KEY_NOT_FOUND = "Key not found in redis"
class RedisDriver:
def __init__(self, redis_config):
self.service = redis_config["service_name"]
self.__connect(redis_config)
def __connect(self, redis_config):
self.connection = sentinel.Sentinel(
[
(redis_config["sentinel_host"], redis_config["sentinel_port"]),
],
socket_timeout=0.5,
)
# print(self.connection.master_for("mymaster"))
# print(self.connection.discover_slaves("mymaster"))
def set(self, key, value):
key_str = str(key)
val_str = str(value)
try:
master = self.connection.master_for(self.service)
master.set(key_str, val_str)
return {"success": True}
except RedisError as err:
error_str = "Error while connecting to redis : " + str(err)
return {"success": False, "error": error_str}
def get(self, key):
key_str = str(key)
try:
master = self.connection.master_for(self.service)
value = master.get(key_str)
except RedisError as err:
error_str = "Error while retrieving value from redis : " + str(err)
return {"success": False, "error": error_str}
if value is not None:
return {"success": True, "value": value}
else:
return {"success": False, "error": ERROR_KEY_NOT_FOUND}
def delete(self, key):
key_str = str(key)
try:
master = self.connection.master_for(self.service)
value = master.delete(key_str)
except RedisError as err:
error_str = "Error while deleting key from redis : " + str(err)
return {"success": False, "error": error_str}
return {"success": True}
if __name__ == "__main__":
print("*" * 75)
redis_config = {
"service_name": "mymaster",
"sentinel_host": "sentinel-1",
"sentinel_port": 26379,
}
redis_driver = RedisDriver(redis_config)
while True:
result = redis_driver.set("hello", "world")
print(result)
if result["success"]:
result = redis_driver.get("hello")
print(result)
slave = redis_driver.connection.slave_for(redis_driver.service)
try:
slave.set("slave", "slave")
except ReadOnlyError:
print("Slave is readonly")
print("******** SLEEPING *********")
time.sleep(10)
print("******** AGAIN *********")
result = redis_driver.set("hello2", "world2")
print(result)
if result["success"]:
result = redis_driver.get("hello2")
print(result)
redis_driver.delete("hello")
time.sleep(10)
When I run docker-compose up --build everything seems to be working fine, in the infinite loop I keep modifying keys, then in another terminal I run docker-compose pause redis-master but failover doesn’t work.
Some logs, after sdwon master sentinel didn’t select a new master