I am running this redis cluster implementation in kubernetes:
https://github.com/sobotklp/kubernetes-redis-cluster
After some time running the master starts consuming more and more memory. It gets to the point where its consuming about 7gb of memory and then It crashes (my kubernetes node gets filled up). If I run INFO * it never gets above 550 mb. of memory used. I dont know where this memory is going or how can I solve the issue.
It's a fairly "intensive" setup where I'm doing writes at all times and I store over 3500 keys but I can't see how can it gets so memory hungry.
All the other nodes run as expected except sometimes the fifth node (Its a 6 pod cluster) reach above 2Gi.
Below I attach the terminal logs out of INFO *.
Thanks in advance for any help.
# Server
redis_version:3.2.1
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:ed627d97d5dc311e
redis_mode:cluster
os:Linux 4.4.52+ x86_64
arch_bits:64
multiplexing_api:epoll
gcc_version:4.9.2
process_id:11
run_id:54afedf002881bd3c6423024c8813d2b40b79222
tcp_port:6379
uptime_in_seconds:2255
uptime_in_days:0
hz:10
lru_clock:10456862
executable:/data/redis-server
config_file:/conf/redis.conf
# Clients
connected_clients:15
client_longest_output_list:0
client_biggest_input_buf:0
blocked_clients:0
# Memory
used_memory:364433096
used_memory_human:347.55M
used_memory_rss:372293632
used_memory_rss_human:355.05M
used_memory_peak:653513856
used_memory_peak_human:623.24M
total_system_memory:15775928320
total_system_memory_human:14.69G
used_memory_lua:37888
used_memory_lua_human:37.00K
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
mem_fragmentation_ratio:1.02
mem_allocator:jemalloc-4.0.3
# Persistence
loading:0
rdb_changes_since_last_save:15686
rdb_bgsave_in_progress:0
rdb_last_save_time:1503629050
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:11
rdb_current_bgsave_time_sec:-1
aof_enabled:1
aof_rewrite_in_progress:1
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:131
aof_current_rewrite_time_sec:35
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_current_size:1159186792
aof_base_size:483390310
aof_pending_rewrite:0
aof_buffer_length:31467834
aof_rewrite_buffer_length:209011083
aof_pending_bio_fsync:1
aof_delayed_fsync:493
# Stats
total_connections_received:776
total_commands_processed:1196590
instantaneous_ops_per_sec:1794
total_net_input_bytes:13285120301
total_net_output_bytes:95137655858
instantaneous_input_kbps:21978.41
instantaneous_output_kbps:181579.59
rejected_connections:0
sync_full:44
sync_partial_ok:32
sync_partial_err:25
expired_keys:0
evicted_keys:0
keyspace_hits:630275
keyspace_misses:8331
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:8392
migrate_cached_sockets:0
# Replication
role:master
connected_slaves:5
slave0:ip=10.20.3.247,port=6379,state=online,offset=13258199186,lag=0
slave1:ip=10.20.0.97,port=6379,state=online,offset=13226242872,lag=5
slave2:ip=10.20.1.107,port=6379,state=online,offset=13226242872,lag=5
slave3:ip=10.20.3.246,port=6379,state=online,offset=13251620139,lag=0
slave4:ip=10.20.2.195,port=6379,state=online,offset=13243822483,lag=0
master_repl_offset:13260123117
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:13259074542
repl_backlog_histlen:1048576
# CPU
used_cpu_sys:134.94
used_cpu_user:45.00
used_cpu_sys_children:7.29
used_cpu_user_children:9.47
# Commandstats
cmdstat_get:calls=638606,usec=1584575,usec_per_call=2.48
cmdstat_set:calls=547540,usec=3241427,usec_per_call=5.92
cmdstat_keys:calls=2,usec=1722,usec_per_call=861.00
cmdstat_ping:calls=759,usec=468,usec_per_call=0.62
cmdstat_psync:calls=76,usec=151663,usec_per_call=1995.57
cmdstat_replconf:calls=8902,usec=13637,usec_per_call=1.53
cmdstat_info:calls=2,usec=159,usec_per_call=79.50
cmdstat_cluster:calls=4,usec=1357,usec_per_call=339.25
cmdstat_client:calls=14,usec=23,usec_per_call=1.64
cmdstat_command:calls=685,usec=399604,usec_per_call=583.36
# Cluster
cluster_enabled:1
# Keyspace
db0:keys=2930,expires=0,avg_ttl=0
