kubernetes version: v1.24
Error Replication:
I runned a replicated mysql application as url: https://kubernetes.io/docs/tasks/run-application/run-replicated-stateful-application/,There is an error:
[root@k8smaster01 wordpress-sts]# kubectl get po -l app=mysql
NAME READY STATUS RESTARTS AGE
mysql-0 1/2 CrashLoopBackOff 9 (30s ago) 22m
mysql-1 1/2 CrashLoopBackOff 8 (4m51s ago) 22m
mysql-2 0/2 Init:CrashLoopBackOff 9 (33s ago) 21m
[root@k8smaster01 wordpress-sts]# kubectl logs mysql-0 -c xtrabackup
+ cd /var/lib/mysql
+ [[ -f xtrabackup_slave_info ]]
+ [[ -f xtrabackup_binlog_info ]]
+ [[ -f change_master_to.sql.in ]]
+ exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c 'xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root'
220729 11:17:06 version_check Connecting to MySQL server with DSN 'dbi:mysql:;mysql_read_default_group=xtrabackup;host=127.0.0.1;port=3306;mysql_socket=/var/run/mysqld/mysqld.sock' as 'root' (using password: NO).
220729 11:17:06 version_check Connected to MySQL server
220729 11:17:06 version_check Executing a version check against the server...
220729 11:17:06 version_check Done.
220729 11:17:06 Connecting to MySQL server host: 127.0.0.1, user: root, password: not set, port: 3306, socket: /var/run/mysqld/mysqld.sock
Using server version 5.7.39-log
xtrabackup version 2.4.4 based on MySQL server 5.7.13 Linux (x86_64) (revision id: df58cf2)
xtrabackup: uses posix_fadvise().
xtrabackup: cd to /var/lib/mysql
xtrabackup: open files limit requested 0, set to 1048576
xtrabackup: using the following InnoDB configuration:
xtrabackup: innodb_data_home_dir = .
xtrabackup: innodb_data_file_path = ibdata1:12M:autoextend
xtrabackup: innodb_log_group_home_dir = ./
xtrabackup: innodb_log_files_in_group = 2
xtrabackup: innodb_log_file_size = 50331648
InnoDB: Number of pools: 1
220729 11:17:06 >> log scanned up to (12140831)
xtrabackup: Generating a list of tablespaces
InnoDB: Allocated tablespace ID 2 for mysql/plugin, old maximum was 0
220729 11:17:06 [01] Streaming ./ibdata1
xtrabackup: Error writing file 'UNOPENED' (Errcode: 32 - Broken pipe)
xb_stream_write_data() failed.
xtrabackup: Error writing file 'UNOPENED' (Errcode: 32 - Broken pipe)
[01] xtrabackup: Error: xtrabackup_copy_datafile() failed.
[01] xtrabackup: Error: failed to copy datafile.
Ncat: assertion failed: count <= INT_MAX QUITTING.
[root@k8smaster01 wordpress-sts]# kubectl logs mysql-1 -c mysql
2022-07-29 11:15:35+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 5.7.39-1.el7 started.
2022-07-29 11:15:36+00:00 [Note] [Entrypoint]: Switching to dedicated user 'mysql'
2022-07-29 11:15:36+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 5.7.39-1.el7 started.
'/var/lib/mysql/mysql.sock' -> '/var/run/mysqld/mysqld.sock'
2022-07-29T11:15:36.416759Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
2022-07-29T11:15:36.419290Z 0 [Note] mysqld (mysqld 5.7.39) starting as process 1 ...
2022-07-29T11:15:36.423857Z 0 [Note] InnoDB: PUNCH HOLE support available
2022-07-29T11:15:36.423900Z 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins
2022-07-29T11:15:36.423905Z 0 [Note] InnoDB: Uses event mutexes
2022-07-29T11:15:36.423911Z 0 [Note] InnoDB: GCC builtin __atomic_thread_fence() is used for memory barrier
2022-07-29T11:15:36.423918Z 0 [Note] InnoDB: Compressed tables use zlib 1.2.12
2022-07-29T11:15:36.423924Z 0 [Note] InnoDB: Using Linux native AIO
2022-07-29T11:15:36.424579Z 0 [Note] InnoDB: Number of pools: 1
2022-07-29T11:15:36.424801Z 0 [Note] InnoDB: Using CPU crc32 instructions
2022-07-29T11:15:36.427994Z 0 [Note] InnoDB: Initializing buffer pool, total size = 128M, instances = 1, chunk size = 128M
2022-07-29T11:15:36.441191Z 0 [Note] InnoDB: Completed initialization of buffer pool
2022-07-29T11:15:36.445172Z 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().
2022-07-29T11:15:36.458609Z 0 [Note] InnoDB: Highest supported file format is Barracuda.
2022-07-29T11:15:36.472269Z 0 [Note] InnoDB: Removed temporary tablespace data file: "ibtmp1"
2022-07-29T11:15:36.472305Z 0 [Note] InnoDB: Creating shared tablespace for temporary tables
2022-07-29T11:15:36.472476Z 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...
2022-07-29T11:15:36.543258Z 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.
2022-07-29T11:15:36.544497Z 0 [Note] InnoDB: 96 redo rollback segment(s) found. 96 redo rollback segment(s) are active.
2022-07-29T11:15:36.544534Z 0 [Note] InnoDB: 32 non-redo rollback segment(s) are active.
2022-07-29T11:15:36.545123Z 0 [Note] InnoDB: Waiting for purge to start
2022-07-29T11:15:36.595475Z 0 [Note] InnoDB: 5.7.39 started; log sequence number 12141096
2022-07-29T11:15:36.595917Z 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool
2022-07-29T11:15:36.596200Z 0 [Note] Plugin 'FEDERATED' is disabled.
2022-07-29T11:15:36.601414Z 0 [Note] InnoDB: Buffer pool(s) load completed at 220729 11:15:36
2022-07-29T11:15:36.603184Z 0 [Note] Salting uuid generator variables, current_pid: 1, server_start_time: 1659093336, bytes_sent: 0,
2022-07-29T11:15:36.603305Z 0 [Note] Generated uuid: 'c5478a88-0f2f-11ed-be07-3eb42d573df8', server_start_time: 281476635803993, bytes_sent: 46893056
2022-07-29T11:15:36.603322Z 0 [Warning] No existing UUID has been found, so we assume that this is the first time that this server has been started. Generating a new UUID: c5478a88-0f2f-11ed-be07-3eb42d573df8.
2022-07-29T11:15:37.384914Z 0 [Note] Auto generated SSL certificates are placed in data directory.
2022-07-29T11:15:37.384971Z 0 [Warning] A deprecated TLS version TLSv1 is enabled. Please use TLSv1.2 or higher.
2022-07-29T11:15:37.384976Z 0 [Warning] A deprecated TLS version TLSv1.1 is enabled. Please use TLSv1.2 or higher.
2022-07-29T11:15:37.385612Z 0 [Warning] CA certificate ca.pem is self signed.
2022-07-29T11:15:37.469457Z 0 [Note] Auto generated RSA key files are placed in data directory.
2022-07-29T11:15:37.470101Z 0 [Note] Server hostname (bind-address): '*'; port: 3306
2022-07-29T11:15:37.470199Z 0 [Note] IPv6 is available.
2022-07-29T11:15:37.470226Z 0 [Note] - '::' resolves to '::';
2022-07-29T11:15:37.470262Z 0 [Note] Server socket created on IP: '::'.
2022-07-29T11:15:37.472863Z 0 [Warning] Insecure configuration for --pid-file: Location '/var/run/mysqld' in the path is accessible to all OS users. Consider choosing a different directory.
2022-07-29T11:15:37.479344Z 0 [Note] Failed to start slave threads for channel ''
2022-07-29T11:15:37.488009Z 0 [Note] Event Scheduler: Loaded 0 events
2022-07-29T11:15:37.488235Z 0 [Note] mysqld: ready for connections.
Version: '5.7.39' socket: '/var/run/mysqld/mysqld.sock' port: 3306 MySQL Community Server (GPL)
2022-07-29T11:15:37.519790Z 2 [Note] Got packets out of order
2022-07-29T11:15:38.140458Z 4 [Warning] Neither --relay-log nor --relay-log-index were used; so replication may break when this MySQL server acts as a slave and has his hostname changed!! Please use '--relay-log=mysql-1-relay-bin' to avoid this problem.
2022-07-29T11:15:38.151140Z 4 [Note] 'CHANGE MASTER TO FOR CHANNEL '' executed'. Previous state master_host='', master_port= 3306, master_log_file='', master_log_pos= 4, master_bind=''. New state master_host='mysql-0.mysql', master_port= 3306, master_log_file='mysql-0-bin.000003', master_log_pos= 154, master_bind=''.
2022-07-29T11:15:38.161937Z 5 [Warning] Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
2022-07-29T11:15:38.162584Z 6 [Warning] Slave SQL for channel '': If a crash happens this configuration does not guarantee that the relay log info will be consistent, Error_code: 0
2022-07-29T11:15:38.162638Z 6 [Note] Slave SQL thread for channel '' initialized, starting replication in log 'mysql-0-bin.000003' at position 154, relay log './mysql-1-relay-bin.000001' position: 4
2022-07-29T11:15:38.165098Z 5 [Note] Slave I/O thread for channel '': connected to master 'root@mysql-0.mysql:3306',replication started in log 'mysql-0-bin.000003' at position 154
2022-07-29T11:15:39.655293Z 8 [Note] Aborted connection 8 to db: 'unconnected' user: 'root' host: '127.0.0.1' (Got an error reading communication packets)
2022-07-29T11:15:40.044342Z 9 [Note] Got packets out of order
2022-07-29T11:15:47.991617Z 15 [Note] Aborted connection 15 to db: 'unconnected' user: 'root' host: '127.0.0.1' (Got an error reading communication packets)
2022-07-29T11:15:51.611584Z 18 [Note] Got packets out of order
2022-07-29T11:15:52.519893Z 20 [Note] Got packets out of order
2022-07-29T11:15:55.044458Z 22 [Note] Got an error reading communication packets
2022-07-29T11:16:06.610807Z 30 [Note] Got packets out of order
2022-07-29T11:16:07.520340Z 31 [Note] Got packets out of order
2022-07-29T11:16:09.651807Z 34 [Note] Aborted connection 34 to db: 'unconnected' user: 'root' host: '127.0.0.1' (Got an error reading communication packets)
2022-07-29T11:16:10.044930Z 35 [Note] Got packets out of order
2022-07-29T11:16:21.611634Z 43 [Note] Got packets out of order
2022-07-29T11:16:22.520049Z 45 [Note] Got packets out of order
2022-07-29T11:16:25.044650Z 47 [Note] Got packets out of order
2022-07-29T11:16:36.611181Z 56 [Note] Got packets out of order
2022-07-29T11:16:37.519829Z 57 [Note] Got packets out of order
2022-07-29T11:16:40.044648Z 59 [Note] Got packets out of order
2022-07-29T11:16:46.094424Z 64 [Note] Aborted connection 64 to db: 'unconnected' user: 'root' host: '127.0.0.1' (Got an error reading communication packets)
2022-07-29T11:16:51.611271Z 69 [Note] Got an error reading communication packets
2022-07-29T11:16:52.520205Z 71 [Note] Got packets out of order
2022-07-29T11:16:55.044111Z 73 [Note] Got an error reading communication packets
2022-07-29T11:17:06.610868Z 82 [Note] Got packets out of order
2022-07-29T11:17:07.520287Z 83 [Note] Got packets out of order
2022-07-29T11:17:10.044176Z 85 [Note] Got an error reading communication packets
2022-07-29T11:17:21.610851Z 93 [Note] Got an error reading communication packets
2022-07-29T11:17:22.520416Z 95 [Note] Got packets out of order
2022-07-29T11:17:25.044223Z 97 [Note] Got an error reading communication packets
2022-07-29T11:17:36.611034Z 106 [Note] Got packets out of order
2022-07-29T11:17:37.520433Z 107 [Note] Got an error reading communication packets
2022-07-29T11:17:40.045078Z 109 [Note] Got packets out of order
2022-07-29T11:17:46.111895Z 114 [Note] Aborted connection 114 to db: 'unconnected' user: 'root' host: '127.0.0.1' (Got an error reading communication packets)
2022-07-29T11:17:51.611113Z 119 [Note] Got packets out of order
2022-07-29T11:17:52.519802Z 121 [Note] Got an error reading communication packets
2022-07-29T11:17:55.045084Z 123 [Note] Got an error reading communication packets
1 master and 2 slaves mysql cluster statefulset yaml file as followed:
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mysql
spec:
selector:
matchLabels:
app: mysql
app.kubernetes.io/name: mysql
serviceName: mysql
replicas: 3
template:
metadata:
labels:
app: mysql
app.kubernetes.io/name: mysql
spec:
initContainers:
- name: init-mysql
image: mysql:5.7
command:
- bash
- "-c"
- |
set -ex
# Generate mysql server-id from pod ordinal index.
[[ $HOSTNAME =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
echo [mysqld] > /mnt/conf.d/server-id.cnf
# Add an offset to avoid reserved server-id=0 value.
echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
# Copy appropriate conf.d files from config-map to emptyDir.
if [[ $ordinal -eq 0 ]]; then
cp /mnt/config-map/primary.cnf /mnt/conf.d/
else
cp /mnt/config-map/replica.cnf /mnt/conf.d/
fi
volumeMounts:
- name: conf
mountPath: /mnt/conf.d
- name: config-map
mountPath: /mnt/config-map
- name: clone-mysql
image: gcr.io/google-samples/xtrabackup:1.0
command:
- bash
- "-c"
- |
set -ex
[[ -d /var/lib/mysql/mysql ]] && exit 0
[[ $HOSTNAME =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
[[ $ordinal -eq 0 ]] && exit 0
ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
xtrabackup --prepare --target-dir=/var/lib/mysql
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
containers:
- name: mysql
image: mysql:5.7
env:
- name: MYSQL_ALLOW_EMPTY_PASSWORD
value: "1"
ports:
- name: mysql
containerPort: 3306
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
resources:
requests:
cpu: 500m
memory: 1Gi
livenessProbe:
exec:
command: ["mysqladmin", "ping"]
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
exec:
command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
initialDelaySeconds: 5
periodSeconds: 2
timeoutSeconds: 1
- name: xtrabackup
image: gcr.io/google-samples/xtrabackup:1.0
ports:
- name: xtrabackup
containerPort: 3307
command:
- bash
- "-c"
- |
set -ex
cd /var/lib/mysql
if [[ -f xtrabackup_slave_info && "x$(<xtrabackup_slave_info)" != "x" ]]; then
cat xtrabackup_slave_info | sed -E 's/;$//g' > change_master_to.sql.in
rm -f xtrabackup_slave_info xtrabackup_binlog_info
elif [[ -f xtrabackup_binlog_info ]]; then
[[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
rm -f xtrabackup_binlog_info xtrabackup_slave_info
echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
fi
if [[ -f change_master_to.sql.in ]]; then
echo "Waiting for mysqld to be ready (accepting connections)"
until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
echo "Initializing replication from clone position"
mysql -h 127.0.0.1 \
-e "$(<change_master_to.sql.in), \
MASTER_HOST='mysql-0.mysql', \
MASTER_USER='root', \
MASTER_PASSWORD='', \
MASTER_CONNECT_RETRY=10; \
START SLAVE;" || exit 1
mv change_master_to.sql.in change_master_to.sql.orig
fi
exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
"xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
resources:
requests:
cpu: 100m
memory: 100Mi
volumes:
- name: conf
emptyDir: {}
- name: config-map
configMap:
name: mysql
volumeClaimTemplates:
- metadata:
name: data
spec:
storageClassName: local-path
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 10Gi
I've configured configmap/service/pvc:
[root@k8smaster01 wordpress-sts]# kubectl get svc,configmap | grep mysql
service/mysql ClusterIP None <none> 3306/TCP 8h
service/mysql-read ClusterIP 10.101.192.107 <none> 3306/TCP 8h
service/wordpress-mysql ClusterIP None <none> 3306/TCP 43h
configmap/mysql 2 8h
[root@k8smaster01 wordpress-sts]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
data-mysql-0 Bound pvc-996d2551-dd0f-4960-8111-fc500bb9d828 10Gi RWO local-path 27m
data-mysql-1 Bound pvc-a0fafb00-2d24-4b8d-85d7-96f153ce6f5c 10Gi RWO local-path 26m
data-mysql-2 Bound pvc-80e4bf42-dbf6-4699-8eab-866b86d49f9c 10Gi RWO local-path 26m
