i have this docker compose file, that contains bitnami/mongodb
containers for creating a replica set. and I've used zookeeper
, kafka
& debezium-connector
for monitoring my mongodb replica set.
version: '2.2'
services:
mongodb-primary:
hostname: mongodb-primary
image: 'bitnami/mongodb:latest'
ports:
- 27017:27017
environment:
- MONGODB_REPLICA_SET_NAME=rs0
- MONGODB_REPLICA_SET_MODE=primary
- MONGODB_ROOT_PASSWORD=another_root
- MONGODB_DATABASE=genomics
- MONGODB_REPLICA_SET_KEY=replicasetkey123
- MONGODB_USERNAME=genomics_user
- MONGODB_PASSWORD=another
- MONGODB_ADVERTISED_HOSTNAME=mongodb-primary
volumes:
- 'mongodb_master_data:/bitnami'
mongodb-secondary:
hostname: mongodb-secondary
image: 'bitnami/mongodb:latest'
depends_on:
- mongodb-primary
environment:
- MONGODB_REPLICA_SET_NAME=rs0
- MONGODB_REPLICA_SET_MODE=secondary
- MONGODB_PRIMARY_HOST=mongodb-primary
- MONGODB_PRIMARY_PORT_NUMBER=27017
- MONGODB_PRIMARY_ROOT_PASSWORD=another_root
- MONGODB_REPLICA_SET_KEY=replicasetkey123
- MONGODB_USERNAME=genomics_user
- MONGODB_PASSWORD=another
- MONGODB_DATABASE=genomics
- MONGODB_ADVERTISED_HOSTNAME=mongodb-secondary
mongodb-arbiter:
hostname: mongodb-arbiter
image: 'bitnami/mongodb:latest'
depends_on:
- mongodb-primary
environment:
- MONGODB_REPLICA_SET_NAME=rs0
- MONGODB_REPLICA_SET_MODE=arbiter
- MONGODB_PRIMARY_HOST=mongodb-primary
- MONGODB_PRIMARY_PORT_NUMBER=27017
- MONGODB_PRIMARY_ROOT_PASSWORD=another_root
- MONGODB_REPLICA_SET_KEY=replicasetkey123
- MONGODB_USERNAME=genomics_user
- MONGODB_PASSWORD=another
- MONGODB_DATABASE=genomics
- MONGODB_ADVERTISED_HOSTNAME=mongodb-arbiter
debezium_connect:
image: debezium/connect:0.7
ports:
- 8083:8083
environment:
- BOOTSTRAP_SERVERS=kafka:9092
- GROUP_ID=1
- CONFIG_STORAGE_TOPIC=my_connect_configs
- OFFSET_STORAGE_TOPIC=my_connect_offsets
links:
- kafka
- mongodb-primary
kafka:
image: debezium/kafka:0.7
environment:
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ADVERTISED_PORT: 9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_CREATE_TOPICS: "topic-jhipster:1:1"
ports:
- 9092:9092
links:
- zookeeper
zookeeper:
image: debezium/zookeeper:0.7
ports:
- 2181:2181
- 2888:2888
- 3888:3888
consul:
image: consul:0.9.3
command: consul agent -dev -ui -client 0.0.0.0
ports:
- 8300:8300
- 8500:8500
- 8600:8600
consul-config-loader:
image: jhipster/consul-config-loader:v0.2.2
volumes:
- ./central-server-config:/config
environment:
- INIT_SLEEP_SECONDS=5
- CONSUL_URL=consul
- CONSUL_PORT=8500
# Uncomment to load configuration into Consul from a Git repository
# as configured in central-server-config/git2consul.json
# Also set SPRING_CLOUD_CONSUL_CONFIG_FORMAT=files on your apps
# - CONFIG_MODE=git
volumes:
mongodb_master_data:
driver: local
# networks:
# app-tier:
# driver: bridge
so i started this file by
docker-compose -f app.yaml up
and now debezium_connect
image is ready for accepting a mongodb connector properties:
{
"name": "run-connector",
"config": {
"connector.class" : "io.debezium.connector.mongodb.MongoDbConnector",
"tasks.max" : "1",
"mongodb.hosts" : "rs0/mongodb-primary:27017",
"mongodb.name" : "genomics",
"mongodb.user" : "genomics_user",
"mongodb.password" : "another",
"database.whitelist" : "inventory",
"database.history.kafka.bootstrap.servers" : "kafka:9092"
}
}
which is done by using this command:
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-mongodb.json
in which register-mongodb.json
content is displayed above.
but after curling this error comes out of debezium_connect:
2018-06-05 15:49:35,080 INFO MongoDB|genomics|disc No server chosen by ReadPreferenceServerSelector{readPreference=primary} from cluster description ClusterDescription{type=UNKNOWN, connectionMode=MULTIPLE, serverDescriptions=[ServerDescription{address=mongodb-primary:27017, type=UNKNOWN, state=CONNECTING, exception={com.mongodb.MongoSecurityException: Exception authenticating MongoCredential{mechanism=null, userName='genomics_user', source='admin', password=<hidden>, mechanismProperties={}}}, caused by {com.mongodb.MongoCommandException: Command failed with error 18: 'Authentication failed.' on server mongodb-primary:27017. The full response is { "ok" : 0.0, "errmsg" : "Authentication failed.", "code" : 18, "codeName" : "AuthenticationFailed", "operationTime" : { "$timestamp" : { "t" : 1528213769, "i" : 1 } }, "$clusterTime" : { "clusterTime" : { "$timestamp" : { "t" : 1528213769, "i" : 1 } }, "signature" : { "hash" : { "$binary" : "vSvTuw+hQCycX/rSliCcxWEh1BM=", "$type" : "00" }, "keyId" : { "$numberLong" : "6563606422322413569" } } } }}}]}. Waiting for 30000 ms before timing out [org.mongodb.driver.cluster]
2018-06-05 15:50:05,081 ERROR MongoDB|genomics|disc Error while reading the 'shards' collection in the 'config' database: Timed out after 30000 ms while waiting for a server that matches ReadPreferenceServerSelector{readPreference=primary}. Client view of cluster state is {type=UNKNOWN, servers=[{address=mongodb-primary:27017, type=UNKNOWN, state=CONNECTING, exception={com.mongodb.MongoSecurityException: Exception authenticating MongoCredential{mechanism=null, userName='genomics_user', source='admin', password=<hidden>, mechanismProperties={}}}, caused by {com.mongodb.MongoCommandException: Command failed with error 18: 'Authentication failed.' on server mongodb-primary:27017. The full response is { "ok" : 0.0, "errmsg" : "Authentication failed.", "code" : 18, "codeName" : "AuthenticationFailed", "operationTime" : { "$timestamp" : { "t" : 1528213799, "i" : 1 } }, "$clusterTime" : { "clusterTime" : { "$timestamp" : { "t" : 1528213799, "i" : 1 } }, "signature" : { "hash" : { "$binary" : "tk+Fd4ytnt/O8TMDyG43h79fnkk=", "$type" : "00" }, "keyId" : { "$numberLong" : "6563606422322413569" } } } }}}] [io.debezium.connector.mongodb.ReplicaSetDiscovery]
com.mongodb.MongoTimeoutException: Timed out after 30000 ms while waiting for a server that matches ReadPreferenceServerSelector{readPreference=primary}. Client view of cluster state is {type=UNKNOWN, servers=[{address=mongodb-primary:27017, type=UNKNOWN, state=CONNECTING, exception={com.mongodb.MongoSecurityException: Exception authenticating MongoCredential{mechanism=null, userName='genomics_user', source='admin', password=<hidden>, mechanismProperties={}}}, caused by {com.mongodb.MongoCommandException: Command failed with error 18: 'Authentication failed.' on server mongodb-primary:27017. The full response is { "ok" : 0.0, "errmsg" : "Authentication failed.", "code" : 18, "codeName" : "AuthenticationFailed", "operationTime" : { "$timestamp" : { "t" : 1528213799, "i" : 1 } }, "$clusterTime" : { "clusterTime" : { "$timestamp" : { "t" : 1528213799, "i" : 1 } }, "signature" : { "hash" : { "$binary" : "tk+Fd4ytnt/O8TMDyG43h79fnkk=", "$type" : "00" }, "keyId" : { "$numberLong" : "6563606422322413569" } } } }}}]
at com.mongodb.connection.BaseCluster.createTimeoutException(BaseCluster.java:377)
at com.mongodb.connection.BaseCluster.selectServer(BaseCluster.java:104)
at com.mongodb.binding.ClusterBinding$ClusterBindingConnectionSource.<init>(ClusterBinding.java:75)
at com.mongodb.binding.ClusterBinding$ClusterBindingConnectionSource.<init>(ClusterBinding.java:71)
at com.mongodb.binding.ClusterBinding.getReadConnectionSource(ClusterBinding.java:63)
at com.mongodb.operation.OperationHelper.withConnection(OperationHelper.java:402)
at com.mongodb.operation.ListDatabasesOperation.execute(ListDatabasesOperation.java:102)
at com.mongodb.operation.ListDatabasesOperation.execute(ListDatabasesOperation.java:54)
at com.mongodb.Mongo.execute(Mongo.java:836)
at com.mongodb.Mongo$2.execute(Mongo.java:823)
at com.mongodb.OperationIterable.iterator(OperationIterable.java:47)
at com.mongodb.ListDatabasesIterableImpl.iterator(ListDatabasesIterableImpl.java:57)
at com.mongodb.MappingIterable.iterator(MappingIterable.java:36)
at io.debezium.connector.mongodb.MongoUtil.contains(MongoUtil.java:181)
at io.debezium.connector.mongodb.MongoUtil.contains(MongoUtil.java:170)
at io.debezium.connector.mongodb.MongoUtil.onDatabase(MongoUtil.java:114)
at io.debezium.connector.mongodb.MongoUtil.onCollection(MongoUtil.java:129)
at io.debezium.connector.mongodb.MongoUtil.onCollectionDocuments(MongoUtil.java:148)
at io.debezium.connector.mongodb.ReplicaSetDiscovery.getReplicaSets(ReplicaSetDiscovery.java:67)
at io.debezium.connector.mongodb.ReplicaSetMonitorThread.run(ReplicaSetMonitorThread.java:63)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2018-06-05 15:50:05,082 INFO MongoDB|genomics|disc Cluster description not yet available. Waiting for 30000 ms before timing out [org.mongodb.driver.cluster]
is there any problem with my network? i think debezium_connect can't see mongodb-primary, so it can't authenticate..
you can delete existing "run-connector" by this command:
curl -X DELETE localhost:8083/connectors/run-connector/
i can authenticate into mongo-replica-set by other third party tools like robo3t
or another application which i wrote myself. but can't do it with debezium_connect
in docker network. also should note that the application i wrote myself has no problem connecting to mongodb in the docker network.