How to migrate a mongodb replica set from one kubernetes cluster to another using replication

Question

My goal is migrating data via replication from one mongodb replica set to another mongodb replicaset.

I am attempting to migrate data from replicaset a (on kubernetes cluster a) to replicaset b (on kubernetes cluster b)

See my attempt below:

bitnami/mongodb helm chart https://github.com/bitnami/charts/tree/master/bitnami/mongodb

global:
  namespaceOverride: mongodb
architecture: replicaset
auth:
  rootPassword: "<redacted>"
  replicaSetKey: "samevalue"
replicaCount: 1
serviceAccount:
  create: true
rbac:
  create: true
persistence:
  storageClass: "mongodb"
  size: 250Gi
externalAccess:
  enabled: true
  autoDiscovery:
    enabled: true

Steps

1. Create replicaset A on kubernetes cluster A
2. Create replicaset B on kubernetes cluster B
3. Add a member from replicaset B to replicaset A using the command

rs.add( { host: "<host>:27017", priority: 0, votes: 0 } )

member fails to join

rs0 [direct: primary] admin> rs.status()
{
  set: 'rs0',
  date: ISODate("2022-05-26T09:03:03.196Z"),
  myState: 1,
  term: Long("3"),
  syncSourceHost: '',
  syncSourceId: -1,
  heartbeatIntervalMillis: Long("2000"),
  majorityVoteCount: 2,
  writeMajorityCount: 1,
  votingMembersCount: 2,
  writableVotingMembersCount: 1,
  optimes: {
    lastCommittedOpTime: { ts: Timestamp({ t: 1653555782, i: 1 }), t: Long("3") },
    lastCommittedWallTime: ISODate("2022-05-26T09:03:02.147Z"),
    readConcernMajorityOpTime: { ts: Timestamp({ t: 1653555782, i: 1 }), t: Long("3") },
    appliedOpTime: { ts: Timestamp({ t: 1653555782, i: 1 }), t: Long("3") },
    durableOpTime: { ts: Timestamp({ t: 1653555782, i: 1 }), t: Long("3") },
    lastAppliedWallTime: ISODate("2022-05-26T09:03:02.147Z"),
    lastDurableWallTime: ISODate("2022-05-26T09:03:02.147Z")
  },
  lastStableRecoveryTimestamp: Timestamp({ t: 1653555762, i: 1 }),
  electionCandidateMetrics: {
    lastElectionReason: 'electionTimeout',
    lastElectionDate: ISODate("2022-05-26T08:59:02.135Z"),
    electionTerm: Long("3"),
    lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 1653555514, i: 1 }), t: Long("2") },
    lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1653555514, i: 1 }), t: Long("2") },
    numVotesNeeded: 2,
    priorityAtElection: 5,
    electionTimeoutMillis: Long("10000"),
    numCatchUpOps: Long("0"),
    newTermStartDate: ISODate("2022-05-26T08:59:02.141Z"),
    wMajorityWriteAvailabilityDate: ISODate("2022-05-26T08:59:02.142Z")
  },
  members: [
    {
      _id: 0,
      name: '<redacted_a>:27017',
      health: 1,
      state: 1,
      stateStr: 'PRIMARY',
      uptime: 167634,
      optime: { ts: Timestamp({ t: 1653555782, i: 1 }), t: Long("3") },
      optimeDate: ISODate("2022-05-26T09:03:02.000Z"),
      lastAppliedWallTime: ISODate("2022-05-26T09:03:02.147Z"),
      lastDurableWallTime: ISODate("2022-05-26T09:03:02.147Z"),
      syncSourceHost: '',
      syncSourceId: -1,
      infoMessage: '',
      electionTime: Timestamp({ t: 1653555542, i: 1 }),
      electionDate: ISODate("2022-05-26T08:59:02.000Z"),
      configVersion: 5,
      configTerm: 3,
      self: true,
      lastHeartbeatMessage: ''
    },
    {
      _id: 1,
      name: 'mongodb-prod-arbiter-0.mongodb-prod-arbiter-headless.mongodb.svc.cluster.local:27017',
      health: 1,
      state: 7,
      stateStr: 'ARBITER',
      uptime: 250,
      lastHeartbeat: ISODate("2022-05-26T09:03:01.611Z"),
      lastHeartbeatRecv: ISODate("2022-05-26T09:03:01.691Z"),
      pingMs: Long("0"),
      lastHeartbeatMessage: '',
      syncSourceHost: '',
      syncSourceId: -1,
      infoMessage: '',
      configVersion: 5,
      configTerm: 3
    },
    {
      _id: 2,
      name: '<redacted_b>',
      health: 0,
      state: 8,
      stateStr: '(not reachable/healthy)',
      uptime: 0,
      optime: { ts: Timestamp({ t: 0, i: 0 }), t: Long("-1") },
      optimeDurable: { ts: Timestamp({ t: 0, i: 0 }), t: Long("-1") },
      optimeDate: ISODate("1970-01-01T00:00:00.000Z"),
      optimeDurableDate: ISODate("1970-01-01T00:00:00.000Z"),
      lastAppliedWallTime: ISODate("1970-01-01T00:00:00.000Z"),
      lastDurableWallTime: ISODate("1970-01-01T00:00:00.000Z"),
      lastHeartbeat: ISODate("2022-05-26T09:03:01.719Z"),
      lastHeartbeatRecv: ISODate("1970-01-01T00:00:00.000Z"),
      pingMs: Long("0"),
      lastHeartbeatMessage: "replica set IDs do not match, ours: 628cb373d4eda0a12d1a75e0; remote node's: 628f3e9b4d975fb84fc5a5b1",
      syncSourceHost: '',
      syncSourceId: -1,
      infoMessage: '',
      configVersion: -1,
      configTerm: -1
    }
  ],
  ok: 1,
  '$clusterTime': {
    clusterTime: Timestamp({ t: 1653555782, i: 1 }),
    signature: {
      hash: Binary(Buffer.from("cf0cde3a5498d190d7258912ea99822796fda70f", "hex"), 0),
      keyId: Long("7101248018959040517")
    }
  },
  operationTime: Timestamp({ t: 1653555782, i: 1 })
}

{"t":{"$date":"2022-05-26T09:07:22.362+00:00"},"s":"I", "c":"REPL_HB", "id":23974, "ctx":"ReplCoord-4","msg":"Heartbeat failed after max retries","attr":{"target":"<redacted_b>","maxHeartbeatRetries":2,"error":{"code":93,"codeName":"InvalidReplicaSetConfig","errmsg":"replica set IDs do not match, ours: 628cb373d4eda0a12d1a75e0; remote node's: 628f3e9b4d975fb84fc5a5b1"}}}