what i want
my setup has a dynamic amount of LXC containers and therefore i need some dynamic ipv6 address allocation. the interface brNC-internet is a simple bridge which is mapped into the LXC based container.
i need a way to assign ipv6 addresses in LXC containers to the internet
interface of each. manually doing so works, as shown below but this should be automated using dhcpd6 (or something similar).
i'd love to try:
- stateful dhcp: where a client is reassigned the same address when rebooted within some time limit (days, weeks)
- stateless dhcp: a client gets just any address for the time it is up
note: i can't use radvd with SLAAC because my network prefix is /66 and radvd requires at least a /64 for this, see this.
note: i'd like to focus on ipv6 for the moment.
note: i'm using nixos linux and i did probably just configure something wrongly, have a firewall rule which breaks things or a general misunderstanding about ipv6 or even some LXC specific internal. in either case, please point out what i could be trying next.
my problem
i've setup radvd and dhcpd6 on the host (a VM) but while radvd is able to push a ipv6 default gateway and a prefix the client never seems to talk to the dhcpd6 server using dhcpcd from the client.
if i disable the dhcpcd client on the guest LXC instance i can assign a ipv6 address within the prefix and ping6 to google works:
[root@10:/]# ip a replace 2a01:4f8:221:3744:4000::4 dev internet
[root@10:/]# ping -6 -I internet 2a00:1450:4001:80b::2003
PING 2a00:1450:4001:80b::2003(2a00:1450:4001:80b::2003) from fe80::10af:ffff:fef4:318a internet: 56 data bytes
From fe80::2044:c6ff:fef3:cd5d%internet icmp_seq=1 Destination unreachable: Beyond scope of source address
64 bytes from 2a00:1450:4001:80b::2003: icmp_seq=2 ttl=55 time=5.36 ms
64 bytes from 2a00:1450:4001:80b::2003: icmp_seq=3 ttl=55 time=5.26 ms
64 bytes from 2a00:1450:4001:80b::2003: icmp_seq=4 ttl=55 time=5.27 ms
i've also tried to disable the firewalls on both host and LXC client but no change.
dhcpcd from lxc
dhcpcd -6 --config /nix/store/7n7ysqf92rlafihs9dm2gzsbh06cw64z-dhcpcd.conf
DUID 00:01:00:01:22:4f:af:36:ee:ae:40:b5:d7:d3
internet: IAID 98:0e:c7:c8
internet: soliciting an IPv6 router
internet: Router Advertisement from fe80::2044:c6ff:fef3:cd5d
forked to background, child pid 1238
tcpdump on the VM
tcpdump -i brNC-internet ip6
14:36:18.618554 IP6 fe80::dc5d:98ff:fe0e:c7c8 > ff02::2: ICMP6, router solicitation, length 16
14:36:18.618681 IP6 status.nixcloud.io > fe80::dc5d:98ff:fe0e:c7c8: ICMP6, router advertisement, length 112
14:36:20.578582 IP6 status.nixcloud.io > ff02::1: ICMP6, router advertisement, length 112
14:36:24.059514 IP6 status.nixcloud.io > fe80::dc5d:98ff:fe0e:c7c8: ICMP6, neighbor solicitation, who has fe80::dc5d:98ff:fe0e:c7c8, length 32
14:36:24.059598 IP6 fe80::dc5d:98ff:fe0e:c7c8 > status.nixcloud.io: ICMP6, neighbor advertisement, tgt is fe80::dc5d:98ff:fe0e:c7c8, length 24
dhcpd6 log on the vm
journalctl -u dhcpd6 -f
-- Logs begin at Tue 2018-02-13 02:31:51 CET. --
Mar 30 14:00:08 status.nixcloud.io dhcpd[17605]: Wrote 0 NA, 0 TA, 0 PD leases to lease file.
Mar 30 14:00:08 status.nixcloud.io dhcpd6[17605]: Bound to *:547
Mar 30 14:00:08 status.nixcloud.io dhcpd[17605]: Bound to *:547
Mar 30 14:00:08 status.nixcloud.io dhcpd[17605]: Listening on Socket/5/brNC-internet/2a01:4f8:221:3744:4000::/66
Mar 30 14:00:08 status.nixcloud.io dhcpd[17605]: Sending on Socket/5/brNC-internet/2a01:4f8:221:3744:4000::/66
Mar 30 14:00:08 status.nixcloud.io dhcpd6[17605]: Listening on Socket/5/brNC-internet/2a01:4f8:221:3744:4000::/66
Mar 30 14:00:08 status.nixcloud.io dhcpd6[17605]: Sending on Socket/5/brNC-internet/2a01:4f8:221:3744:4000::/66
Mar 30 14:00:08 status.nixcloud.io systemd[1]: dhcpd6.service: Can't open PID file /run/dhcpd6/dhcpd.pid (yet?) after start: No such file or directory
Mar 30 14:00:08 status.nixcloud.io dhcpd6[17615]: Server starting service.
Mar 30 14:00:08 status.nixcloud.io systemd[1]: Started DHCPv6 server.
my setup
host vm
ip -6 a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 2a01:4f8:221:3744::1:26/128 scope global
valid_lft forever preferred_lft forever
inet6 fe80::5054:ff:fefb:d8d0/64 scope link
valid_lft forever preferred_lft forever
3: enp0s2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 fe80::5054:ff:fe08:4db9/64 scope link
valid_lft forever preferred_lft forever
4: brNC-hostonly: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 fe80::5880:6aff:fe77:cd16/64 scope link
valid_lft forever preferred_lft forever
5: brNC-internet: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 2a01:4f8:221:3744:4000::2/128 scope global
valid_lft forever preferred_lft forever
inet6 fc00::261/128 scope global
valid_lft forever preferred_lft forever
inet6 fe80::2044:c6ff:fef3:cd5d/64 scope link
valid_lft forever preferred_lft forever
121: vethVIEF2K@if120: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 fe80::fc55:8fff:fe5d:a50a/64 scope link
valid_lft forever preferred_lft forever
123: vethPIKVFW@if122: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 fe80::fc4f:adff:fe61:180b/64 scope link
valid_lft forever preferred_lft forever
133: vethC387B7@if132: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 fe80::fc5c:16ff:fe64:444b/64 scope link
valid_lft forever preferred_lft forever
135: vethE0I0FG@if134: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 fe80::fcbf:d4ff:fe33:a9d0/64 scope link
valid_lft forever preferred_lft forever
ip -6 r
2a01:4f8:221:3744::1:26 dev enp0s3 proto kernel metric 256 pref medium
2a01:4f8:221:3744:4000::2 dev brNC-internet proto kernel metric 256 pref medium
2a01:4f8:221:3744:4000::/66 dev brNC-internet proto kernel metric 256 expires 9700sec pref medium
fc00::26 dev enp0s3 metric 1024 pref medium
fc00::261 dev brNC-internet proto kernel metric 256 pref medium
fe80::/64 dev enp0s3 proto kernel metric 256 pref medium
fe80::/64 dev brNC-hostonly proto kernel metric 256 pref medium
fe80::/64 dev brNC-internet proto kernel metric 256 pref medium
fe80::/64 dev enp0s2 proto kernel metric 256 pref medium
fe80::/64 dev vethVIEF2K proto kernel metric 256 pref medium
fe80::/64 dev vethPIKVFW proto kernel metric 256 pref medium
fe80::/64 dev vethE0I0FG proto kernel metric 256 pref medium
fe80::/64 dev vethC387B7 proto kernel metric 256 pref medium
default via fc00::26 dev enp0s3 metric 1024 pref medium
dhcpd configuration
cat /nix/store/5zvcjwvlj9n7cvrppkw1mxsxwhxwx3cm-dhcpd.conf
default-lease-time 600;
max-lease-time 7200;
authoritative;
ddns-update-style interim;
log-facility local1; # see dhcpd.nix
subnet6 2a01:4f8:221:3744:4000::/66 {
#range6 2a01:4f8:221:3744:4000::/66 temporary;
option dhcp6.name-servers 2a01:4f8:0:1::add:1010, 2a01:4f8:0:1::add:9999, 2a01:4f8:0:1::add:9898;
}
radvd configuration
cat /nix/store/89j6hg4qhnd9nyijf9p2dcr4f5ygjz6r-radvd.conf
interface brNC-internet {
AdvSendAdvert on;
MinRtrAdvInterval 3;
MaxRtrAdvInterval 10;
prefix 2a01:4f8:221:3744:4000::/66 {
AdvOnLink on;
AdvAutonomous off;
};
RDNSS 2a01:4f8:0:1::add:1010 2a01:4f8:0:1::add:9999 2a01:4f8:0:1::add:9898 { };
};
guest LXC
[root@10:/]# ip -6 a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
132: hostonly@if133: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 fe80::484f:50ff:fe0c:fa62/64 scope link
valid_lft forever preferred_lft forever
134: internet@if135: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
inet6 2a01:4f8:221:3744:4000::4/128 scope global
valid_lft forever preferred_lft forever
inet6 fe80::10af:ffff:fef4:318a/64 scope link
valid_lft forever preferred_lft forever
[root@10:/]# ip -6 r
2a01:4f8:221:3744:4000::4 dev internet proto kernel metric 256 pref medium
2a01:4f8:221:3744:4000::/66 dev internet proto kernel metric 256 expires 86395sec pref medium
fe80::/64 dev hostonly proto kernel metric 256 pref medium
fe80::/64 dev internet proto kernel metric 256 pref medium
default via fe80::2044:c6ff:fef3:cd5d dev internet proto ra metric 1024 expires 25sec hoplimit 64 pref medium
lxc config
lxc.uts.name = 10
# Fixme also support other architectures?
lxc.arch = x86_64
# Not needed, just makes spares a few cpu cycles as LXC doesn't have
# to detect the backend.
#lxc.rootfs.backend = dir
lxc.rootfs.path = /var/lib/lxc/10/rootfs
lxc.init.cmd = /init/container/init
#lxc.rootfs = /var/lib/lxc/10/rootfs
# Ensures correct functionality with user namespaces. Since mknod is not possible stuff like
# /dev/console, /dev/tty, /dev/urandom, etc. need to be bind mounted. Note the order
# of the file inclusion here is important.
lxc.include = /nix/store/3hz7xkd86pzrvr4z53fa079q61qar02x-lxc-2.1.1/share/lxc/config/common.conf
lxc.include = /nix/store/3hz7xkd86pzrvr4z53fa079q61qar02x-lxc-2.1.1/share/lxc/config/userns.conf
## Network
# see also https://wiki.archlinux.org/index.php/Linux_Containers
lxc.net.0.type = veth
lxc.net.0.name = hostonly
#lxc.net.0.ipv4.address = 10.101.0.63 (we assign this using nix, not from lxc)
lxc.net.0.flags = up
lxc.net.0.link = brNC-hostonly
lxc.net.1.type = veth
lxc.net.1.name = internet
lxc.net.1.flags = up
lxc.net.1.link = brNC-internet
# Specifiy {u,g}id mapping.
lxc.idmap = u 0 100000 65536
lxc.idmap = g 0 100000 65536
# FIXME apparmor support
# Nixos does not provide AppArmor support.
#lxc.aa_profile = unconfined
#lxc.aa_allow_incomplete = 1
lxc.apparmor.profile = unconfined
lxc.apparmor.allow_incomplete = 1
# Tweaks for systemd.
lxc.autodev = 1
# Additional mount entries.
lxc.mount.entry = /nix/store nix/store none defaults,bind.ro 0.0
lxc.mount.entry = /nix/var/nix/profiles/nixcloud-container-10 init none defaults,bind.ro 0 0
# Mount entries that lead to a cleaner boot experience.
lxc.mount.entry = /sys/kernel/debug sys/kernel/debug none bind,optional 0 0
lxc.mount.entry = /sys/kernel/security sys/kernel/security none bind,optional 0 0
lxc.mount.entry = /sys/fs/pstore sys/fs/pstore none bind,optional 0 0
lxc.mount.entry = mqueue dev/mqueue mqueue rw,relatime,create=dir,optional 0 0
# LXC autostart
lxc.start.auto = 0
lxc.rootfs.path = dir:/var/lib/lxc/10/rootfs