My server connects to the public internet via eth0 (50.131.xxx.xxx). The same server connects to the LAN via eth1 (192.168.138.1 on 192.168.138.0/24) On the same server, I run libvirt, which created another subnet at 192.168.122.0/24. The currently only guest is at 192.168.122.10.
I'm trying to get routing on the server set up, and I'm stuck.
From my laptop on the LAN (192.168.138.18), I can ssh into the guest, the server, and the public internet. However, I can only partially do the reverse: From my guest, I can ssh into the server but I cannot ssh into the laptop, or reach the public internet.
This is Ubuntu 12.04 LTS on the server, and 13.04 on the guest.
On the guest:
guest> traceroute serverfault.com
1 192.168.122.1 (192.168.122.1) 0.644 ms 0.577 ms 0.564 ms
2 * * *
(and so forth)
Here's some data that hopefully helps somebody more knowledgeable than me.
guest> ip r
default via 192.168.122.1 dev eth0
192.168.122.0/24 dev eth0 proto kernel scope link src 192.168.122.10
server> cat /proc/sys/net/ipv4/ip_forward
1
server> ifconfig
eth0 Link encap:Ethernet HWaddr 6c:f0:49:0e:09:b2
inet addr:50.131.xxx.xxx Bcast:255.255.255.255 Mask:255.255.252.0
UP BROADCAST RUNNING MULTICAST MTU:576 Metric:1
RX packets:670219 errors:0 dropped:0 overruns:0 frame:0
TX packets:532895 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:271653035 (271.6 MB) TX bytes:70403086 (70.4 MB)
Interrupt:41
eth1 Link encap:Ethernet HWaddr 00:c0:49:fa:1f:da
inet addr:192.168.138.1 Bcast:192.168.138.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:539044 errors:0 dropped:0 overruns:0 frame:0
TX packets:563204 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:72237497 (72.2 MB) TX bytes:272391132 (272.3 MB)
Interrupt:19 Base address:0x2000
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:14143 errors:0 dropped:0 overruns:0 frame:0
TX packets:14143 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:8000728 (8.0 MB) TX bytes:8000728 (8.0 MB)
virbr0 Link encap:Ethernet HWaddr 52:54:00:9e:51:10
inet addr:192.168.122.1 Bcast:192.168.122.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:1655 errors:0 dropped:0 overruns:0 frame:0
TX packets:2067 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:188811 (188.8 KB) TX bytes:242584 (242.5 KB)
vnet0 Link encap:Ethernet HWaddr fe:54:00:cf:1f:41
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:1655 errors:0 dropped:0 overruns:0 frame:0
TX packets:4105 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:500
RX bytes:211981 (211.9 KB) TX bytes:348362 (348.3 KB)
server> brctl show
bridge name bridge id STP enabled interfaces
virbr0 8000.5254009e5110 yes virbr0-nic
vnet0
server> ip route
default via 50.131.xxx.1 dev eth0 metric 100
50.131.xxx.0/22 dev eth0 proto kernel scope link src 50.131.xxx.xxx
192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1
192.168.138.0/24 dev eth1 proto kernel scope link src 192.168.138.1
server> virsh net-list
Name State Autostart
-----------------------------------------
default active yes
server> virsh net-edit default
<network>
<name>default</name>
<uuid>ddc855bf-8794-f4de-f1f9-7480edf9f419</uuid>
<forward mode='route'/>
<bridge name='virbr0' stp='on' delay='20' />
<mac address='52:54:00:9E:51:10'/>
<ip address='192.168.122.1' netmask='255.255.255.0'>
<dhcp>
<range start='192.168.122.100' end='192.168.122.254' />
<host mac='52:54:00:cf:1f:41' name='guest.example.com' ip='192.168.122.10' />
</dhcp>
</ip>
</network>
server> tail /etc/sysctl.conf
...
net.bridge.bridge-nf-call-ip6tables = 0
net.bridge.bridge-nf-call-iptables = 0
net.bridge.bridge-nf-call-arptables = 0
I also run ufw. It says:
server> ufw status verbose
Status: active
Logging: on (low)
Default: deny (incoming), allow (outgoing)
New profiles: skip
To Action From
-- ------ ----
OpenSSH ALLOW Anywhere
53/tcp ALLOW 192.168.138.0/24
53/udp ALLOW 192.168.138.0/24
21 ALLOW 192.168.138.0/24
192.168.138.0/24 ALLOW 192.168.138.0/24
192.168.122.0/24 ALLOW 192.168.138.0/24
192.168.138.0/24 ALLOW 192.168.122.0/24
192.168.122.0/24 ALLOW 192.168.122.0/24
To me, it sounds like something really obvious, but then, not to me it seems ...
-- Added: The behavior is the same even if I disable ufw. But here is the output of iptables-save:
# Generated by iptables-save v1.4.12 on Mon Jul 29 08:57:10 2013
*mangle
:PREROUTING ACCEPT [4511095:1341076448]
:INPUT ACCEPT [79374:20510726]
:FORWARD ACCEPT [4428917:1318506209]
:OUTPUT ACCEPT [72504:23698077]
:POSTROUTING ACCEPT [4501421:1342204286]
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
COMMIT
# Completed on Mon Jul 29 08:57:10 2013
# Generated by iptables-save v1.4.12 on Mon Jul 29 08:57:10 2013
*nat
:PREROUTING ACCEPT [33012:2764507]
:INPUT ACCEPT [16436:1476129]
:OUTPUT ACCEPT [10423:864202]
:POSTROUTING ACCEPT [10487:868042]
-A POSTROUTING -s 192.168.138.0/24 -o eth0 -j MASQUERADE
-A POSTROUTING -s 192.168.138.0/24 -o eth0 -j MASQUERADE
COMMIT
# Completed on Mon Jul 29 08:57:10 2013
# Generated by iptables-save v1.4.12 on Mon Jul 29 08:57:10 2013
*filter
:INPUT DROP [1335:71456]
:FORWARD ACCEPT [3360867:988924571]
:OUTPUT ACCEPT [18:936]
:ufw-after-forward - [0:0]
:ufw-after-input - [0:0]
:ufw-after-logging-forward - [0:0]
:ufw-after-logging-input - [0:0]
:ufw-after-logging-output - [0:0]
:ufw-after-output - [0:0]
:ufw-before-forward - [0:0]
:ufw-before-input - [0:0]
:ufw-before-logging-forward - [0:0]
:ufw-before-logging-input - [0:0]
:ufw-before-logging-output - [0:0]
:ufw-before-output - [0:0]
:ufw-logging-allow - [0:0]
:ufw-logging-deny - [0:0]
:ufw-not-local - [0:0]
:ufw-reject-forward - [0:0]
:ufw-reject-input - [0:0]
:ufw-reject-output - [0:0]
:ufw-skip-to-policy-forward - [0:0]
:ufw-skip-to-policy-input - [0:0]
:ufw-skip-to-policy-output - [0:0]
:ufw-track-input - [0:0]
:ufw-track-output - [0:0]
:ufw-user-forward - [0:0]
:ufw-user-input - [0:0]
:ufw-user-limit - [0:0]
:ufw-user-limit-accept - [0:0]
:ufw-user-logging-forward - [0:0]
:ufw-user-logging-input - [0:0]
:ufw-user-logging-output - [0:0]
:ufw-user-output - [0:0]
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j ufw-before-logging-input
-A INPUT -j ufw-before-input
-A INPUT -j ufw-after-input
-A INPUT -j ufw-after-logging-input
-A INPUT -j ufw-reject-input
-A INPUT -j ufw-track-input
-A FORWARD -d 192.168.122.0/24 -o virbr0 -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j ufw-before-logging-forward
-A FORWARD -j ufw-before-forward
-A FORWARD -j ufw-after-forward
-A FORWARD -j ufw-after-logging-forward
-A FORWARD -j ufw-reject-forward
-A OUTPUT -j ufw-before-logging-output
-A OUTPUT -j ufw-before-output
-A OUTPUT -j ufw-after-output
-A OUTPUT -j ufw-after-logging-output
-A OUTPUT -j ufw-reject-output
-A OUTPUT -j ufw-track-output
-A ufw-after-input -p udp -m udp --dport 137 -j ufw-skip-to-policy-input
-A ufw-after-input -p udp -m udp --dport 138 -j ufw-skip-to-policy-input
-A ufw-after-input -p tcp -m tcp --dport 139 -j ufw-skip-to-policy-input
-A ufw-after-input -p tcp -m tcp --dport 445 -j ufw-skip-to-policy-input
-A ufw-after-input -p udp -m udp --dport 67 -j ufw-skip-to-policy-input
-A ufw-after-input -p udp -m udp --dport 68 -j ufw-skip-to-policy-input
-A ufw-after-input -m addrtype --dst-type BROADCAST -j ufw-skip-to-policy-input
-A ufw-after-logging-input -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] "
-A ufw-before-forward -j ufw-user-forward
-A ufw-before-input -i lo -j ACCEPT
-A ufw-before-input -m state --state RELATED,ESTABLISHED -j ACCEPT
-A ufw-before-input -m state --state INVALID -j ufw-logging-deny
-A ufw-before-input -m state --state INVALID -j DROP
-A ufw-before-input -p icmp -m icmp --icmp-type 3 -j ACCEPT
-A ufw-before-input -p icmp -m icmp --icmp-type 4 -j ACCEPT
-A ufw-before-input -p icmp -m icmp --icmp-type 11 -j ACCEPT
-A ufw-before-input -p icmp -m icmp --icmp-type 12 -j ACCEPT
-A ufw-before-input -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A ufw-before-input -p udp -m udp --sport 67 --dport 68 -j ACCEPT
-A ufw-before-input -j ufw-not-local
-A ufw-before-input -d 224.0.0.251/32 -p udp -m udp --dport 5353 -j ACCEPT
-A ufw-before-input -d 239.255.255.250/32 -p udp -m udp --dport 1900 -j ACCEPT
-A ufw-before-input -j ufw-user-input
-A ufw-before-output -o lo -j ACCEPT
-A ufw-before-output -m state --state RELATED,ESTABLISHED -j ACCEPT
-A ufw-before-output -j ufw-user-output
-A ufw-logging-allow -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW ALLOW] "
-A ufw-logging-deny -m state --state INVALID -m limit --limit 3/min --limit-burst 10 -j RETURN
-A ufw-logging-deny -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] "
-A ufw-not-local -m addrtype --dst-type LOCAL -j RETURN
-A ufw-not-local -m addrtype --dst-type MULTICAST -j RETURN
-A ufw-not-local -m addrtype --dst-type BROADCAST -j RETURN
-A ufw-not-local -m limit --limit 3/min --limit-burst 10 -j ufw-logging-deny
-A ufw-not-local -j DROP
-A ufw-skip-to-policy-forward -j ACCEPT
-A ufw-skip-to-policy-input -j DROP
-A ufw-skip-to-policy-output -j ACCEPT
-A ufw-track-output -p tcp -m state --state NEW -j ACCEPT
-A ufw-track-output -p udp -m state --state NEW -j ACCEPT
-A ufw-user-input -p tcp -m tcp --dport 22 -m comment --comment "\'dapp_OpenSSH\'" -j ACCEPT
-A ufw-user-input -s 192.168.138.0/24 -p tcp -m tcp --dport 53 -j ACCEPT
-A ufw-user-input -s 192.168.138.0/24 -p udp -m udp --dport 53 -j ACCEPT
-A ufw-user-input -s 192.168.138.0/24 -p tcp -m tcp --dport 21 -j ACCEPT
-A ufw-user-input -s 192.168.138.0/24 -p udp -m udp --dport 21 -j ACCEPT
-A ufw-user-input -s 192.168.138.0/24 -d 192.168.138.0/24 -j ACCEPT
-A ufw-user-input -s 192.168.138.0/24 -d 192.168.122.0/24 -j ACCEPT
-A ufw-user-input -s 192.168.122.0/24 -d 192.168.138.0/24 -j ACCEPT
-A ufw-user-input -s 192.168.122.0/24 -d 192.168.122.0/24 -j ACCEPT
-A ufw-user-limit -m limit --limit 3/min -j LOG --log-prefix "[UFW LIMIT BLOCK] "
-A ufw-user-limit -j REJECT --reject-with icmp-port-unreachable
-A ufw-user-limit-accept -j ACCEPT
COMMIT
# Completed on Mon Jul 29 08:57:10 2013