I have check a lot of other L2TP/IPsec VPN post and none of them seem to quite match the issues I am having so here is what going on.
I am trying to set up a VPN on my Arch Linux server that I can connect to from my local devices (most of which are running Windows 8.1). For the whole post, I am going to use the fake external IPs of 123.1.1.1 and ff:ff:ff:ff for the Arch serverand 123.2.2.2 as the IP for the Windows 8.1 desktop I am trying to connect from.
I have everything set up and running following this config setup. Ports 1701 TCP, 4500 UDP and 500 UDP are open properly on the Arch server and it is not an ARM server, but rather a 64-bit server. When I try to connect from the Windows 8 device, I get the following error:
Error 789: The L2TP connection attempt failed because the security error encountered a processing error during the initial negotions with the remote computer.
ipsec auto --status (after I try to connect)
000 using kernel interface: netkey
000 interface lo/lo ::1
000 interface eth0/eth0 ff:ff:ff:ff::1
000 interface eth0/eth0 123.1.1.1
000 interface eth0/eth0 123.1.1.1
000 %myid = (none)
000 debug none
000
000 virtual_private (%priv):
000 - allowed 5 subnets: 10.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12, fd00::/8, fe80::/10
000 - disallowed 0 subnets:
000 WARNING: Disallowed subnets in virtual_private= is empty. If you have
000 private address space in internal use, it should be excluded!
000
000 algorithm ESP encrypt: id=2, name=ESP_DES, ivlen=8, keysizemin=64, keysizemax=64
000 algorithm ESP encrypt: id=3, name=ESP_3DES, ivlen=8, keysizemin=192, keysizemax=192
000 algorithm ESP encrypt: id=6, name=ESP_CAST, ivlen=8, keysizemin=40, keysizemax=128
000 algorithm ESP encrypt: id=7, name=ESP_BLOWFISH, ivlen=8, keysizemin=40, keysizemax=448
000 algorithm ESP encrypt: id=11, name=ESP_NULL, ivlen=0, keysizemin=0, keysizemax=0
000 algorithm ESP encrypt: id=12, name=ESP_AES, ivlen=8, keysizemin=128, keysizemax=256
000 algorithm ESP encrypt: id=13, name=ESP_AES_CTR, ivlen=8, keysizemin=160, keysizemax=288
000 algorithm ESP encrypt: id=14, name=ESP_AES_CCM_A, ivlen=8, keysizemin=128, keysizemax=256
000 algorithm ESP encrypt: id=15, name=ESP_AES_CCM_B, ivlen=8, keysizemin=128, keysizemax=256
000 algorithm ESP encrypt: id=16, name=ESP_AES_CCM_C, ivlen=8, keysizemin=128, keysizemax=256
000 algorithm ESP encrypt: id=18, name=ESP_AES_GCM_A, ivlen=8, keysizemin=160, keysizemax=288
000 algorithm ESP encrypt: id=19, name=ESP_AES_GCM_B, ivlen=12, keysizemin=160, keysizemax=288
000 algorithm ESP encrypt: id=20, name=ESP_AES_GCM_C, ivlen=16, keysizemin=160, keysizemax=288
000 algorithm ESP encrypt: id=22, name=ESP_CAMELLIA, ivlen=8, keysizemin=128, keysizemax=256
000 algorithm ESP encrypt: id=252, name=ESP_SERPENT, ivlen=8, keysizemin=128, keysizemax=256
000 algorithm ESP encrypt: id=253, name=ESP_TWOFISH, ivlen=8, keysizemin=128, keysizemax=256
000 algorithm ESP auth attr: id=1, name=AUTH_ALGORITHM_HMAC_MD5, keysizemin=128, keysizemax=128
000 algorithm ESP auth attr: id=2, name=AUTH_ALGORITHM_HMAC_SHA1, keysizemin=160, keysizemax=160
000 algorithm ESP auth attr: id=5, name=AUTH_ALGORITHM_HMAC_SHA2_256, keysizemin=256, keysizemax=256
000 algorithm ESP auth attr: id=6, name=AUTH_ALGORITHM_HMAC_SHA2_384, keysizemin=384, keysizemax=384
000 algorithm ESP auth attr: id=7, name=AUTH_ALGORITHM_HMAC_SHA2_512, keysizemin=512, keysizemax=512
000 algorithm ESP auth attr: id=8, name=AUTH_ALGORITHM_HMAC_RIPEMD, keysizemin=160, keysizemax=160
000 algorithm ESP auth attr: id=9, name=AUTH_ALGORITHM_AES_CBC, keysizemin=128, keysizemax=128
000 algorithm ESP auth attr: id=251, name=AUTH_ALGORITHM_NULL_KAME, keysizemin=0, keysizemax=0
000
000 algorithm IKE encrypt: id=0, name=(null), blocksize=16, keydeflen=131
000 algorithm IKE encrypt: id=5, name=OAKLEY_3DES_CBC, blocksize=8, keydeflen=192
000 algorithm IKE encrypt: id=7, name=OAKLEY_AES_CBC, blocksize=16, keydeflen=128
000 algorithm IKE hash: id=1, name=OAKLEY_MD5, hashsize=16
000 algorithm IKE hash: id=2, name=OAKLEY_SHA1, hashsize=20
000 algorithm IKE hash: id=4, name=OAKLEY_SHA2_256, hashsize=32
000 algorithm IKE hash: id=6, name=OAKLEY_SHA2_512, hashsize=64
000 algorithm IKE dh group: id=2, name=OAKLEY_GROUP_MODP1024, bits=1024
000 algorithm IKE dh group: id=5, name=OAKLEY_GROUP_MODP1536, bits=1536
000 algorithm IKE dh group: id=14, name=OAKLEY_GROUP_MODP2048, bits=2048
000 algorithm IKE dh group: id=15, name=OAKLEY_GROUP_MODP3072, bits=3072
000 algorithm IKE dh group: id=16, name=OAKLEY_GROUP_MODP4096, bits=4096
000 algorithm IKE dh group: id=17, name=OAKLEY_GROUP_MODP6144, bits=6144
000 algorithm IKE dh group: id=18, name=OAKLEY_GROUP_MODP8192, bits=8192
000 algorithm IKE dh group: id=22, name=OAKLEY_GROUP_DH22, bits=1024
000 algorithm IKE dh group: id=23, name=OAKLEY_GROUP_DH23, bits=2048
000 algorithm IKE dh group: id=24, name=OAKLEY_GROUP_DH24, bits=2048
000
000 stats db_ops: {curr_cnt, total_cnt, maxsz} :context={0,0,0} trans={0,0,0} attrs={0,0,0}
000
000 "L2TP-PSK-noNAT": 123.1.1.1<123.1.1.1>:17/1701...%any:17/%any; unrouted; eroute owner: #0
000 "L2TP-PSK-noNAT": myip=unset; hisip=unset;
000 "L2TP-PSK-noNAT": ike_life: 28800s; ipsec_life: 3600s; rekey_margin: 540s; rekey_fuzz: 100%; keyingtries: 3
000 "L2TP-PSK-noNAT": policy: PSK+ENCRYPT+PFS+IKEv2ALLOW+SAREFTRACK+lKOD+rKOD; prio: 32,32; interface: eth0;
000 "L2TP-PSK-noNAT": dpd: action:clear; delay:10; timeout:20;
000 "L2TP-PSK-noNAT": newest ISAKMP SA: #0; newest IPsec SA: #0;
000 "L2TP-PSK-noNAT"[2]: 123.1.1.1<123.1.1.1>:17/1701...123.2.2.2[10.0.0.231]:17/1701; unrouted; eroute owner: #0
000 "L2TP-PSK-noNAT"[2]: myip=unset; hisip=unset;
000 "L2TP-PSK-noNAT"[2]: ike_life: 28800s; ipsec_life: 3600s; rekey_margin: 540s; rekey_fuzz: 100%; keyingtries: 3
000 "L2TP-PSK-noNAT"[2]: policy: PSK+ENCRYPT+PFS+IKEv2ALLOW+SAREFTRACK+lKOD+rKOD; prio: 32,32; interface: eth0;
000 "L2TP-PSK-noNAT"[2]: dpd: action:clear; delay:10; timeout:20;
000 "L2TP-PSK-noNAT"[2]: newest ISAKMP SA: #1; newest IPsec SA: #0;
000 "L2TP-PSK-noNAT"[2]: IKE algorithm newest: AES_CBC_256-SHA1-MODP2048
000
000 #2: "L2TP-PSK-noNAT"[2] 123.2.2.2:55371 STATE_QUICK_R0 (expecting QI1); EVENT_CRYPTO_FAILED in 286s; nodpd; idle; import:not set
000 #1: "L2TP-PSK-noNAT"[2] 123.2.2.2:55371 STATE_MAIN_R3 (sent MR3, ISAKMP SA established); EVENT_SA_REPLACE in 28516s; newest ISAKMP; nodpd; idle; import:not set
000
ipsec verify
Checking if IPsec got installed and started correctly:
Version check and ipsec on-path [OK]
Openswan U2.6.41/K3.14.12-1-lts (netkey)
See `ipsec --copyright' for copyright information.
Checking for IPsec support in kernel [OK]
NETKEY: Testing XFRM related proc values
ICMP default/send_redirects [OK]
ICMP default/accept_redirects [OK]
XFRM larval drop [OK]
Hardware random device check [N/A]
Two or more interfaces found, checking IP forwarding [OK]
Checking rp_filter [ENABLED]
/proc/sys/net/ipv4/conf/eth0/rp_filter [ENABLED]
Checking that pluto is running [OK]
Pluto listening for IKE on udp 500 [OK]
Pluto listening for IKE on tcp 500 [NOT IMPLEMENTED]
Pluto listening for IKE/NAT-T on udp 4500 [OK]
Pluto listening for IKE/NAT-T on tcp 4500 [NOT IMPLEMENTED]
Pluto listening for IKE on tcp 10000 (cisco) [OK]
Checking NAT and MASQUERADEing [TEST INCOMPLETE]
Checking 'ip' command [OK]
Checking 'iptables' command [OK]
ipsec verify: encountered errors
Configs:
/etc/ipsec.conf
version 2
config setup
dumpdir=/var/run/pluto/
nat_traversal=yes virtual_private=%v4:10.0.0.0/8,%v4:192.168.0.0/16,%v4:172.16.0.0/12,%v6:fd00::/8,%v6:fe80::/10
protostack=netkey
plutoopts="--interface=eth0"
force_keepalive=yes
keep_alive=60
conn L2TP-PSK-noNAT
authby=secret
pfs=yes
auto=add
keyingtries=3
ikelifetime=8h
keylife=1h
type=transport
left=123.1.1.1
leftprotoport=17/1701
right=%any
rightprotoport=17/%any
dpddelay=10
dpdtimeout=20
dpdaction=clear
/etc/ipsec.secrets (values changed to bogus stuff for security reason, but the pre-shared keys has been verifed on the client)
: RSA {
# RSA 2192 bits mydomain.net Thu Jul 17 09:16:05 2014
# for signatures only, UNSAFE FOR ENCRYPTION
#pubkey=0xf1f1f1
Modulus: 0xf1f1f1
PublicExponent: 0x03
# everything after this point is secret
PrivateExponent: 0xf1f1f1
Prime1: 0xf1f1f1
Prime2: 0xf1f1f1
Exponent1: 0xf1f1f1
Exponent2: 0xf1f1f1
Coefficient: 0xf1f1f1
}
# do not change the indenting of that "}"
123.1.1.1 %any: PSK "somereallylongstring"
/etc/xl2tpd/xl2tpd.conf
[global]
ipsec saref = yes
saref refinfo = 30
[lns default]
ip range = 172.16.1.30-172.16.1.100
local ip = 172.16.1.1
unix authentication = yes
require authentication = yes
ppp debug = yes
pppoptfile = /etc/ppp/options.xl2tpd
length bit = yes
/etc/ppp/options.xl2tpd
login
ms-dns 208.67.222.222
ms-dns 208.67.220.220
auth
mtu 1200
mru 1000
crtscts
hide-password
modem
name l2tpd
proxyarp
lcp-echo-interval 30
lcp-echo-failure 4
/etc/ppp/pap-secrets
# Secrets for authentication using PAP
# client server secret IP addresses
* l2tpd "" *
/etc/pam.d/ppp
auth required pam_nologin.so
auth required pam_unix.so
account required pam_unix.so
session required pam_unix.so
