I've created a small OpenStack cloud consisting of 3 nodes that all contain the same services, using kolla ansible. I can start instances using the Cirros toy image no problem, and also an offical debian cloud image, however starting a Windows 10 image I created myself does not work.
First a couple of details on the images I have tried:
$ openstack image list --long
+-----------+-------------+------------------+-------------+--------+------------+-----------+
| Name | Disk Format | Container Format | Size | Status | Visibility | Protected |
+-----------+-------------+------------------+-------------+--------+------------+-----------+
| Win10Pro | qcow2 | bare | 12238913536 | active | shared | False |
| cirros | qcow2 | bare | 16338944 | active | public | False |
| debian-11 | qcow2 | bare | 253231104 | active | public | False |
+-----------+-------------+------------------+-------------+--------+------------+-----------+
My hunch is that it has something to do with the size (Windows is huuuge at more than 10GB), as API calls are slow. Most calls take more than 1 second, like e.g. openstack server list, which takes between 1.5 and 2 seconds. Ping times between the machines seem fine to me, around 0.4 ms.
So how can I best debug this?
This is the error when launching a Windows instance:
'code': 500, 'created': '2022-06-25T06:21:06Z', 'message': 'Build of instance 3d0498e2-ddde-4bcf-ab54-7881360fe9f7 aborted:
("Connection broken: TimeoutError(110, \'Connection timed out\')", TimeoutError(110, \'Connection timed out\'))', 'details': 'Traceback (most recent call last):
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2746, in _build_resources
self._shutdown_instance(context, instance,
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 3005, in _shutdown_instance
self._try_deallocate_network(context, instance, requested_networks)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2922, in _try_deallocate_network
self._set_instance_obj_error_state(instance)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 227, in __exit__
self.force_reraise()
File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
raise self.value
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2917, in _try_deallocate_network
_deallocate_network_with_retries()
File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_service/loopingcall.py", line 436, in func
return evt.wait()
File "/var/lib/kolla/venv/lib/python3.9/site-packages/eventlet/event.py", line 125, in wait
result = hub.switch()
File "/var/lib/kolla/venv/lib/python3.9/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop
result = func(*self.args, **self.kw)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_service/loopingcall.py", line 407, in _func
result = f(*args, **kwargs)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2906, in _deallocate_network_with_retries
self._deallocate_network(
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2150, in _deallocate_network
self.network_api.deallocate_for_instance(
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/network/neutron.py", line 1811, in deallocate_for_instance
self._delete_ports(neutron, instance, ports, raise_if_fail=True)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/network/neutron.py", line 1785, in _delete_ports
raise exceptions[0]
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/network/neutron.py", line 1775, in _delete_ports
neutron.delete_port(port)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/network/neutron.py", line 197, in wrapper
ret = obj(*args, **kwargs)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/neutronclient/v2_0/client.py", line 831, in delete_port
return self.delete(self.port_path % (port))
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/network/neutron.py", line 197, in wrapper
ret = obj(*args, **kwargs)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/neutronclient/v2_0/client.py", line 352, in delete
return self.retry_request("DELETE", action, body=body,
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/network/neutron.py", line 197, in wrapper
ret = obj(*args, **kwargs)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request
return self.do_request(method, action, body=body,
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/network/neutron.py", line 197, in wrapper
ret = obj(*args, **kwargs)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/neutronclient/v2_0/client.py", line 297, in do_request
self._handle_fault_response(status_code, replybody, resp)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/network/neutron.py", line 197, in wrapper
ret = obj(*args, **kwargs)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response
exception_handler_v20(status_code, error_body)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20
raise client_exc(message=error_message,
neutronclient.common.exceptions.NeutronClientException: <html><body><h1>504 Gateway Time-out</h1>
The server didn\'t respond in time.
</body></html>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2312, in _do_build_and_run_instance
self._build_and_run_instance(context, instance, image,
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2521, in _build_and_run_instance
compute_utils.notify_about_instance_create(
File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 227, in __exit__
self.force_reraise()
File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
raise self.value
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2494, in _build_and_run_instance
LOG.info(\'Took %0.2f seconds to spawn the instance on \'
File "/usr/lib/python3.9/contextlib.py", line 135, in __exit__
self.gen.throw(type, value, traceback)
File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 2754, in _build_resources
raise exception.BuildAbortException(
nova.exception.BuildAbortException: Build of instance 3d0498e2-ddde-4bcf-ab54-7881360fe9f7 aborted: ("Connection broken: TimeoutError(110, \'Connection timed out\')", TimeoutError(110, \'Connection timed out\'))
'}
