Problem:
Foreman fails to add DHCP reservation during provisioning causing provisioning to fail.
Expected outcome:
DHCP reservation is set and provisioning proceeds.
Foreman and Proxy versions:
Katello 3.5.2
Foreman 1.16.1
Other relevant data:
## /etc/dhcp/dhcpd.conf is readable by foreman-proxy.
# su -s /bin/bash foreman-proxy
bash-4.2$ cat /etc/dhcp/dhcpd.conf
# dhcpd.conf
authoritative;
allow booting;
allow bootp;
ddns-update-style none;
omapi-port 7911;
omapi-key omapi_key;
key omapi_key {
algorithm HMAC-MD5;
secret "somesecret";
}
default-lease-time 43200;
max-lease-time 86400;
#option domain-name "domain.internal";
option domain-name-servers 10.x.x.xxx;
option fqdn.no-client-update on; # set the "O" and "S" flag bits
option fqdn.rcode2 255;
option pxegrub code 150 = text ;
# Bootfile Handoff
next-server 10.1.105.12;
option architecture code 93 = unsigned integer 16 ;
if option architecture = 00:06 {
filename "grub2/shim.efi";
} elsif option architecture = 00:07 {
filename "grub2/shim.efi";
} elsif option architecture = 00:09 {
filename "grub2/shim.efi";
} else {
filename "pxelinux.0";
}
log-facility local6;
include "/etc/dhcp/dhcpd.hosts";
# domain.internal
subnet 10.1.105.0 netmask 255.255.255.0 {
pool
{
range 10.1.105.250 10.1.105.253;
}
option subnet-mask 255.255.255.0;
option routers 10.1.105.12;
option domain-search "domain.internal";
}
## /etc/foreman-proxy/settings.d/dhcp.yml
:enabled: https
:use_provider: dhcp_isc
:server: 127.0.0.1
:subnets: [10.1.47.0/255.255.255.0, 10.5.47.0/255.255.255.0, 10.1.105.0/255.255.255.0]
---
## /etc/foreman-proxy/settings.d/dhcp_isc.yml
:config: /etc/dhcp/dhcpd.conf
:leases: /var/lib/dhcpd/dhcpd.leases
:key_name: omapi_key
:key_secret: somesecret
:omapi_port: 7911
## Permissions in /etc/dhcp/
$ ll /etc/dhcp/
bash: ll: command not found
bash-4.2$ ls -altr /etc/dhcp/
total 44
-rw-r--r-- 1 root root 120 May 15 22:56 dhcpd6.conf
drwxr-xr-x+ 2 root root 37 May 15 22:56 dhclient.d
-rw-r--r--+ 1 dhcpd dhcpd 20 May 17 08:49 dhcpd.hosts
-rwxr-xr-x 1 dhcpd dhcpd 1350 Jun 9 11:52 dhcpd.conf
drwxr-x---+ 2 root dhcpd 28 Jun 9 12:10 scripts
drwxr-x---+ 7 root root 4096 Jun 9 12:10 .
drwxr-xr-x+ 2 root root 28 Jun 9 12:10 dhclient-exit-hooks.d
drwxr-xr-x. 129 root root 12288 Jun 9 12:10 .
## /var/lib/dhcpd/ is readable by foreman-proxy
$ cat /var/lib/dhcpd/dhcpd.leases
# The format of this file is documented in the dhcpd.leases(5) manual page.
# This lease file was written by isc-dhcp-4.2.5
server-duid "\000\001\000\001\"\217\216/\000PV\230\332\350";
## Permissions in /var/lib/dhcpd/dhcpd.leases
$ ls -altr /var/lib/dhcpd/
total 20
-rw-r--r-- 1 dhcpd dhcpd 0 May 15 22:56 dhcpd6.leases
drwxr-xr-x. 63 root root 4096 Jun 1 13:19 ..
-rw-r--r-- 1 dhcpd dhcpd 187 Jun 9 12:43 dhcpd.leases~
-rw-r--r-- 1 dhcpd dhcpd 187 Jun 9 12:44 dhcpd.leases
drwxr-xr-x+ 2 dhcpd dhcpd 138 Jun 9 12:44 .
# curl -kvs --cert /etc/foreman-proxy/foreman_ssl_cert.pem --key /etc/foreman-proxy/foreman_ssl_key.pem --cacert /etc/foreman-proxy/foreman_ssl_ca.pem https://$(hostname):9090/dhcp
* About to connect() to katello port 9090 (#0)
* Trying 10.1.105.12...
* Connected to katello (10.1.105.12) port 9090 (#0)
* Initializing NSS with certpath: sql:/etc/pki/nssdb
* skipping SSL peer certificate verification
* NSS: client certificate from file
* subject: CN=katello.domain.internal
* start date: May 14 04:04:05 2018 GMT
* expire date: May 14 04:04:05 2023 GMT
* common name: katello.domain.internal
* issuer: E=TISS.ToolsMgmt@education.wa.edu.au,CN=puppetca.domain.internal,O=Department of Education,L=Perth,ST=Western Australia,C=AU
* SSL connection using TLS_RSA_WITH_AES_128_GCM_SHA256
* Server certificate:
* subject: CN=katello.domain.internal
* start date: May 14 04:04:05 2018 GMT
* expire date: May 14 04:04:05 2023 GMT
* common name: katello.domain.internal
* issuer: E=admin@us.au,CN=puppetca.domain.internal,O=OurORG,L=Perth,ST=Western Australia,C=AU
> GET /dhcp HTTP/1.1
> User-Agent: curl/7.29.0
> Host: katello:9090
> Accept: */*
>
< HTTP/1.1 200 OK
< Content-Type: application/json
< Content-Length: 176
< X-Content-Type-Options: nosniff
< Server:
< Date: Sat, 09 Jun 2018 07:02:01 GMT
< Connection: Keep-Alive
<
* Connection #0 to host katello left intact
[{"network":"10.1.105.0","netmask":"255.255.255.0","options":{"routers":["10.1.105.12"]}},{"network":"10.5.47.0","netmask":"255.255.255.0","options":{"routers":["10.5.47.1"]}}][root@katello foreman]#
## Error in webui during provisioning
Unable to save
Create DHCP Settings for debra-lazusky.domain.internal task failed with the following error: ERF12-6899 [ProxyAPI::ProxyException]: Unable to set DHCP entry ([RestClient::BadRequest]: 400 Bad Request) for proxy https://katello.domain.internal:9090/dhcp
==> /var/log/foreman-proxy/proxy.log <==
D, [2018-06-09T14:32:37.274516 ] DEBUG -- : accept: 10.1.105.12:57954
D, [2018-06-09T14:32:37.276359 ] DEBUG -- : Rack::Handler::WEBrick is invoked.
D, [2018-06-09T14:32:37.280293 d7a75f2a] DEBUG -- : verifying remote client 10.1.105.12 against trusted_hosts ["katello.domain.internal", "puppetca.domain.internal", "e7359svin2503.domain.internal", "foreman.domain.internal", "puppet.domain.internal"]
D, [2018-06-09T14:32:37.284274 d7a75f2a] DEBUG -- : omshell: executed - set name = "debra-lazusky.domain.internal"
D, [2018-06-09T14:32:37.284454 d7a75f2a] DEBUG -- : nil
D, [2018-06-09T14:32:37.284598 d7a75f2a] DEBUG -- : omshell: executed - set ip-address = 10.1.105.250
D, [2018-06-09T14:32:37.284700 d7a75f2a] DEBUG -- : nil
D, [2018-06-09T14:32:37.284814 d7a75f2a] DEBUG -- : omshell: executed - set hardware-address = 00:50:56:98:1b:92
D, [2018-06-09T14:32:37.284900 d7a75f2a] DEBUG -- : nil
D, [2018-06-09T14:32:37.285044 d7a75f2a] DEBUG -- : omshell: executed - set hardware-type = 1
D, [2018-06-09T14:32:37.285153 d7a75f2a] DEBUG -- : nil
D, [2018-06-09T14:32:37.285453 d7a75f2a] DEBUG -- : omshell: executed - set statements = "filename = \"pxelinux.0\"; next-server = 0a:01:69:0c; option host-name = \"debra-lazusky.domain.internal\";"
D, [2018-06-09T14:32:37.285561 d7a75f2a] DEBUG -- : nil
D, [2018-06-09T14:32:37.285679 d7a75f2a] DEBUG -- : omshell: executed - create
D, [2018-06-09T14:32:37.285759 d7a75f2a] DEBUG -- : nil
E, [2018-06-09T14:32:37.302557 d7a75f2a] ERROR -- : Omshell failed:
> > > > obj: <null>
, > obj: host
, > obj: host
, name = "debra-lazusky.domain.internal"
, > obj: host
, name = "debra-lazusky.domain.internal"
, ip-address = 0a:01:69:fa
, > obj: host
, name = "debra-lazusky.domain.internal"
, ip-address = 0a:01:69:fa
, hardware-address = 00:50:56:98:1b:92
, > obj: host
, name = "debra-lazusky.domain.internal"
, ip-address = 0a:01:69:fa
, hardware-address = 00:50:56:98:1b:92
, hardware-type = 1
, > obj: host
, name = "debra-lazusky.domain.internal"
, ip-address = 0a:01:69:fa
, hardware-address = 00:50:56:98:1b:92
, hardware-type = 1
, statements = "filename = "pxelinux.0"; next-server = 0a:01:69:0c; option host-name = "debra-lazusky.domain.internal";"
, > can't open object: connection reset by peer
, obj: host
, name = "debra-lazusky.domain.internal"
, ip-address = 0a:01:69:fa
, hardware-address = 00:50:56:98:1b:92
, hardware-type = 1
, statements = "filename = "pxelinux.0"; next-server = 0a:01:69:0c; option host-name = "debra-lazusky.domain.internal";"
, >
E, [2018-06-09T14:32:37.303182 d7a75f2a] ERROR -- : Failed to add DHCP reservation for debra-lazusky.domain.internal (10.1.105.250 / 00:50:56:98:1b:92)
D, [2018-06-09T14:32:37.303242 d7a75f2a] DEBUG -- : Failed to add DHCP reservation for debra-lazusky.domain.internal (10.1.105.250 / 00:50:56:98:1b:92) (Proxy::DHCP::Error)
## pastebin of full logs during provisioning attempt
https://pastebin.com/raw/kevawUQK