Environment

OS          : CentOS 7
Master Node : mit1.example.com
Slave Node  : mit2.example.com
SELinux     : Disabled
Firewall    : Disabled

Steps

[Both] Install MIT Kerberos Server

# yum install -y krb5-server krb5-workstation

[Both] Edit /etc/krb5.conf

[libdefaults]
 renew_lifetime = 7d
 forwardable = true
 default_realm = EXAMPLE.COM
 ticket_lifetime = 24h
 dns_lookup_realm = false
 dns_lookup_kdc = false
 udp_preference_limit=1
 default_ccache_name = FILE:/tmp/krb5cache_%{uid}
[domain_realm]
 example.com = EXAMPLE.COM
 .example.com = EXAMPLE.COM
[logging]
 default = FILE:/var/log/krb5kdc.log
 admin_server = FILE:/var/log/kadmind.log
 kdc = FILE:/var/log/krb5kdc.log
[realms]
 EXAMPLE.COM = {
   admin_server = mit1.example.com
   kdc = mit1.example.com
   kdc = mit2.example.com
 }

[Both] Edit /var/kerberos/krb5kdc/kadm5.acl
```
*/admin@EXAMPLE.COM *
```

[Both] Edit /var/kerberos/krb5kdc/kdc.conf

[kdcdefaults]
 kdc_ports = 88, 750
 kdc_tcp_ports = 88, 750

[realms]
 EXAMPLE.COM = {
  kadmind_port = 749
  max_life = 12h 0m 0s
  max_renewable_life = 7d 0h 0m 0s
  master_key_type = aes256-cts
  acl_file = /var/kerberos/krb5kdc/kadm5.acl
  dict_file = /usr/share/dict/words
  admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
  supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal camellia256-cts:normal camellia128-cts:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
 }

[Slave] Edit /var/kerberos/krb5kdc/kpropd.acl

host/mit1.example.com@EXAMPLE.COM
host/mit2.example.com@EXAMPLE.COM

[Master] Initialized KDC database

# kdb5_util create -s
Loading random data
Initializing database '/var/kerberos/krb5kdc/principal' for realm 'EXAMPLE.COM',
master key name 'K/M@EXAMPLE.COM'
You will be prompted for the database Master Password.
It is important that you NOT FORGET this password.
Enter KDC database master key:
Re-enter KDC database master key to verify:

[Master] Start KDC and kadmin in Master Node Only

# systemctl start krb5kdc
# systemctl enable krb5kdc
# systemctl start kadmin
# systemctl enable kadmin

[Master] Create Administrator principal in Master KDC

# kadmin.local -q "addprinc root/admin" 
Authenticating as principal root/admin@EXAMPLE.COM with password. 
WARNING: no policy specified for root/admin@EXAMPLE.COM; defaulting to no policy 
Enter password for principal "root/admin@EXAMPLE.COM": <admin_password>
Re-enter password for principal "root/admin@EXAMPLE.COM": <admin_password>
Principal "root/admin@EXAMPLE.COM" created.

[Master] Create both host keytab and copy to Slave Node

# kadmin
kadmin: addprinc -randkey host/mit1.example.com
kadmin: addprinc -randkey host/mit2.example.com
kadmin: ktadd –k /etc/krb5.keytab host/mit1.example.com
kadmin: ktadd –k /etc/krb5.keytab host/mit2.example.com

Make sure /etc/krb5.keytab contain both Master and Slave node principal:

[root@mit1 ~]# klist -ekt /etc/krb5.keytab
Keytab name: FILE:/etc/krb5.keytab
KVNO Timestamp           Principal
---- ------------------- ------------------------------------------------------
2 03/25/2022 16:15:05 host/mit1.example.com@EXAMPLE.COM (aes256-cts-hmac-sha1-96)
2 03/25/2022 16:15:05 host/mit1.example.com@EXAMPLE.COM (aes128-cts-hmac-sha1-96)
2 03/25/2022 16:15:05 host/mit1.example.com@EXAMPLE.COM (des3-cbc-sha1)
2 03/25/2022 16:15:05 host/mit1.example.com@EXAMPLE.COM (arcfour-hmac)
2 03/25/2022 16:15:05 host/mit1.example.com@EXAMPLE.COM (camellia256-cts-cmac)
2 03/25/2022 16:15:05 host/mit1.example.com@EXAMPLE.COM (camellia128-cts-cmac)
2 03/25/2022 16:15:05 host/mit1.example.com@EXAMPLE.COM (des-hmac-sha1)
2 03/25/2022 16:15:05 host/mit1.example.com@EXAMPLE.COM (des-cbc-md5)
2 03/25/2022 16:15:05 host/mit2.example.com@EXAMPLE.COM (aes256-cts-hmac-sha1-96)
2 03/25/2022 16:15:05 host/mit2.example.com@EXAMPLE.COM (aes128-cts-hmac-sha1-96)
2 03/25/2022 16:15:05 host/mit2.example.com@EXAMPLE.COM (des3-cbc-sha1)
2 03/25/2022 16:15:05 host/mit2.example.com@EXAMPLE.COM (arcfour-hmac)
2 03/25/2022 16:15:05 host/mit2.example.com@EXAMPLE.COM (camellia256-cts-cmac)
2 03/25/2022 16:15:05 host/mit2.example.com@EXAMPLE.COM (camellia128-cts-cmac)
2 03/25/2022 16:15:05 host/mit2.example.com@EXAMPLE.COM (des-hmac-sha1)
2 03/25/2022 16:15:05 host/mit2.example.com@EXAMPLE.COM (des-cbc-md5)

Copy to slave node:

[root@mit1 ~]# scp /etc/krb5.keytab mit2:/etc/krb5.keytab

[Both] Create kpropd service using xinetd on both Master and Slave node

# yum install -y xinetd

Create /etc/xinetd.d/krb5_prop file

service krb5_prop
{
   disable         = no
   socket_type     = stream
   protocol        = tcp
   user            = root
   wait            = no
   server          = /usr/sbin/kpropd
}

Start xinetd service

# systemctl start xinetd
# systemctl enable xinetd

[Slave] Make sure these files available in Slave node, else you can copy from Master node.

/etc/krb5.conf 
/var/kerberos/krb5kdc/kadm5.acl 
/var/kerberos/krb5kdc/kdc.conf
/var/kerberos/krb5kdc/kpropd.acl
/var/kerberos/krb5kdc/.k5.EXAMPLE.COM

You can copy /var/kerberos/krb5kdc/.k5.EXAMPLE.COM from Master node

[Master] Perform the initial KDC database propagation to the Slave KDC

Dump database. It will generate slave_datatrans file.

[root@mit1 ]# kdb5_util dump /var/kerberos/krb5kdc/slave_datatrans
[root@mit1 ]# ll /var/kerberos/krb5kdc
total 52
-rw------- 1 root root    22 Sep 30  2020 kadm5.acl
-rw------- 1 root root   539 Mar 25 15:12 kdc.conf
-rw------- 1 root root 16384 Mar 25 15:52 principal
-rw------- 1 root root  8192 Mar 25 15:44 principal.kadm5
-rw------- 1 root root     0 Mar 25 15:44 principal.kadm5.lock
-rw------- 1 root root     0 Mar 25 15:52 principal.ok
-rw------- 1 root root 10100 Mar 25 16:02 slave_datatrans
-rw------- 1 root root     1 Mar 25 16:02 slave_datatrans.dump_ok

Propagate to Slave node:

[root@mit1 ]# kprop -f /var/kerberos/krb5kdc/slave_datatrans mit2.example.com
Database propagation to mit2.example.com: SUCCEEDED

Check if there file call from_master in Slave node with same exactly size

[root@mit2 ]# ll /var/kerberos/krb5kdc
total 60
-rw------- 1 root root 10100 Mar 25 16:03 from_master
-rw------- 1 root root    22 Sep 30  2020 kadm5.acl
-rw------- 1 root root   539 Mar 25 15:12 kdc.conf
-rw-r--r-- 1 root root    68 Mar 25 15:42 kpropd.acl
-rw------- 1 root root 20480 Mar 25 16:45 principal
-rw------- 1 root root  8192 Mar 25 16:45 principal.kadm5
-rw------- 1 root root     0 Mar 25 16:21 principal.kadm5.lock
-rw------- 1 root root     0 Mar 25 16:45 principal.ok

[Slave] Start KDC Service in Slave node
```
# systemctl start krb5kdc
# systemctl enable krb5kdc
```
kadmin service won't be able to start in slave node.

[Master] Create cronjob script to propagate the updates from the Master KDC to the Slave KDC.

[root@mit1 ~]# cat /var/kerberos/kdc-slave-propogate.sh
#!/usr/bin/env bash
kdclist="mit2.example.com"
/sbin/kdb5_util dump /var/kerberos/krb5kdc/slave_datatrans
for kdc in $kdclist
do
  /sbin/kprop -f /var/kerberos/krb5kdc/slave_datatrans $kdc
done

[root@mit1 ~]# chmod +x /var/kerberos/kdc-slave-propogate.sh
[root@mit1 ~]# crontab -l
*/5 * * * * bash -x /var/kerberos/kdc-slave-propogate.sh > /var/kerberos/kdc-slave-propogate.log 2>&1

Test HA

Test Case 1: Test on synced user: "user01"

Kinit user user01 on Client node

kholis@air:~$ date
Fri Mar 25 17:38:39 WIB 2022
kholis@air:~$ kinit user01
Password for user01@EXAMPLE.COM:
kholis@air:~$ klist
Ticket cache: FILE:/tmp/krb5cache_501
Default principal: user01@EXAMPLE.COM

Valid starting       Expires              Service principal
25/03/2022 17:38:43  26/03/2022 05:38:43  krbtgt/EXAMPLE.COM@EXAMPLE.COM
  renew until 01/04/2022 17:38:43
kholis@air:~$ kdestroy

Stop KDC Server on Master node (mit1.example.com)

[root@mit1 ~]# date
Fri Mar 25 17:38:57 WIB 2022
[root@mit1 ~]# systemctl stop krb5kdc
[root@mit1 ~]# systemctl stop kadmin

Kinit again on Client node. Kinit is succeed even Master Node is stopped.

kholis@air:~$ date
Fri Mar 25 17:39:16 WIB 2022
kholis@air:~$ kinit user01
Password for user01@EXAMPLE.COM:
kholis@air:~$ klist
Ticket cache: FILE:/tmp/krb5cache_501
Default principal: user01@EXAMPLE.COM

Valid starting       Expires              Service principal
25/03/2022 17:39:20  26/03/2022 05:39:20  krbtgt/EXAMPLE.COM@EXAMPLE.COM
  renew until 01/04/2022 17:39:20
kholis@air:~$ kdestroy

Test Case 2: Test on non-synced user: "user02"

Create new user user02 on Master Node

[root@mit1 ~]# kadmin.local
Authenticating as principal host/admin@EXAMPLE.COM with password.
kadmin.local:  addprinc user02
WARNING: no policy specified for user02@EXAMPLE.COM; defaulting to no policy
Enter password for principal "user02@EXAMPLE.COM":
Re-enter password for principal "user02@EXAMPLE.COM":
Principal "user02@EXAMPLE.COM" created.
kadmin.local:  quit

Kinit user user02 on Client Node

kholis@air:~$ date
Fri Mar 25 17:46:15 WIB 2022
kholis@air:~$ kinit user02
Password for user02@EXAMPLE.COM:
kholis@air:~$ klist
Ticket cache: FILE:/tmp/krb5cache_501
Default principal: user02@EXAMPLE.COM

Valid starting       Expires              Service principal
25/03/2022 17:46:19  26/03/2022 05:46:19  krbtgt/EXAMPLE.COM@EXAMPLE.COM
  renew until 01/04/2022 17:46:19
kholis@air:~$ kdestroy

Stop KDC Server on Master node (mit1.example.com)

[root@mit1 ~]# date
Fri Mar 25 17:46:34 WIB 2022
[root@mit1 ~]# systemctl stop kadmin
[root@mit1 ~]# systemctl stop krb5kdc

Kinit user user02 on Client Node. Failed because KDC db is not sync to Slave Node.

kholis@air:~$ date
Fri Mar 25 17:46:45 WIB 2022
(base) kholis@air:~$ kinit user02
kinit: Client 'user02@EXAMPLE.COM' not found in Kerberos database while getting initial credentials
(base) kholis@air:~$ kinit user02
kinit: Client 'user02@EXAMPLE.COM' not found in Kerberos database while getting initial credentials

Wait until db is propagate to slave (my crontab is every 5m)

Kinit user02 from Client. Success kinit to MIT Slave Node.

kholis@air:~$ date
Fri Mar 25 17:50:13 WIB 2022
kholis@air:~$ kinit user02
Password for user02@EXAMPLE.COM:
kholis@air:~$ klist
Ticket cache: FILE:/tmp/krb5cache_501
Default principal: user02@EXAMPLE.COM

Valid starting       Expires              Service principal
25/03/2022 17:50:15  26/03/2022 05:50:15  krbtgt/EXAMPLE.COM@EXAMPLE.COM
  renew until 01/04/2022 17:50:15

MIT Kerberos: Setting up KDC Master-Slave for High Availability

Table of contents

Environment

Steps

Test HA

Test Case 1: Test on synced user: "user01"

Test Case 2: Test on non-synced user: "user02"

References: