Debian: LVS (Direct Route/DR)

Direct Routing: referred to as DR, adopts a semi open network structure, similar to the structure of TUN mode (IP tunnel), but the nodes are not scattered in different places, but located in the same physical network with the scheduler. The load scheduler connects with each node server through the local network, so there is no need to establish a dedicated IP tunnel.

In a direct routing LVS setup, the LVS router needs to receive incoming requests and route them to the proper real server for processing. The real servers then need to directly route the response to the client. So, for example, if the client is on the Internet, and sends the packet through the LVS router to a real server, the real server must be able to go directly to the client via the Internet. This can be done by configuring a gateway for the real server to pass packets to the Internet. Each real server in the server pool can have its own separate gateway (and each gateway with its own connection to the Internet), allowing for maximum throughput and scalability. For typical LVS setups, however, the real servers can communicate through one gateway (and therefore one network connection).

In order to configure direct routing using each real server must have their virtual IP address configured, so they can directly route packets. ARP requests for the VIP are ignored entirely by the real servers, and any ARP packets that might otherwise be sent containing the VIPs are mangled to contain the real server’s IP instead of the VIPs.

In the following example the following IP addresses will be defined:

• Load Balancer: VIP: 192.168.100.9
• Load Balancer #1: 192.168.100.10
• Load Balancer #2: 192.168.100.11
• Web Server #1: 192.168.100.20
• Web Server #2: 192.168.100.21

Step 1: Web Server #1

Configure the network interfaces by editing /etc/network/interfaces

# The loopback interface
auto lo
iface lo inet loopback
    up ip addr add dev lo 192.168.100.9/32
    up route add -host 192.168.100.9 dev lo
    down ip addr del dev 192.168.100.9/32
    down route del -host 192.168.100.9 dev lo

allow-hotplug ens224
iface ens224 inet static
    address 192.168.100.20/28
    dns-nameservers 192.168.100.30 192.168.100.31
    gateway 192.168.100.1

Disable arp announces for LVS-DR to work on the webservers by defining a sysctl configuration file
/etc/sysctl.d/99-lvs-dr.conf

net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2

Reload sysctl

# sysctl --system

Make sure apache is installed and configured..

# apt install apache2

Step 2: Web Server #2

Configure the network interfaces by editing /etc/network/interfaces

# The loopback interface
auto lo
iface lo inet loopback
    up ip addr add dev lo 192.168.100.9/32
    up route add -host 192.168.100.9 dev lo
    down ip addr del dev 192.168.100.9/32
    down route del -host 192.168.100.9 dev lo

allow-hotplug ens224
iface ens224 inet static
    address 192.168.100.21/28
    dns-nameservers 192.168.100.30 192.168.100.31
    gateway 192.168.100.1

Disable arp announces for LVS-DR to work on the webservers by defining a sysctl configuration file
/etc/sysctl.d/99-lvs-dr.conf

net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2

Reload sysctl

# sysctl --system

Make sure apache is installed and configured..

# apt install apache2

Step 3: LoadBalancer #1

Configure the network interfaces by editing /etc/network/interfaces

# The loopback interface
allow-hotplug ens224
iface ens224 inet static
    address 192.168.100.10/28
    dns-nameservers 192.168.100.30 192.168.100.31
    gateway 192.168.100.1

Install Keepalived

# apt-install keepalived

Configure keepalived by defining the configuration file
/etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     admin@mydomain.org
   }
   notification_email_from dmz-lb-keepalived@mydomain.org
   smtp_server 192.168.100.33
   smtp_connect_timeout 30
   router_id lb-01
}

vrrp_instance VI_1_IPv4 {
    state MASTER
    interface ens224
    smtp_alert
    virtual_router_id 51
    priority 200

    unicast_src_ip 192.168.100.10
    unicast_peer {
        192.168.100.11
    }

    authentication {
        auth_type PASS
        auth_pass generate-a-pasword
   }

   advert_int 1
   virtual_ipaddress {
        192.168.100.9/28 dev ens224
   }
}

virtual_server 192.168.100.9 80 {
    delay_loop 6
    protocol TCP
    lb_algo rr
    lb_kind DR
    persistence_timeout 600

    real_server 192.168.100.20 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.100.21 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

virtual_server 192.168.100.9 443 {
    delay_loop 6
    protocol TCP
    lb_algo rr
    lb_kind DR
    persistence_timeout 600

    real_server 192.168.100.20 443 {
        weight 1
        TCP_CHECK {
            connect_port 443
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.100.21 443 {
        weight 1
        TCP_CHECK {
            connect_port 443
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

Make sure keepalived is started during system startup, and restart

# systemctl enable keepalived
# systemctl start keepalived

Check that the vip is online

# ip addr list
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens224: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:50:56:9f:be:d6 brd ff:ff:ff:ff:ff:ff
altname enp19s0
inet 192.168.100.20/28 brd 192.168.100.255 scope global ens224
valid_lft forever preferred_lft forever
inet 192.168.100.9/28 scope global secondary ens224

Check that ipvsadm is properly configured

# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 90-145-51-131.bbserv.nl:http rr persistent 600
-> 90-145-51-138.bbserv.nl:http Route 1 1 0
-> 90-145-51-139.bbserv.nl:http Route 1 0 0
TCP 90-145-51-131.bbserv.nl:http rr persistent 600
-> 90-145-51-138.bbserv.nl:http Route 1 0 0
-> 90-145-51-139.bbserv.nl:http Route 1 93 0

Step 4: LoadBalancer #2

Configure the network interfaces by editing /etc/network/interfaces

# The loopback interface
allow-hotplug ens224
iface ens224 inet static
    address 192.168.100.11/28
    dns-nameservers 192.168.100.30 192.168.100.31
    gateway 192.168.100.1

Install Keepalived

# apt-install keepalived

Configure keepalived by defining the configuration file
/etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     admin@mydomain.org
   }
   notification_email_from dmz-lb-keepalived@mydomain.org
   smtp_server 192.168.100.33
   smtp_connect_timeout 30
   router_id lb-02
}

vrrp_instance VI_1_IPv4 {
    state BACKUP
    interface ens224
    smtp_alert
    virtual_router_id 51
    priority 200

    unicast_src_ip 192.168.100.11
    unicast_peer {
        192.168.100.10
    }

    authentication {
        auth_type PASS
        auth_pass generate-a-pasword
   }

   advert_int 1
   virtual_ipaddress {
        192.168.100.9/28 dev ens224
   }
}

virtual_server 192.168.100.9 80 {
    delay_loop 6
    protocol TCP
    lb_algo rr
    lb_kind DR
    persistence_timeout 600

    real_server 192.168.100.20 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.100.21 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

virtual_server 192.168.100.9 443 {
    delay_loop 6
    protocol TCP
    lb_algo rr
    lb_kind DR
    persistence_timeout 600

    real_server 192.168.100.20 443 {
        weight 1
        TCP_CHECK {
            connect_port 443
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.100.21 443 {
        weight 1
        TCP_CHECK {
            connect_port 443
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

Make sure keepalived is started during system startup, and restart

# systemctl enable keepalived
# systemctl start keepalived

Check that ipvsadm is properly configured

# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 90-145-51-131.bbserv.nl:http rr persistent 600
-> 90-145-51-138.bbserv.nl:http Route 1 1 0
-> 90-145-51-139.bbserv.nl:http Route 1 0 0
TCP 90-145-51-131.bbserv.nl:http rr persistent 600
-> 90-145-51-138.bbserv.nl:http Route 1 0 0
-> 90-145-51-139.bbserv.nl:http Route 1 93 0