-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbare-metal-setup
73 lines (58 loc) · 7.61 KB
/
bare-metal-setup
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
### Setup xrdp (for deb packages refer to https://github.com/nbritton/mcp_lab_tools/blob/master/xrdp_xenial_howto.sh)
dpkg -i xrdp_0.9.9-1ubuntu1_amd64.deb;
dpkg -i xorgxrdp_0.2.9-1ubuntu1_amd64.deb;
adduser xrdp ssl-cert;
systemctl restart xrdp;
### Install Dell OpenManage:
echo 'deb http://linux.dell.com/repo/community/openmanage/911/xenial xenial main' | sudo tee -a /etc/apt/sources.list.d/linux.dell.com.sources.list;
gpg --keyserver ha.pool.sks-keyservers.net --recv-key 1285491434D8786F;
gpg -a --export 1285491434D8786F | sudo apt-key add -;
apt update && apt -y install srvadmin-base srvadmin-webserver srvadmin-idracadm7 syscfg raidcfg dcism
### Enable SR-IOV device passthrough
sed -i 's/GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt /' /etc/default/grub;
update-grub;
### Setup SR-IOV and hypervisor passthrough on Intel igb devices:
sed -i 's/GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="pci=assign-busses /' /etc/default/grub;
update-grub;
echo -e "options igb max_vfs=7\nblacklist igbvf" > /etc/modprobe.d/igb.conf;
update-initramfs -u;
### Enable KVM virtualization:
echo "options kvm-intel nested=y" >> /etc/modprobe.d/kvm.conf;
apt -y install bridge-utils cpu-checker libguestfs-rescue libguestfs-tools linux-tools-common linux-tools-generic numactl numad qemu-kvm qemu-system-common qemu-system-x86 qemu-utils virtinst virt-manager libvirt-bin spice-client-gtk gir1.2-spice-client-gtk-3.0;
add-apt-repository ppa:jonathonf/zfs;
apt update;
apt upgrade;
apt install zfs-dkms -y
zfs create data/images;
zfs set mountpoint=/var/lib/libvirt/images data/images;
qemu-img create -f qcow2 /var/lib/libvirt/images/kvm01.exabit.io.images.qcow2 5T;
qemu-img create -f qcow2 /var/lib/libvirt/images/kvm02.exabit.io.images.qcow2 5T;
qemu-img create -f qcow2 /var/lib/libvirt/images/kvm03.exabit.io.images.qcow2 5T;
# Change --host-device to the correct device, on a Dell R820, R830, and/or R840 the kvm node should be pinned to NUMA domain 2 or 3, as 0 and 1 are reserved for SR-IOV and Ceph due to CPU sockets 3 and 4 having no direct physical connection to the PCIe slots.
name="kvm01.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 131072 --vcpus 16 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --disk /var/lib/libvirt/images/${name}.images.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62 --numatune=2,mode=preferred --host-device pci_0000_01_01_1 --host-device pci_0000_01_02_1
name="kvm02.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 131072 --vcpus 16 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --disk /var/lib/libvirt/images/${name}.images.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62 --numatune=2,mode=preferred --host-device pci_0000_01_01_1 --host-device pci_0000_01_03_1
name="kvm03.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 131072 --vcpus 16 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --disk /var/lib/libvirt/images/${name}.images.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62 --numatune=2,mode=preferred --host-device pci_0000_01_01_1 --host-device pci_0000_01_03_1
# Find NUMA node of PERC H710 Storage Controller
cat /sys/devices/pci0000:00/0000:00:$(lspci | grep MegaRAID | awk -F: '{printf $1}')*/numa_node;
# Set to NUMA node which has the RAID storage controller
name="osd01.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 32768 --vcpus 16 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60 --numatune=0,mode=preferred --host-device pci_0000_01_01_2 --host-device pci_0000_01_02_2 --host-device pci_0000_03_00_2 --host-device pci_0000_03_01_2
name="osd02.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 32768 --vcpus 16 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60 --numatune=0,mode=preferred --host-device pci_0000_01_01_2 --host-device pci_0000_01_03_2 --host-device pci_0000_03_00_2 --host-device pci_0000_03_01_2
name="osd03.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 32768 --vcpus 16 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60 --numatune=0,mode=preferred --host-device pci_0000_01_01_2 --host-device pci_0000_01_03_2 --host-device pci_0000_03_00_2 --host-device pci_0000_03_01_2
# Create ceph backing devices, qcow2 is on average faster then zvols. 35TB raw, 30TB useable, thinly over-provisioned to 48TiB thanks to ZFS on-the-fly compression, compression factor 1.75x
# OSD disks are 1.5 * CPU cores. 8 Real cores = 16 threads = 12 OSD disk.
for i in $(seq -w 1 12); do qemu-img create -f qcow2 /ceph/ceph${i}.qcow2 4T; done
name="osd01.exabit.io";
for device in {b..m}; do echo "$device"; done | awk '{printf $1" "; printf ("%02d\n",NR)}' | while read device number; do virsh attach-disk ${name} --source /ceph/ceph${number}.qcow2 --persistent --targetbus virtio --subdriver qcow2 --target vd${device}; done
# Setup gtw nodes
name="gtw01.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 8192 --vcpus 4 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63 --numatune=3,mode=preferred --host-device pci_0000_01_01_3 --host-device pci_0000_01_02_3 --host-device pci_0000_01_09_3 --host-device pci_0000_01_0a_3
name="gtw02.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 8192 --vcpus 4 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63 --numatune=3,mode=preferred --host-device pci_0000_01_01_3 --host-device pci_0000_01_03_3 --host-device pci_0000_01_09_3 --host-device pci_0000_01_0b_3
name="gtw03.exabit.io"; virt-install --machine q35 --import --name ${name} --memory 8192 --vcpus 4 --cpu host-model-only --video qxl --sound none --disk /var/lib/libvirt/images/${name}.qcow2 --boot hd --os-variant ubuntu16.04 --autostart --noautoconsole --network none --cpuset=3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63 --numatune=3,mode=preferred --host-device pci_0000_01_01_3 --host-device pci_0000_01_03_3 --host-device pci_0000_01_09_3 --host-device pci_0000_01_0b_3
# 1,000 Hz low latency kernel for better interrupt servicing
apt -y install --install-recommends linux-lowlatency-hwe-16.04;
apt -y remove linux-*-4.15.0-*-generic;
update-grub;
update-initramfs -u -k all;
mlxconfig -d /dev/mst/mt4099_pciconf0 set SRIOV_EN=1 NUM_OF_VFS=26 LINK_TYPE_P1=2 LINK_TYPE_P2=2;
mlxconfig -d /dev/mst/mt4099_pciconf1 set SRIOV_EN=1 NUM_OF_VFS=26 LINK_TYPE_P1=2 LINK_TYPE_P2=2;
echo "options mlx4_core num_vfs=13 probe_vf=0 port_type_array=2,2 msi_x=1 mlx4_en_only_mode=1" > /etc/modprobe.d/mlx4_core.conf;
update-initramfs -u;