From 180c82376f324e5c68c05421b3265b65e5181918 Mon Sep 17 00:00:00 2001
From: Ville Granroth <ville.granroth@gmail.com>
Date: Wed, 19 May 2021 17:49:55 +0300
Subject: [PATCH] First push

---
 qemu.d/cad/prepare/begin/alloc_hugepages.sh  | 28 +++++++++++
 qemu.d/cad/release/end/dealloc_hugepages.sh  |  2 +
 qemu.d/game/prepare/begin/alloc_hugepages.sh | 53 ++++++++++++++++++++
 qemu.d/game/release/end/dealloc_hugepages.sh |  5 ++
 qemu.d/win/prepare/begin/alloc_hugepages.sh  | 29 +++++++++++
 qemu.d/win/release/end/dealloc_hugepages.sh  |  2 +
 6 files changed, 119 insertions(+)
 create mode 100755 qemu.d/cad/prepare/begin/alloc_hugepages.sh
 create mode 100755 qemu.d/cad/release/end/dealloc_hugepages.sh
 create mode 100755 qemu.d/game/prepare/begin/alloc_hugepages.sh
 create mode 100755 qemu.d/game/release/end/dealloc_hugepages.sh
 create mode 100755 qemu.d/win/prepare/begin/alloc_hugepages.sh
 create mode 100755 qemu.d/win/release/end/dealloc_hugepages.sh

diff --git a/qemu.d/cad/prepare/begin/alloc_hugepages.sh b/qemu.d/cad/prepare/begin/alloc_hugepages.sh
new file mode 100755
index 0000000..38f4a1b
--- /dev/null
+++ b/qemu.d/cad/prepare/begin/alloc_hugepages.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+MEMORY=24576
+## Calculate number of hugepages to allocate from memory (in MB)
+HUGEPAGES="$(($MEMORY/$(($(grep Hugepagesize /proc/meminfo | awk '{print $2}')/1024))))"
+## Node to allocate on (node1 for this VM)
+NODESPATH=/sys/devices/system/node/node1
+
+echo "Allocating hugepages..."
+echo $HUGEPAGES > $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages
+ALLOC_PAGES=$(cat $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages)
+
+TRIES=0
+while (( $ALLOC_PAGES != $HUGEPAGES && $TRIES < 1000 ))
+do
+    echo 1 > /proc/sys/vm/compact_memory            ## defrag ram
+    echo $HUGEPAGES > $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages
+    ALLOC_PAGES=$(cat $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages)
+    echo "Succesfully allocated $ALLOC_PAGES / $HUGEPAGES"
+    let TRIES+=1
+done
+
+if [ "$ALLOC_PAGES" -ne "$HUGEPAGES" ]
+then
+    echo "Not able to allocate all hugepages. Reverting..."
+    echo 0 > $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages
+    exit 1
+fi
diff --git a/qemu.d/cad/release/end/dealloc_hugepages.sh b/qemu.d/cad/release/end/dealloc_hugepages.sh
new file mode 100755
index 0000000..009eb52
--- /dev/null
+++ b/qemu.d/cad/release/end/dealloc_hugepages.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+echo 0 > /proc/sys/vm/nr_hugepages
diff --git a/qemu.d/game/prepare/begin/alloc_hugepages.sh b/qemu.d/game/prepare/begin/alloc_hugepages.sh
new file mode 100755
index 0000000..9bcf736
--- /dev/null
+++ b/qemu.d/game/prepare/begin/alloc_hugepages.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+
+MEMORY=32768
+SSDADDRESS=0000:04:00.0
+
+## Calculate number of hugepages to allocate from memory (in MB)
+HUGEPAGES="$(($MEMORY/$(($(grep Hugepagesize /proc/meminfo | awk '{print $2}')/1024))))"
+## Node to allocate on (node1 for this VM)
+NODESPATH=/sys/devices/system/node/node1
+
+# BIND SSD
+echo $SSDADDRESS > /sys/bus/pci/devices/$SSDADDRESS/driver/unbind
+echo $SSDADDRESS > /sys/bus/pci/drivers/vfio-pci/bind
+
+echo "Allocating hugepages..."
+echo $HUGEPAGES > $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages
+ALLOC_PAGES=$(cat $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages)
+
+TRIES=0
+while (( $ALLOC_PAGES != $HUGEPAGES && $TRIES < 10000 ))
+do
+    echo 1 > /proc/sys/vm/compact_memory            ## defrag ram
+    echo $HUGEPAGES > $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages
+    ALLOC_PAGES=$(cat $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages)
+    echo "Succesfully allocated $ALLOC_PAGES / $HUGEPAGES"
+    let TRIES+=1
+done
+
+if [ "$ALLOC_PAGES" -ne "$HUGEPAGES" ]
+then
+    echo "Not able to allocate all hugepages. Reverting..."
+    echo 0 > $NODESPATH/hugepages/hugepages-2048kB/nr_hugepages
+    exit 1
+fi
+
+# libvirt likes to create a cpuset called "machine" and leave it behind after closing
+# this confuses cset. remove it just in case
+rmdir /sys/fs/cgroup/cpuset/machine
+
+# reserve cpu 0,4 (core 0) for the host and give 1-3,5-7 (core 1-3) to the guest/vm
+cset shield --kthread on --cpu 14-27,42-55
+
+# cset reserves the cpusets for exclusive use but libvirt also wants to create
+# new cpusets containing the same cpus. Remove the exclusive flag.
+echo 0 > /sys/fs/cgroup/cpuset/system/cpuset.cpu_exclusive
+echo 0 > /sys/fs/cgroup/cpuset/user/cpuset.cpu_exclusive
+
+# The kernel's dirty page writeback mechanism uses kthread workers. They introduce
+# massive arbitrary latencies when doing disk writes on the host and aren't
+# migrated by cset. Restrict the workqueue to use only cpu 0.
+
+echo 1 > /sys/devices/virtual/workqueue/cpumask
diff --git a/qemu.d/game/release/end/dealloc_hugepages.sh b/qemu.d/game/release/end/dealloc_hugepages.sh
new file mode 100755
index 0000000..4167401
--- /dev/null
+++ b/qemu.d/game/release/end/dealloc_hugepages.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+sysctl vm.stat_interval=1
+sysctl -w kernel.watchdog=1
+echo 1 > /sys/bus/workqueue/devices/writeback/numa
+echo 0 > /proc/sys/vm/nr_hugepages
diff --git a/qemu.d/win/prepare/begin/alloc_hugepages.sh b/qemu.d/win/prepare/begin/alloc_hugepages.sh
new file mode 100755
index 0000000..8f3a699
--- /dev/null
+++ b/qemu.d/win/prepare/begin/alloc_hugepages.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+MEMORY=98304
+
+## Calculate number of hugepages to allocate from memory (in MB)
+HUGEPAGES="$(($MEMORY/$(($(grep Hugepagesize /proc/meminfo | awk '{print $2}')/1024))))"
+
+echo "Allocating hugepages..."
+echo $HUGEPAGES > /proc/sys/vm/nr_hugepages
+ALLOC_PAGES=$(cat /proc/sys/vm/nr_hugepages)
+
+TRIES=0
+while (( $ALLOC_PAGES != $HUGEPAGES && $TRIES < 1000 ))
+do
+    echo 1 > /proc/sys/vm/compact_memory            ## defrag ram
+    echo $HUGEPAGES > /proc/sys/vm/nr_hugepages
+    ALLOC_PAGES=$(cat /proc/sys/vm/nr_hugepages)
+    echo "Succesfully allocated $ALLOC_PAGES / $HUGEPAGES"
+    let TRIES+=1
+done
+
+if [ "$ALLOC_PAGES" -ne "$HUGEPAGES" ]
+then
+    echo "Not able to allocate all hugepages. Reverting..."
+    echo 0 > /proc/sys/vm/nr_hugepages
+    exit 1
+fi
+
+
diff --git a/qemu.d/win/release/end/dealloc_hugepages.sh b/qemu.d/win/release/end/dealloc_hugepages.sh
new file mode 100755
index 0000000..009eb52
--- /dev/null
+++ b/qemu.d/win/release/end/dealloc_hugepages.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+echo 0 > /proc/sys/vm/nr_hugepages