openqa/worker: bump load average threshold for big worker hosts

This is a new feature in openQA that prevents worker hosts
picking up new jobs if their load average is above a certain
threshold. It defaults to 40. Our big worker hosts tend to run
above this, so let's bump it on those.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

inventory/host_vars/openqa-a64-worker04.iad2.fedoraproject.org

@@ -60,6 +60,8 @@ openqa_rngd: true
 openqa_tap_iface: eth2
 # this is a powerful machine, can handle more openQA workers
 openqa_workers: 35
+# also means we need to bump the load average threshold
+openqa_critical_threshold: 60
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 # firewall ports for server->worker websockets connections
 # this port is 'QEMUPORT plus 1'

inventory/host_vars/openqa-x86-worker01.iad2.fedoraproject.org

@@ -63,6 +63,8 @@ openqa_rngd: true
 openqa_tap_iface: em1
 # this is a powerful machine, can handle more openQA workers
 openqa_workers: 30
+# also means we need to bump the load average threshold
+openqa_critical_threshold: 50
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 # firewall ports for server->worker websockets connections
 # this port is 'QEMUPORT plus 1'

inventory/host_vars/openqa-x86-worker02.iad2.fedoraproject.org

@@ -61,6 +61,8 @@ openqa_rngd: true
 openqa_tap_iface: em1
 # this is a powerful machine, can handle more openQA workers
 openqa_workers: 30
+# also means we need to bump the load average threshold
+openqa_critical_threshold: 50
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 # firewall ports for server->worker websockets connections
 # this port is 'QEMUPORT plus 1'

inventory/host_vars/openqa-x86-worker04.iad2.fedoraproject.org

@@ -63,6 +63,8 @@ openqa_rngd: true
 openqa_tap_iface: em1
 # this is a powerful machine, can handle more openQA workers
 openqa_workers: 30
+# also means we need to bump the load average threshold
+openqa_critical_threshold: 50
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 # firewall ports for server->worker websockets connections
 # this port is 'QEMUPORT plus 1'

inventory/host_vars/openqa-x86-worker05.iad2.fedoraproject.org

@@ -61,6 +61,7 @@ openqa_rngd: true
 openqa_tap_iface: eno1
 # xeon e5-2683v4, 32 physical CPUs
 openqa_workers: 30
+openqa_critical_threshold: 50
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 # firewall ports for server->worker websockets connections
 # this port is 'QEMUPORT plus 1'

inventory/host_vars/openqa-x86-worker06.iad2.fedoraproject.org

@@ -60,6 +60,8 @@ nrpe_procs_warn: 1400
 openqa_rngd: true
 # this is a powerful machine, can handle more openQA workers
 openqa_workers: 30
+# also means we need to bump the load average threshold
+openqa_critical_threshold: 50
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 # firewall ports for server->worker websockets connections
 # this port is 'QEMUPORT plus 1'

roles/openqa/worker/tasks/main.yml

@@ -49,13 +49,17 @@
 # - openqa_worker_class
 ##      string - custom WORKER_CLASS value for workers.ini
 # - openqa_static_uid
-##       int - a static ID for the geekotest group if desired. this is useful
-##             for NFS mounting openQA data files. The _openqa_worker user,
-##             which os-autoinst runs as, will be added to this group. The
-##             idea is that the same group with the same GID exists on the
-##             NFS server and is the group of the shared asset directories,
-##             so os-autoinst can write to the shared asset directories,
-##             which it needs to do when uncompressing compressed disk assets
+##      int - a static ID for the geekotest group if desired. this is useful
+##            for NFS mounting openQA data files. The _openqa_worker user,
+##            which os-autoinst runs as, will be added to this group. The
+##            idea is that the same group with the same GID exists on the
+##            NFS server and is the group of the shared asset directories,
+##            so os-autoinst can write to the shared asset directories,
+##            which it needs to do when uncompressing compressed disk assets
+# - openqa_critical_threshold
+##      string - custom CRITICAL_LOAD_AVG_THRESHOLD value for workers.ini.
+##               If worker host is above this load average, it will not pick
+##               up new jobs
 
 - name: Remove old scratch repo directory
   file: path=/var/tmp/scratchrepo state=absent

roles/openqa/worker/templates/workers.ini.j2

@@ -11,3 +11,6 @@ WORKER_CLASS = {{ openqa_tap }},tpm,qemu_aarch64,qemu_arm
 WORKER_CLASS = {{ openqa_tap }},tpm,qemu_x86_64,qemu_i686,qemu_i586
 {% endif %}
 {% endif %}
+{% if openqa_critical_threshold is defined %}
+CRITICAL_LOAD_AVG_THRESHOLD = {{ openqa_critical_threshold }}
+{% endif %}