From cd2ec90a5b9c8e3c4fb9045459ab1bb5f2fa84ec Mon Sep 17 00:00:00 2001
From: James Antill <james@and.org>
Date: Tue, 5 Aug 2025 14:48:37 -0400
Subject: [PATCH] playbooks/updates-uptimes: Add distro. to data and script to
 view data.

Signed-off-by: James Antill <james@and.org>
---
 ...generate-updates-uptimes-per-host-file.yml |   2 +-
 scripts/updates-uptime-cmd.py                 | 403 ++++++++++++++++++
 2 files changed, 404 insertions(+), 1 deletion(-)
 create mode 100755 scripts/updates-uptime-cmd.py

diff --git a/playbooks/generate-updates-uptimes-per-host-file.yml b/playbooks/generate-updates-uptimes-per-host-file.yml
index 09741c28e2..5ecc432247 100644
--- a/playbooks/generate-updates-uptimes-per-host-file.yml
+++ b/playbooks/generate-updates-uptimes-per-host-file.yml
@@ -31,7 +31,7 @@
   - name: Generate the Upgrade+uptime report
     ansible.builtin.lineinfile:
       regexp: '^{{inventory_hostname}} '
-      line: "{{inventory_hostname}} {{pkgoutput.results|length}} {{ansible_uptime_seconds}} {{ansible_date_time['date']}}"
+      line: "{{inventory_hostname}} {{pkgoutput.results|length}} {{ansible_uptime_seconds}} {{ansible_date_time['date']}} {{ansible_distribution}} {{ansible_distribution_version}}"
       path: /var/log/ansible-list-updates-uptime.txt
       create: yes
     delegate_to: localhost
diff --git a/scripts/updates-uptime-cmd.py b/scripts/updates-uptime-cmd.py
new file mode 100755
index 0000000000..0abbc606ff
--- /dev/null
+++ b/scripts/updates-uptime-cmd.py
@@ -0,0 +1,403 @@
+#! /usr/bin/python3
+
+#  Create/view a "txt" file using the ansible playbook
+# "generate-updates-uptimes-per-host-file.yml" which records the number of rpms
+# available to be upgraded for ansible hosts, and the uptime of those hosts.
+#  This is very helpful when doing upgrade+reboot runs, as we can easily see
+# what has/hasn't been upgraded and/or rebooted.
+
+# $0 update       = create the file and/or do backups
+# $0 diff [x] [y] = see the difference between the current state and backups
+# $0 uptime [x]   = see the current state, can be filtered for uptime >= x
+# $0 info [x]     = see the current state, in long form, can be filtered by name
+# $0 host [x]     = see the current state of a host(s), can be filtered by name
+# $0 list [x]     = see the current state, can be filtered by name
+# $0 backups      = see backups
+# $0 backups-keep = clenaup old backups
+# $0 stats [x]    = see stats, can specify a backup
+
+# Examples:
+# $0 update ... run it daily or so as root.
+# $0 diff ... see what changed.
+# $0 list '*.stg.*' ... see what staging looks like.
+# $0 list '*copr*' ... see what copr looks like.
+# $0 backups-keep 4 ... keep four days of backups (including today)
+# $0 uptime 1d ... see what hasn't been rebooted in the last 24 hours.
+# $0 uptime 25w ... see what hasn't been rebooted in too damn long
+
+import os
+import sys
+
+import fnmatch
+import glob
+import time
+
+path = "/var/log/"
+fname = path + "ansible-list-updates-uptime.txt"
+fname_today = fname + '.' + time.strftime("%Y-%m-%d")
+
+backups = sorted(x.removeprefix(fname + '.') for x in glob.glob(fname + '.*'))
+
+cmd = "diff"
+if len(sys.argv) >= 2:
+    if sys.argv[1] in ("backups", "backups-keep", "diff", "diff-u", "help", "host", "info", "list", "stats", "update", "update-fast", "update-daily", "uptime",):
+        cmd = sys.argv.pop(1)
+
+_tm_d = {'d' : 60*60*24, 'h' : 60*60, 'm' : 60, 's' : 1,
+         'w' : 60*60*24*7,
+         'q' : 60*60*24*7*13}
+def parse_duration(seconds):
+    if seconds is None:
+        return None
+    if seconds.isdigit():
+        return int(seconds)
+
+    ret = 0
+    for mark in ('w', 'd', 'h', 'm', 's'):
+        pos = seconds.find(mark)
+        if pos == -1:
+            continue
+        val = seconds[:pos]
+        seconds = seconds[pos+1:]
+        if not val.isdigit():
+            # dbg("!isdigit", val)
+            return None
+        ret += _tm_d[mark]*int(val)
+    if seconds.isdigit():
+        ret += int(seconds)
+    elif seconds != '':
+        # dbg("!empty", seconds)
+        return None
+
+    return ret
+
+
+def _add_dur(dur, ret, nummod, suffix, static=False):
+    mod = dur % nummod
+    dur = dur // nummod
+    if mod > 0 or (static and dur > 0):
+        ret.append(suffix)
+        if static and dur > 0:
+            ret.append("%0*d" % (len(str(nummod)), mod))
+        else:
+            ret.append(str(mod))
+    return dur
+
+def format_duration(seconds, static=False):
+    if seconds is None:
+        seconds = 0
+    dur = int(seconds)
+
+    ret = []
+    dur = _add_dur(dur, ret, 60, "s", static=static)
+    dur = _add_dur(dur, ret, 60, "m", static=static)
+    dur = _add_dur(dur, ret, 24, "h", static=static)
+    dur = _add_dur(dur, ret,  7, "d", static=static)
+    if dur > 0:
+        ret.append("w")
+        ret.append(str(dur))
+    return "".join(reversed(ret))
+
+
+cmp_arg = False
+cmp = None
+def _cmp_arg():
+    global cmp
+    global cmp_arg
+
+    if len(sys.argv) < 2:
+        cmp = backups[-1] # Most recent
+    elif sys.argv[1] not in backups:
+        _usage()
+        print("Backups:", ", ".join(backups))
+        sys.exit(1)
+    else:
+        cmp = sys.argv[1]
+        cmp_arg = True
+
+def line2data(line):
+    name, rpms, uptime, date = line.split(' ', 3)
+    osname = "Unknown"
+    osvers = "0"
+    osinfo = "Unknown/?"
+    if ' ' in date:
+        date, osname, osvers = date.split(' ', 2)
+        osinfo = osname + '/' + osvers
+        if osname == 'CentOS':
+            osinfo = 'EL/' + osvers
+        if osname == 'Redhat':
+            osinfo = 'EL/' + osvers
+        if osname == 'Fedora':
+            osinfo = 'F/' + osvers
+
+    updates = name + ' ' + rpms
+
+    rpms = int(rpms)
+    uptime = int(uptime)
+
+    return locals()
+
+def fname2lines(fname):
+    return [x.strip() for x in open(fname).readlines()]
+
+def bfname2lines(b):
+    return fname2lines(fname + '.' + b)
+
+def fname1():
+    if cmp_arg:
+        return bfname2lines(cmp)
+    return fname2lines(fname)
+
+def _usage():
+  prog = "updates+uptime"
+  if sys.argv:
+      prog = os.path.basename(sys.argv[0])
+  pl = " " * len(prog)
+  print("""
+ Usage: %s <cmd>
+   Cmds:
+        help
+            = This message.
+
+        backups
+            = Show backups.
+        backups-trim [days]
+            = Cleanup old backups.
+
+        diff         [backup1] [backup2]
+        diff-u       [backup1] [backup2]
+            = See the difference between the current state and backups.
+
+        host         [host*] [backup]
+            = See the current state of a host(s), can be filtered by name.
+        info         [host*] [backup]
+            = See the current state, in long form, can be filtered by name.
+        list         [host*] [backup]
+            = See the current state, can be filtered by name.
+
+        stats        [backup]
+            = Show stats.
+
+        update
+            = Create the file and/or do backups.
+
+        uptime       [duration] [backup]
+            = See the current state, can be filtered for uptime >= duration.
+""" % (prog,))
+
+if cmd == "help":
+  _usage()
+
+if cmd == "backups":
+  print("Backups:")
+  for backup in backups:
+      print('', backup, len(bfname2lines(backup)))
+
+if cmd == "backups-keep":
+  keep = 8
+  if len(sys.argv) >= 2:
+      keep = int(sys.argv.pop(1))
+  if keep <= 0:
+      _usage()
+      sys.exit(1)
+  while keep < len(backups):
+      # We just keep the newest N
+      fn = fname + '.' + backups.pop()
+      os.unlink(fn)
+
+if cmd == "update":
+  if not os.path.exists(fname):
+    cmd = "update-daily" # This does the sorting etc.
+  elif not os.path.exists(fname_today):
+    cmd = "update-daily"
+  else:
+    mtime = os.path.getmtime(fname)
+    if (time.now() - mtime) <= (60*60*8): # 8 hours.
+        cmd = "update-fast"
+
+if cmd == "update": # Get the latest uptime.
+  os.chdir("/srv/web/infra/ansible/playbooks")
+  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates --flush-cache")
+if cmd == "update-fast": # Use ansible FACT cache for uptime.
+  os.chdir("/srv/web/infra/ansible/playbooks")
+  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates")
+if cmd == "update-daily": # Also create backup file.
+  os.chdir("/srv/web/infra/ansible/playbooks")
+  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml --flush-cache")
+
+if cmd == "stats":
+  _cmp_arg()
+  data = fname1()
+  osinfo = {}
+  print("Hosts:", len(data))
+  most = 0, None
+  updates = 0
+  awake = 0
+  awakest = 0, None
+  for line in data:
+    d2 = line2data(line)
+
+    if d2['osname'] not in osinfo:
+        osinfo[d2['osname']] = 0
+    osinfo[d2['osname']] += 1
+    if d2['osinfo'] not in osinfo:
+        osinfo[d2['osinfo']] = 0
+    osinfo[d2['osinfo']] += 1
+    updates += d2['rpms']
+    if d2['rpms'] >= most[0]:
+        most = d2['rpms'], d2['name']
+
+    awake += d2['uptime']
+    if d2['uptime'] >= awakest[0]:
+        awakest = d2['uptime'], d2['name']
+  for osi in sorted(osinfo):
+    print("  Hosts:", osi, osinfo[osi])
+  print("Updates:", updates)
+  print("  Most:", most[1], most[0])
+  print("Uptime:", format_duration(awake))
+  print("  Most:", awakest[1], format_duration(awakest[0]))
+
+def _print_info(host, lines):
+  hosts = []
+  for x in lines:
+      x = line2data(x)
+      if fnmatch.fnmatch(x['name'], host):
+          hosts.append(x)
+  if not hosts:
+      print("Not found host:", host)
+      sys.exit(2)
+  for host in hosts:
+    print("Host:", host['name'])
+    print("     OS:", host['osinfo'], host['osname'], host['osvers'])
+    print("   Rpms:", host['rpms'])
+    print(" Uptime:", format_duration(host['uptime']))
+    print("   Date:", host['date'])
+
+if cmd in ("host", "info"):
+  if cmd == "host":
+      host = "batcave*"
+  else:
+      host = "*"
+  if len(sys.argv) >= 2:
+      host = sys.argv.pop(1)
+  if len(sys.argv) >= 2 and sys.argv[1] == "all":
+      for b in backups:
+          print("Backup:", b)
+          _print_info(bfname2lines(b))
+      sys.argv = [sys.argv[0]]
+      print("Main:")
+  _cmp_arg()
+  _print_info(host, fname1())
+
+_max_len_name  = 0
+_max_len_rpms  = 0
+_max_len_updur = 0
+_max_len_date  = 0 # 4+1+2+1+2
+def _max_update(lines):
+    global _max_len_name
+    global _max_len_rpms
+    global _max_len_updur
+    global _max_len_date
+
+    for line in lines:
+        data = line2data(line)
+        if len(data['name']) > _max_len_name:
+            _max_len_name = len(data['name'])
+        if len(str(data['rpms'])) > _max_len_rpms:
+            _max_len_rpms = len(str(data['rpms']))
+        updur_len = len(format_duration(data['uptime'], static=True))
+        if updur_len > _max_len_updur:
+            _max_len_updur = updur_len
+        if len(data['date']) > _max_len_date:
+            _max_len_date = len(data['date'])
+
+def _print_line(prefix, data):
+    print("%s%-*s %*u %*s %*s %s" % (prefix,
+          _max_len_name, data['name'],
+          _max_len_rpms, data['rpms'],
+          _max_len_updur, format_duration(data['uptime'], static=True),
+          _max_len_date, data['date'], data['osinfo']))
+
+if cmd == "list":
+  host = "*"
+  if len(sys.argv) >= 2:
+      host = sys.argv.pop(1)
+
+  _cmp_arg()
+  lines = fname1()
+  _max_update(lines)
+  for line in lines:
+      d1 = line2data(line)
+      if not fnmatch.fnmatch(d1['name'], host):
+          continue
+      _print_line('', d1)
+
+if cmd == "uptime":
+  age = 0
+  if len(sys.argv) >= 2:
+      age = parse_duration(sys.argv.pop(1))
+
+  _cmp_arg()
+  lines = fname1()
+  _max_update(lines)
+  for line in lines:
+      d1 = line2data(line)
+      if age > d1['uptime']:
+          continue
+      _print_line('', d1)
+
+if cmd in ("diff", "diff-u"):
+  _cmp_arg()
+  fn1 = fname + '.' + cmp
+  fn2 = fname
+  data1 = fname2lines(fn1)
+  if len(sys.argv) >= 2 and sys.argv[1] in backups:
+    # Doing a diff. between two backups...
+    fn2 = fname + '.' + sys.argv[1]
+  data2 = fname2lines(fn2)
+  print("diff %s %s" % (fn1, fn2), file=sys.stderr)
+  _max_update(data1)
+  _max_update(data2)
+  while len(data1) > 0 or len(data2) > 0:
+    if len(data1) <= 0:
+        _print_line('+', line2data(data2[0]))
+        data2.pop(0)
+        continue
+    if len(data2) <= 0:
+        _print_line('-', line2data(data1[0]))
+        data1.pop(0)
+        continue
+
+    d1 = line2data(data1[0])
+    d2 = line2data(data2[0])
+
+    if d1['name'] < d2['name']:
+        _print_line('-', d1)
+        data1.pop(0)
+        continue
+
+    if d1['name'] > d2['name']:
+        _print_line('+', d2)
+        data2.pop(0)
+        continue
+
+    if d1['updates'] == d2['updates']:
+        _print_line(' ', d2)
+        data1.pop(0)
+        data2.pop(0)
+        continue
+
+    if cmd == "diff-u":
+        _print_line('-', d1)
+        data1.pop(0)
+        _print_line('+', d2)
+        data2.pop(0)
+        continue
+
+    # diff
+    data1.pop(0)
+    _print_line('!', d2)
+    data2.pop(0)
+
+    continue
+