fedora-infra_ansible/scripts/updates-uptime-cmd.py

#! /usr/bin/python3

#  Create/view a "txt" file using the ansible playbook
# "generate-updates-uptimes-per-host-file.yml" which records the number of rpms
# available to be upgraded for ansible hosts, and the uptime of those hosts.
#  This is very helpful when doing upgrade+reboot runs, as we can easily see
# what has/hasn't been upgraded and/or rebooted.

# $0 update       = create the file and/or do backups
# $0 diff [x] [y] = see the difference between the current state and backups
# $0 uptime [x]   = see the current state, can be filtered for uptime >= x
# $0 info [x]     = see the current state, in long form, can be filtered by name
# $0 host [x]     = see the current state of a host(s), can be filtered by name
# $0 list [x]     = see the current state, can be filtered by name
# $0 backups      = see backups
# $0 backups-keep = clenaup old backups
# $0 stats [x]    = see stats, can specify a backup

# Examples:
# $0 update ... run it daily or so as root.
# $0 diff ... see what changed.
# $0 list '*.stg.*' ... see what staging looks like.
# $0 list '*copr*' ... see what copr looks like.
# $0 backups-keep 4 ... keep four days of backups (including today)
# $0 uptime 1d ... see what hasn't been rebooted in the last 24 hours.
# $0 uptime 25w ... see what hasn't been rebooted in too damn long

import os
import sys

import fnmatch
import glob
import shutil
import time

#  If we try to update this seconds since the file changed, flush the
# ansible FACT cache.
conf_dur_flush_cache = (60*60*8)

#  How many hosts to show in tier 4 updates/uptimes...
conf_stat_4_hosts = 4

#  Remove suffix noise in names.
conf_suffix_dns_replace = {
  '.fedoraproject.org'    : '.<FP>.org',
  '.fedorainfracloud.org' : '.<FIC>.org',
}
_suffix_dns_replace = {}
for x in conf_suffix_dns_replace:
    _suffix_dns_replace[x] = False

#  Dir. where we put, and look for, the files...
conf_path = "/var/log/"

fname = conf_path + "ansible-list-updates-uptime.txt"
fname_today = fname + '.' + time.strftime("%Y-%m-%d")

backups = sorted(x.removeprefix(fname + '.') for x in glob.glob(fname + '.*'))

if len(sys.argv) >= 2:
    if '-v' in sys.argv:
        sys.argv.remove('-v') # In theory sys.argv[0] but meh
        conf_suffix_dns_replace = {}

cmd = "diff"
if len(sys.argv) >= 2:
    if sys.argv[1] in ("backups", "backups-keep", "diff", "diff-u", "help", "host", "info", "list", "stats", "update", "update-fast", "update-flush", "update-daily", "update-daily-refresh", "uptime",):
        cmd = sys.argv.pop(1)

_tm_d = {'d' : 60*60*24, 'h' : 60*60, 'm' : 60, 's' : 1,
         'w' : 60*60*24*7,
         'q' : 60*60*24*7*13}
def parse_duration(seconds):
    if seconds is None:
        return None
    if seconds.isdigit():
        return int(seconds)

    ret = 0
    for mark in ('w', 'd', 'h', 'm', 's'):
        pos = seconds.find(mark)
        if pos == -1:
            continue
        val = seconds[:pos]
        seconds = seconds[pos+1:]
        if not val.isdigit():
            # dbg("!isdigit", val)
            return None
        ret += _tm_d[mark]*int(val)
    if seconds.isdigit():
        ret += int(seconds)
    elif seconds != '':
        # dbg("!empty", seconds)
        return None

    return ret


def _add_dur(dur, ret, nummod, suffix, static=False):
    mod = dur % nummod
    dur = dur // nummod
    if mod > 0 or (static and dur > 0):
        ret.append(suffix)
        if static and dur > 0:
            ret.append("%0*d" % (len(str(nummod)), mod))
        else:
            ret.append(str(mod))
    return dur

def format_duration(seconds, static=False):
    if seconds is None:
        seconds = 0
    dur = int(seconds)

    ret = []
    dur = _add_dur(dur, ret, 60, "s", static=static)
    dur = _add_dur(dur, ret, 60, "m", static=static)
    dur = _add_dur(dur, ret, 24, "h", static=static)
    dur = _add_dur(dur, ret,  7, "d", static=static)
    if dur > 0:
        ret.append("w")
        ret.append(str(dur))
    return "".join(reversed(ret))


cmp_arg = False
cmp = None
def _cmp_arg():
    global cmp
    global cmp_arg

    if len(sys.argv) < 2:
        cmp = backups[-1] # Most recent
    elif sys.argv[1] not in backups:
        _usage()
        print("Backups:", ", ".join(backups))
        sys.exit(1)
    else:
        cmp = sys.argv[1]
        cmp_arg = True

def line2data(line):
    name, rpms, uptime, date = line.split(' ', 3)
    osname = "Unknown"
    osvers = "?"
    osinfo = "Unknown/?"
    osinfo_small = "?/?"
    if ' ' in date:
        date, osname, osvers = date.split(' ', 2)
        osinfo = osname + '/' + osvers
        osinfo_small = osinfo
        if osname == 'CentOS':
            osinfo_small = 'EL/' + osvers
        if osname == 'RedHat':
            osinfo_small = 'EL/' + osvers
        if osname == 'Fedora':
            osinfo_small = 'F/' + osvers

    # This is the primary key for diff.
    update_info = name + ' ' + rpms + ' ' + osinfo

    rpms = int(rpms)
    uptime = int(uptime)

    return locals()

# Sub. suffix of DNS names for UI
def _ui_name(name):
    for suffix in conf_suffix_dns_replace:
        if name.endswith(suffix):
            _suffix_dns_replace[suffix] = True
            return name[:-len(suffix)] + conf_suffix_dns_replace[suffix]
    return name

# Reset the usage after _max_update()
def _reset_ui_name():
    for suffix in sorted(_suffix_dns_replace):
        _suffix_dns_replace[suffix] = False

# Explain if we used any suffix subs.
def _explain_ui_name():
    done = False
    pre = "* NOTE:"
    for suffix in sorted(_suffix_dns_replace):
        if _suffix_dns_replace[suffix]:
            print("%s %12s = %s" % (pre,conf_suffix_dns_replace[suffix],suffix))
            pre = "      :"
            done = True
    if done:
        print("      : Use -v to show full names.")

def fname2lines(fname):
    return [x.strip() for x in open(fname).readlines()]

def bfname2lines(b):
    return fname2lines(fname + '.' + b)

def fname1():
    if cmp_arg:
        return bfname2lines(cmp)
    return fname2lines(fname)

_max_len_name  = 0
_max_len_rpms  = 0
_max_len_updur = 0
_max_len_date  = 0 # 4+1+2+1+2
_max_terminal_width = shutil.get_terminal_size().columns
if _max_terminal_width < 20:
    _max_terminal_width = 80
_max_terminal_width -= 10
def _max_update(prefix, lines):
    global _max_len_name
    global _max_len_rpms
    global _max_len_updur
    global _max_len_date

    for line in lines:
        data = line2data(line)
        name = _ui_name(data['name'])
        if len(name) > _max_len_name:
            _max_len_name = len(name)
        if len(str(data['rpms'])) > _max_len_rpms:
            _max_len_rpms = len(str(data['rpms']))
        updur_len = len(format_duration(data['uptime'], static=True))
        if updur_len > _max_len_updur:
            _max_len_updur = updur_len
        if len(data['date']) > _max_len_date:
            _max_len_date = len(data['date'])
    mw = _max_terminal_width - len(prefix)
    while _max_len_name + _max_len_rpms + _max_len_updur + _max_len_date >= mw:
        _max_len_name -= 1


def _usage():
  prog = "updates+uptime"
  if sys.argv:
      prog = os.path.basename(sys.argv[0])
  pl = " " * len(prog)
  print("""
 Usage: %s <cmd>
   Cmds:
        help
            = This message.

        backups
            = Show backups.
        backups-trim [days]
            = Cleanup old backups.

        diff         [backup1] [backup2]
            = See the difference between the current state and backups.
        diff-u       [backup1] [backup2]
            = Shows before/after instead of modified (like diff -u).

        host         [host*] [backup]
            = See the current state of a host(s), can be filtered by name.
        info         [host*] [backup]
            = See the current state, in long form, can be filtered by name.
        list         [host*] [backup]
            = See the current state, can be filtered by name.

        stats        [backup]
            = Show stats.

        update
            = Create the file and/or do backups.
        update-fast
            = Create the file.
        update-flush
            = Create the file, after flushing ansible caches.
        update-daily
            = update-flush and do backups.
        update-daily-refresh
            = update-daily with new main file.

        uptime       [duration] [backup]
            = See the current state, can be filtered for uptime >= duration.
""" % (prog,))

if cmd == "help":
  _usage()

if cmd == "backups":
  print("Backups:")
  for backup in backups:
      print('', backup, len(bfname2lines(backup)))

if cmd == "backups-keep":
  keep = 8
  if len(sys.argv) >= 2:
      keep = int(sys.argv.pop(1))
  if keep <= 0:
      _usage()
      sys.exit(1)
  while keep < len(backups):
      # We just keep the newest N
      fn = fname + '.' + backups.pop(0)
      os.unlink(fn)

if cmd == "update":
  cmd = "update-flush"
  if not os.path.exists(fname):
    cmd = "update-daily" # This does the sorting etc.
  elif not os.path.exists(fname_today):
    cmd = "update-daily"
  else:
    mtime = os.path.getmtime(fname)
    if (int(time.time()) - mtime) > conf_dur_flush_cache:
        cmd = "update-fast"

if cmd == "update-flush": # Get the latest uptime.
  os.chdir("/srv/web/infra/ansible/playbooks")
  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates --flush-cache")
if cmd == "update-fast": # Use ansible FACT cache for uptime.
  os.chdir("/srv/web/infra/ansible/playbooks")
  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates")
if cmd == "update-daily-refresh": # Also recreate the main file.
  if os.path.exists(fname):
    os.unlink(fname)
  cmd = "update-daily"
if cmd == "update-daily": # Also create backup file.
  os.chdir("/srv/web/infra/ansible/playbooks")
  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml --flush-cache")

if cmd == "stats":
  _cmp_arg()
  data = fname1()
  # Basically we have hosts/updates/uptime and we want 4 tiers of data:
  # 1. All. 2. OS name (Eg. Fedora). 3. OS name+version (Eg. Fedora 42).
  # 4. For updates/uptime a "few" hosts with the biggest numbers.
  osinfo = {'hosts' : {}, 'updates' : {}, 'uptimes' : {}}
  updates = 0 # total updates
  most = [] # Tier 4, for updates
  awake = 0 # total uptime
  awakest = [] # Tier 4, for uptime
  _max_update('', data)
  _reset_ui_name()
  for line in data:
    d2 = line2data(line)

    # Tier 2/3 hosts...
    if d2['osname'] not in osinfo['hosts']:
        osinfo['hosts'][d2['osname']] = 0
    osinfo['hosts'][d2['osname']] += 1

    if d2['osinfo'] not in osinfo['hosts']:
        osinfo['hosts'][d2['osinfo']] = 0
    osinfo['hosts'][d2['osinfo']] += 1

    updates += d2['rpms']
    # Tier 2/3 updates...
    if d2['osname'] not in osinfo['updates']:
        osinfo['updates'][d2['osname']] = 0
    osinfo['updates'][d2['osname']] += d2['rpms']

    if d2['osinfo'] not in osinfo['updates']:
        osinfo['updates'][d2['osinfo']] = 0
    osinfo['updates'][d2['osinfo']] += d2['rpms']
    # Tier 4 updates...
    most.append((d2['rpms'], d2['name']))
    most.sort()
    while len(most) > conf_stat_4_hosts:
        most.pop(0)

    awake += d2['uptime']
    # Tier 2/3 uptimes...
    if d2['osname'] not in osinfo['uptimes']:
        osinfo['uptimes'][d2['osname']] = 0
    osinfo['uptimes'][d2['osname']] += d2['uptime']

    if d2['osinfo'] not in osinfo['uptimes']:
        osinfo['uptimes'][d2['osinfo']] = 0
    osinfo['uptimes'][d2['osinfo']] += d2['uptime']
    # Tier 4 uptimes...
    awakest.append((d2['uptime'], d2['name']))
    awakest.sort()
    while len(awakest) > conf_stat_4_hosts:
        awakest.pop(0)

  print("Hosts:", len(data))
  for osi in sorted(osinfo['hosts']):
    if '/' not in osi:
        print("  %s: %s" % (osi, osinfo['hosts'][osi]))
    if '/' in osi:
        print("    %s: %s" % (osi, osinfo['hosts'][osi]))
  print("Updates:", updates)
  for osi in sorted(osinfo['updates']):
    if '/' not in osi:
        print("  %s: %s" % (osi, osinfo['updates'][osi]))
    if '/' in osi:
        print("    %s: %s" % (osi, osinfo['updates'][osi]))
  for m in most:
    print("  Most: %-*s %s" % (_max_len_name, _ui_name(m[1]), m[0]))
  print("Uptime:", format_duration(awake))
  for osi in sorted(osinfo['uptimes']):
    if '/' not in osi:
        print("  %s: %s" % (osi, format_duration(osinfo['uptimes'][osi], True)))
    if '/' in osi:
        print("    %s: %s" % (osi,format_duration(osinfo['uptimes'][osi],True)))
  for a in awakest:
    print("  Most: %-*s %s" % (_max_len_name, _ui_name(a[1]),
                               format_duration(a[0], True)))
  _explain_ui_name()

def _print_info(host, lines):
  hosts = []
  for x in lines:
      x = line2data(x)
      if fnmatch.fnmatch(x['name'], host):
          hosts.append(x)
  if not hosts:
      print("Not found host:", host)
      sys.exit(2)
  for host in hosts:
    print("Host:", host['name'])
    print("     OS:", host['osinfo'])
    print("   Rpms:", host['rpms'])
    print(" Uptime:", format_duration(host['uptime']))
    print("   Date:", host['date'])

if cmd in ("host", "info"):
  if cmd == "host":
      host = "batcave*"
  else:
      host = "*"
  if len(sys.argv) >= 2:
      host = sys.argv.pop(1)
  if len(sys.argv) >= 2 and sys.argv[1] == "all":
      for b in backups:
          print("Backup:", b)
          _print_info(host, bfname2lines(b))
      sys.argv = [sys.argv[0]]
      print("Main:")
  _cmp_arg()
  _print_info(host, fname1())

def _print_line(prefix, data):
    print("%s%-*s %*u %*s %*s %s" % (prefix,
          _max_len_name, _ui_name(data['name']),
          _max_len_rpms, data['rpms'],
          _max_len_updur, format_duration(data['uptime'], static=True),
          _max_len_date, data['date'], data['osinfo_small']))


if cmd == "list":
  host = "*"
  if len(sys.argv) >= 2:
      host = sys.argv.pop(1)

  _cmp_arg()
  lines = fname1()
  _max_update('', lines)
  _reset_ui_name()
  for line in lines:
      d1 = line2data(line)
      if not fnmatch.fnmatch(d1['name'], host):
          continue
      _print_line('', d1)
  _explain_ui_name()

if cmd == "uptime":
  age = 0
  if len(sys.argv) >= 2:
      age = parse_duration(sys.argv.pop(1))

  _cmp_arg()
  lines = fname1()
  _max_update('', lines)
  _reset_ui_name()
  for line in lines:
      d1 = line2data(line)
      if age > d1['uptime']:
          continue
      _print_line('', d1)
  _explain_ui_name()

if cmd in ("diff", "diff-u"):
  _cmp_arg()
  fn1 = fname + '.' + cmp
  fn2 = fname
  data1 = fname2lines(fn1)
  if len(sys.argv) >= 3 and sys.argv[2] in backups:
    # Doing a diff. between two backups...
    fn2 = fname + '.' + sys.argv[2]
  data2 = fname2lines(fn2)
  print("diff %s %s" % (fn1, fn2), file=sys.stderr)
  _max_update(' ', data1)
  _max_update(' ', data2)
  _reset_ui_name()
  while len(data1) > 0 or len(data2) > 0:
    if len(data1) <= 0:
        _print_line('+', line2data(data2[0]))
        data2.pop(0)
        continue
    if len(data2) <= 0:
        _print_line('-', line2data(data1[0]))
        data1.pop(0)
        continue

    d1 = line2data(data1[0])
    d2 = line2data(data2[0])

    if d1['name'] < d2['name']:
        _print_line('-', d1)
        data1.pop(0)
        continue

    if d1['name'] > d2['name']:
        _print_line('+', d2)
        data2.pop(0)
        continue

    if d1['update_info'] == d2['update_info']:
        _print_line(' ', d2)
        data1.pop(0)
        data2.pop(0)
        continue

    if cmd == "diff-u":
        _print_line('-', d1)
        data1.pop(0)
        _print_line('+', d2)
        data2.pop(0)
        continue

    # diff
    data1.pop(0)
    _print_line('!', d2)
    data2.pop(0)

    continue
  _explain_ui_name()