fedora-infra_ansible/scripts/updates-uptime-cmd.py

#! /usr/bin/python3

#  Create/view a "txt" file using the ansible playbook
# "generate-updates-uptimes-per-host-file.yml" which records the number of rpms
# available to be upgraded for ansible hosts, and the uptime of those hosts.
#  This is very helpful when doing upgrade+reboot runs, as we can easily see
# what has/hasn't been upgraded and/or rebooted.

# $0 update       = create the file and/or do backups
# $0 diff [x] [y] = see the difference between the current state and backups
# $0 uptime [x]   = see the current state, can be filtered for uptime >= x
# $0 info [x]     = see the current state, in long form, can be filtered by name
# $0 host [x]     = see the current state of a host(s), can be filtered by name
# $0 list [x]     = see the current state, can be filtered by name
# $0 backups      = see backups
# $0 backups-keep = clenaup old backups
# $0 stats [x]    = see stats, can specify a backup

# Examples:
# $0 update ... run it daily or so as root.
# $0 diff ... see what changed.
# $0 list '*.stg.*' ... see what staging looks like.
# $0 list '*copr*' ... see what copr looks like.
# $0 backups-keep 4 ... keep four days of backups (including today)
# $0 uptime 1d ... see what hasn't been rebooted in the last 24 hours.
# $0 uptime 25w ... see what hasn't been rebooted in too damn long

import os
import sys

import fnmatch
import glob
import time

#  If we try to update this seconds since the file changed, flush the
# ansible FACT cache.
conf_dur_flush_cache = (60*60*8)

#  Dir. where we put, and look for, the files...
conf_path = "/var/log/"

fname = conf_path + "ansible-list-updates-uptime.txt"
fname_today = fname + '.' + time.strftime("%Y-%m-%d")

backups = sorted(x.removeprefix(fname + '.') for x in glob.glob(fname + '.*'))

cmd = "diff"
if len(sys.argv) >= 2:
    if sys.argv[1] in ("backups", "backups-keep", "diff", "diff-u", "help", "host", "info", "list", "stats", "update", "update-fast", "update-daily", "uptime",):
        cmd = sys.argv.pop(1)

_tm_d = {'d' : 60*60*24, 'h' : 60*60, 'm' : 60, 's' : 1,
         'w' : 60*60*24*7,
         'q' : 60*60*24*7*13}
def parse_duration(seconds):
    if seconds is None:
        return None
    if seconds.isdigit():
        return int(seconds)

    ret = 0
    for mark in ('w', 'd', 'h', 'm', 's'):
        pos = seconds.find(mark)
        if pos == -1:
            continue
        val = seconds[:pos]
        seconds = seconds[pos+1:]
        if not val.isdigit():
            # dbg("!isdigit", val)
            return None
        ret += _tm_d[mark]*int(val)
    if seconds.isdigit():
        ret += int(seconds)
    elif seconds != '':
        # dbg("!empty", seconds)
        return None

    return ret


def _add_dur(dur, ret, nummod, suffix, static=False):
    mod = dur % nummod
    dur = dur // nummod
    if mod > 0 or (static and dur > 0):
        ret.append(suffix)
        if static and dur > 0:
            ret.append("%0*d" % (len(str(nummod)), mod))
        else:
            ret.append(str(mod))
    return dur

def format_duration(seconds, static=False):
    if seconds is None:
        seconds = 0
    dur = int(seconds)

    ret = []
    dur = _add_dur(dur, ret, 60, "s", static=static)
    dur = _add_dur(dur, ret, 60, "m", static=static)
    dur = _add_dur(dur, ret, 24, "h", static=static)
    dur = _add_dur(dur, ret,  7, "d", static=static)
    if dur > 0:
        ret.append("w")
        ret.append(str(dur))
    return "".join(reversed(ret))


cmp_arg = False
cmp = None
def _cmp_arg():
    global cmp
    global cmp_arg

    if len(sys.argv) < 2:
        cmp = backups[-1] # Most recent
    elif sys.argv[1] not in backups:
        _usage()
        print("Backups:", ", ".join(backups))
        sys.exit(1)
    else:
        cmp = sys.argv[1]
        cmp_arg = True

def line2data(line):
    name, rpms, uptime, date = line.split(' ', 3)
    osname = "Unknown"
    osvers = "0"
    osinfo = "Unknown/?"
    if ' ' in date:
        date, osname, osvers = date.split(' ', 2)
        osinfo = osname + '/' + osvers
        if osname == 'CentOS':
            osinfo = 'EL/' + osvers
        if osname == 'RedHat':
            osinfo = 'EL/' + osvers
        if osname == 'Fedora':
            osinfo = 'F/' + osvers

    updates = name + ' ' + rpms

    rpms = int(rpms)
    uptime = int(uptime)

    return locals()

def fname2lines(fname):
    return [x.strip() for x in open(fname).readlines()]

def bfname2lines(b):
    return fname2lines(fname + '.' + b)

def fname1():
    if cmp_arg:
        return bfname2lines(cmp)
    return fname2lines(fname)

def _usage():
  prog = "updates+uptime"
  if sys.argv:
      prog = os.path.basename(sys.argv[0])
  pl = " " * len(prog)
  print("""
 Usage: %s <cmd>
   Cmds:
        help
            = This message.

        backups
            = Show backups.
        backups-trim [days]
            = Cleanup old backups.

        diff         [backup1] [backup2]
        diff-u       [backup1] [backup2]
            = See the difference between the current state and backups.

        host         [host*] [backup]
            = See the current state of a host(s), can be filtered by name.
        info         [host*] [backup]
            = See the current state, in long form, can be filtered by name.
        list         [host*] [backup]
            = See the current state, can be filtered by name.

        stats        [backup]
            = Show stats.

        update
            = Create the file and/or do backups.

        uptime       [duration] [backup]
            = See the current state, can be filtered for uptime >= duration.
""" % (prog,))

if cmd == "help":
  _usage()

if cmd == "backups":
  print("Backups:")
  for backup in backups:
      print('', backup, len(bfname2lines(backup)))

if cmd == "backups-keep":
  keep = 8
  if len(sys.argv) >= 2:
      keep = int(sys.argv.pop(1))
  if keep <= 0:
      _usage()
      sys.exit(1)
  while keep < len(backups):
      # We just keep the newest N
      fn = fname + '.' + backups.pop()
      os.unlink(fn)

if cmd == "update":
  if not os.path.exists(fname):
    cmd = "update-daily" # This does the sorting etc.
  elif not os.path.exists(fname_today):
    cmd = "update-daily"
  else:
    mtime = os.path.getmtime(fname)
    if (int(time.time()) - mtime) <= (60*60*8): # 8 hours.
        cmd = "update-fast"

if cmd == "update": # Get the latest uptime.
  os.chdir("/srv/web/infra/ansible/playbooks")
  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates --flush-cache")
if cmd == "update-fast": # Use ansible FACT cache for uptime.
  os.chdir("/srv/web/infra/ansible/playbooks")
  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates")
if cmd == "update-daily": # Also create backup file.
  os.chdir("/srv/web/infra/ansible/playbooks")
  os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml --flush-cache")

if cmd == "stats":
  _cmp_arg()
  data = fname1()
  osinfo = {}
  print("Hosts:", len(data))
  most = 0, None
  updates = 0
  awake = 0
  awakest = 0, None
  for line in data:
    d2 = line2data(line)

    if d2['osname'] not in osinfo:
        osinfo[d2['osname']] = 0
    osinfo[d2['osname']] += 1
    if d2['osinfo'] not in osinfo:
        osinfo[d2['osinfo']] = 0
    osinfo[d2['osinfo']] += 1
    updates += d2['rpms']
    if d2['rpms'] >= most[0]:
        most = d2['rpms'], d2['name']

    awake += d2['uptime']
    if d2['uptime'] >= awakest[0]:
        awakest = d2['uptime'], d2['name']
  for osi in sorted(osinfo):
    print("  Hosts:", osi, osinfo[osi])
  print("Updates:", updates)
  print("  Most:", most[1], most[0])
  print("Uptime:", format_duration(awake))
  print("  Most:", awakest[1], format_duration(awakest[0]))

def _print_info(host, lines):
  hosts = []
  for x in lines:
      x = line2data(x)
      if fnmatch.fnmatch(x['name'], host):
          hosts.append(x)
  if not hosts:
      print("Not found host:", host)
      sys.exit(2)
  for host in hosts:
    print("Host:", host['name'])
    print("     OS:", host['osinfo'], host['osname'], host['osvers'])
    print("   Rpms:", host['rpms'])
    print(" Uptime:", format_duration(host['uptime']))
    print("   Date:", host['date'])

if cmd in ("host", "info"):
  if cmd == "host":
      host = "batcave*"
  else:
      host = "*"
  if len(sys.argv) >= 2:
      host = sys.argv.pop(1)
  if len(sys.argv) >= 2 and sys.argv[1] == "all":
      for b in backups:
          print("Backup:", b)
          _print_info(bfname2lines(b))
      sys.argv = [sys.argv[0]]
      print("Main:")
  _cmp_arg()
  _print_info(host, fname1())

_max_len_name  = 0
_max_len_rpms  = 0
_max_len_updur = 0
_max_len_date  = 0 # 4+1+2+1+2
def _max_update(lines):
    global _max_len_name
    global _max_len_rpms
    global _max_len_updur
    global _max_len_date

    for line in lines:
        data = line2data(line)
        if len(data['name']) > _max_len_name:
            _max_len_name = len(data['name'])
        if len(str(data['rpms'])) > _max_len_rpms:
            _max_len_rpms = len(str(data['rpms']))
        updur_len = len(format_duration(data['uptime'], static=True))
        if updur_len > _max_len_updur:
            _max_len_updur = updur_len
        if len(data['date']) > _max_len_date:
            _max_len_date = len(data['date'])

def _print_line(prefix, data):
    print("%s%-*s %*u %*s %*s %s" % (prefix,
          _max_len_name, data['name'],
          _max_len_rpms, data['rpms'],
          _max_len_updur, format_duration(data['uptime'], static=True),
          _max_len_date, data['date'], data['osinfo']))

if cmd == "list":
  host = "*"
  if len(sys.argv) >= 2:
      host = sys.argv.pop(1)

  _cmp_arg()
  lines = fname1()
  _max_update(lines)
  for line in lines:
      d1 = line2data(line)
      if not fnmatch.fnmatch(d1['name'], host):
          continue
      _print_line('', d1)

if cmd == "uptime":
  age = 0
  if len(sys.argv) >= 2:
      age = parse_duration(sys.argv.pop(1))

  _cmp_arg()
  lines = fname1()
  _max_update(lines)
  for line in lines:
      d1 = line2data(line)
      if age > d1['uptime']:
          continue
      _print_line('', d1)

if cmd in ("diff", "diff-u"):
  _cmp_arg()
  fn1 = fname + '.' + cmp
  fn2 = fname
  data1 = fname2lines(fn1)
  if len(sys.argv) >= 3 and sys.argv[2] in backups:
    # Doing a diff. between two backups...
    fn2 = fname + '.' + sys.argv[2]
  data2 = fname2lines(fn2)
  print("diff %s %s" % (fn1, fn2), file=sys.stderr)
  _max_update(data1)
  _max_update(data2)
  while len(data1) > 0 or len(data2) > 0:
    if len(data1) <= 0:
        _print_line('+', line2data(data2[0]))
        data2.pop(0)
        continue
    if len(data2) <= 0:
        _print_line('-', line2data(data1[0]))
        data1.pop(0)
        continue

    d1 = line2data(data1[0])
    d2 = line2data(data2[0])

    if d1['name'] < d2['name']:
        _print_line('-', d1)
        data1.pop(0)
        continue

    if d1['name'] > d2['name']:
        _print_line('+', d2)
        data2.pop(0)
        continue

    if d1['updates'] == d2['updates']:
        _print_line(' ', d2)
        data1.pop(0)
        data2.pop(0)
        continue

    if cmd == "diff-u":
        _print_line('-', d1)
        data1.pop(0)
        _print_line('+', d2)
        data2.pop(0)
        continue

    # diff
    data1.pop(0)
    _print_line('!', d2)
    data2.pop(0)

    continue