fedora-infra_ansible/files/scripts/updates-uptime-cmd.py

#! /usr/bin/python3

#  Create/view a "txt" file using the ansible playbook
# "generate-updates-uptimes-per-host-file.yml" which records the number of rpms
# available to be upgraded for ansible hosts, and the uptime of those hosts.
#  This is very helpful when doing upgrade+reboot runs, as we can easily see
# what has/hasn't been upgraded and/or rebooted.

# For examples, see the help command.

import os
import sys

import argparse
import fnmatch
import locale
import shutil
import time

#  Use utf8 prefixes in diff, these need to be a "normal" width 1 character
conf_utf8 = True
_conf_utf8_boot_ed = '⚠' # Rebooted
_conf_utf8_boot_up = '⚐' # Rebooted and updated
_conf_utf8_more_up = '➚'
_conf_utf8_less_up = '➘'
_conf_utf8_diff_os = '➜' # '♺' OSinfo is different, but the machine is the same
_conf_utf8_diff_hw = '⇉' # '모' machine_id is different

#  Arrow seperator for host command. Doesn't need to be a single character
conf_host_arrow_asci = '->'
conf_host_arrow_utf8 = '→'

#  Use ansi codes. None means auto, aka. look for a tty on stdout.
conf_ansi_terminal = None
conf_term_cmd = 'dim'
conf_term_diffstat_hostnum = ''
conf_term_diffstat_updates = 'bold'
conf_term_diffstat_added = ''
conf_term_diffstat_instl = ''
conf_term_diffstat_boots = 'underline'
conf_term_highlight = 'bold,underline'
conf_term_host_boot_ed = 'fg:d,bold,underline'
conf_term_host_boot_up = 'bold'
conf_term_host_more_up = ''
conf_term_host_less_up = ''
conf_term_host_diff_os = 'bold'
conf_term_host_diff_hw = 'reverse'
conf_term_keyword = 'underline'
conf_term_time = 'underline'
conf_term_title = 'italic'

#  Use _ instead of , for number seperator.
conf_num_sep_ = False

#  This is kind of a hack, if you run from a cron job then it should run at
# the same time each day, and this should be 1 hour or less. But life isn't
# perfect, so we give it some more time.
#  The two competing problems are 1) reboot machine with low uptime.
# 2) get data at 23:59 yesterday and 0:01 today.
# ...and we can't fix both without boot_id's ... so just have those.
conf_tmdiff_fudge = (60*60*4)

#  How many hosts to show in tier 4 updates/uptimes...
conf_stat_4_hosts = 4

#  Skip when everything is blank.
conf_host_skip_eq = True

#  How many host matches before we show diffstat in "host" command.
conf_host_show_diffstat_hostnum = 8

#  How many total host matches before we end host cmd early (-v shows all).
conf_host_end_total_hostnum = 20

#  How many history files do we show by default (-v shows all).
conf_hist_show = 20

#  Make it easier to see different date's
conf_ui_date = True

#  Do we use a shorter duration by default (drop minutes/seconds)
conf_short_duration = True

#  Do we want a small osinfo in diff/list/etc.
conf_small_osinfo = True

#  Show machine/boot id's in info command, by default (use -v).
conf_info_machine_ids = False

#  Try to print OS/ver even nicer (when small) ... but includes spaces.
conf_align_osinfo_small = True

#  Allow 9,999,999 updates, or try to work out the correct size.
conf_fast_width_history = True

#  Dynamically change the uptime of hosts based on the time since we looked
# at their uptime. Only for the main file. Assume they are still up etc.
conf_dynamic_main_uptime = True

# Hosts that we'll show info. for, by default. info/host cmds.
conf_important_hosts = ["batcave*", "bastion01*", "noc*"]

#  Remove suffix noise in names.
conf_suffix_dns_replace = {
  '.fedoraproject.org'    : '.<FP>.org',
  '.fedorainfracloud.org' : '.<FitC>.org',
}
_suffix_dns_replace = {}

#  Dir. where we put, and look for, the files...
conf_path = "/var/log/"

conf_user_conf_path = "~/.config/updates-uptimes/config.conf"

# Now we can change the above conf_ variables, via. a conf file.
def _user_conf():
    ucp = os.path.expanduser(conf_user_conf_path)
    if not os.path.exists(ucp):
        return

    for line in open(ucp):
        _user_conf_line(line)

def _user_conf_line(line):
    line = line.lstrip()
    if not line: return
    if line[0] == '#': return

    op = "+="
    x = line.split(op, 2)
    if len(x) != 2:
        op = ":="
        x = line.split(op, 2)
    if len(x) != 2:
        op = "="
        x = line.split(op, 2)
    if len(x) != 2:
        print(" Error: Configuration: ", line, file=sys.stderr)
        return
    key,val = x

    key = 'conf_' + key.strip().lower()
    if key not in globals():
        print(" Error: Configuration not found: ", key, file=sys.stderr)
        return

    if False: pass
    elif op == '=':
        val = val.strip()
        if False: pass
        elif val.lower() in ("false", "no"): val = False
        elif val.lower() in ("true", "yes"): val = True
        elif val == '[]': val = []
        elif val == '{}': val = {}
        elif val.isdigit(): val = int(val)

        if type(globals()[key]) != type(val):
            print(" Error: Configuration ", key,'bad:',val, file=sys.stderr)
            return
        globals()[key] = val
    elif op == '+=':
        val = val.strip()
        if type(globals()[key]) != type([]):
            print(" Error: Configuration ", key, 'not []', file=sys.stderr)
            return
        globals()[key].append(val)
    elif op == ':=':
        if type(globals()[key]) != type({}):
            print(" Error: Configuration ", key, 'not {}', file=sys.stderr)
            return

        if '=' not in val:
            print(" Error: Configuration bad :=", file=sys.stderr)
            return
        dkey, dval = val.split('=', 1)
        globals()[key][dkey.strip()] = dval.strip()
    else:
        print(" Error: Configuration ", key,'bad op', file=sys.stderr)
        return

# Have nice "plain" numbers...
def _ui_int(num):
    if conf_num_sep_:
        return "{:_}".format(int(num))
    return "{:,}".format(int(num))

# See: https://en.wikipedia.org/wiki/ANSI_escape_code#Select_Graphic_Rendition_parameters
# We merge the colours for 16 values so fg0-fgf bg0-bgf
ansi = {'bold' : '\033[1m', 'dim' : '\033[2m', 'italic' : '\033[3m',
        'underline' : '\033[4m', 'blink' : '\033[5m', 'reverse' :'\033[7m'}
ansi_stop = '\033[0m'
for i in range(7):
    ansi['fg:' + str(i)] = '\033[3' + str(i) + 'm'
    ansi['bg:' + str(i)] = '\033[4' + str(i) + 'm'
for i, j in ((0, '8'), (1, '9'), (2, 'a'), (3, 'b'),
                (4, 'c'), (5, 'd'), (6, 'e'), (7, 'f')):
    ansi['fg:' + j] = '\033[9'  + str(i) + 'm'
    ansi['bg:' + j] = '\033[10' + str(i) + 'm'
def _ui_t_align(text, align=None, olen=None):
    if align is None or align == 0:
        return text
    if olen is None:
        olen = len(text)
    if abs(align) > olen: # "%*s", align, text
        extra = abs(align) - olen
        if align > 0:
            text = " " * extra + text
        else:
            text = text + " " * extra
    return text
def _ui_t_ansi(text, codes, align=0):
    olen = len(text)
    text = _ui_t_align(text, align)
    if not conf_ansi_terminal or not codes or olen == 0:
        return text

    esc = ''
    for c in codes.split(','):
        if c == 'reset':
            esc = ''
        if c not in ansi: # Ignore bad codes
            continue
        esc += ansi[c]

    if not esc:
        return text
    # Deal with leading/trailing spaces, mainly for underline.
    olen = len(text)
    text = text.lstrip()
    prefix = olen - len(text)
    text = text.rstrip()
    suffix = (olen - prefix) - len(text)
    return "%*s%s%s%s%*s" % (prefix, '', esc, text, ansi_stop, suffix, '')

def _ui_t_cmd(text, align=0):
    return _ui_t_ansi(text, conf_term_cmd, align=align)
def _ui_t_high(text, align=0):
    return _ui_t_ansi(text, conf_term_highlight, align=align)
def _ui_t_key(text, align=0):
    return _ui_t_ansi(text, conf_term_keyword, align=align)
def _ui_t_time(text, align=0):
    return _ui_t_ansi(text, conf_term_time, align=align)
def _ui_t_title(text, align=0):
    return _ui_t_ansi(text, conf_term_title, align=align)

def _ui_t_diffstat_hostnum(text, align=0):
    return _ui_t_ansi(text, conf_term_diffstat_hostnum, align=align)
def _ui_t_diffstat_updates(text, align=0):
    return _ui_t_ansi(text, conf_term_diffstat_updates, align=align)
def _ui_t_diffstat_added(text, align=0):
    return _ui_t_ansi(text, conf_term_diffstat_added, align=align)
def _ui_t_diffstat_instl(text, align=0):
    return _ui_t_ansi(text, conf_term_diffstat_instl, align=align)
def _ui_t_diffstat_boots(text, align=0):
    return _ui_t_ansi(text, conf_term_diffstat_boots, align=align)

# Make it easier to spot when hosts aren't getting their data updated.
def _ui_date(d1, align=None, prev=None):
    if not conf_ui_date:
        return _ui_t_align(d1.date, align)
    if prev is not None and d1.date == prev:
        return _ui_t_align("    \"    ", align)
    if False and d1.date == backup_today: # Better, or no?
        return _ui_t_align("today", align)
    if False and d1.date == backup_yesterday:
        return _ui_t_align("yesterday", align)
    # YYYY-MM-DD
    # 1234567890
    if prev is None:
        prev = backups[-1]
    if conf_ansi_terminal and d1.date != prev:
        for i in (9, 8, 7, 5):
            if d1.date[:i] == prev[:i]:
                ndate = d1.date[:i] + _ui_t_high(d1.date[i:])
                return _ui_t_align(ndate, align, len(d1.date))
    return _ui_t_align(d1.date, align)

# History files are named <fname>.YYYY-MM-DD
def _glob_hist_suffix():
    for fn in os.listdir(os.path.dirname(fname)):
        if not fn.startswith(_fname  + '.'):
            continue
        fn = fn.removeprefix(_fname + '.')
        if len(fn) != len("YYYY-MM-DD"):
            continue
        if fn[0] != '2':              continue # Year
        if fn[1] not in "0123456789": continue
        if fn[2] not in "0123456789": continue
        if fn[3] not in "0123456789": continue
        if fn[4] != '-': continue
        if fn[5] not in "01":         continue # Month
        if fn[6] not in "0123456789": continue
        if fn[7] != '-': continue
        if fn[8] not in "0123":       continue # Day
        if fn[9] not in "0123456789": continue
        yield fn

_fname = "ansible-list-updates-uptime.txt"
def _pre_cmd__setup():
    global conf_path
    global fname
    global fname_today
    global fname_yesterday
    global backups
    global backup_today
    global backup_yesterday
    global conf_ansi_terminal

    conf_path = os.path.expanduser(conf_path)

    if conf_path[0] != '/':
        print(" Warning: Conf path isn't absolute", file=sys.stderr)

    _suffix_dns_replace.clear()
    for x in conf_suffix_dns_replace:
        _suffix_dns_replace[x] = False
    if conf_ansi_terminal is None:
        conf_ansi_terminal = sys.stdout.isatty()
    fname = conf_path + _fname
    backup_today = time.strftime("%Y-%m-%d", time.gmtime())
    fname_today = fname + '.' + backup_today
    backups = sorted(_glob_hist_suffix())

    tm_yesterday = int(time.time()) - (60*60*24)
    backup_yesterday = time.strftime("%Y-%m-%d", time.gmtime(tm_yesterday))
    fname_yesterday = fname + '.' + backup_yesterday

    if len(backups) < 1 or backups[-1] != backup_today:
        fname_today = None
    if len(backups) < 2 or backups[-2] != backup_yesterday:
        if fname_today is None and backups and backups[-1] == backup_yesterday:
            pass # Just missing today
        else:
            fname_yesterday = None


def _pre_cmd__verbose(args):
    if args.verbose <= 0:
        return

    if args.verbose >= 3:
        globals()['conf_ui_date'] = False
    if args.verbose >= 2:
        globals()['conf_small_osinfo'] = False
        globals()['conf_suffix_dns_replace'] = {}
    globals()['conf_hist_show'] = 0
    globals()['conf_host_end_total_hostnum'] = 0
    globals()['conf_host_skip_eq'] = False
    globals()['conf_info_machine_ids'] = True
    globals()['conf_short_duration'] = False
    globals()['conf_stat_4_hosts'] *= (args.verbose * 2)


def _wild_eq(s1, s2):
    """ Compare two strings, but allow '?' to mean anything. """
    if s1 == '?' or s2 == '?':
        return True
    return s1 == s2

_max_len_osnm = 0 # osname_small
_max_len_osvr = 0 # osvers ... upto the first '.'
class Host():
    """ Class for holding the Host data from a line in the files. """

    __slots__ = ['name', 'rpms', 'uptime', 'date', 'osname', 'osvers',
                 'osname_small', 'machine_id', 'boot_id']

    def __init__ (self, data):
        global _max_len_osnm
        global _max_len_osvr

        self.name   = data['name']
        self.rpms   = data['rpms']
        self.uptime = data['uptime']
        self.date   = data['date']

        self.osname = data['osname']
        self.osvers = data['osvers']

        self.machine_id = data['machine_id']
        self.boot_id = data['boot_id']

        if False: pass
        elif self.osname == 'CentOS':
            osname_small = 'EL'
        elif self.osname == 'RedHat':
            osname_small = 'EL'
        elif self.osname == 'Fedora':
            osname_small = 'F'
        else:
            osname_small = self.osname[:3]
        self.osname_small = osname_small

        _max_len_osnm = max(len(osname_small), _max_len_osnm)
        vers = self.osvers
        off = vers.find('.')
        if off != -1:
            vers = vers[:off]
        _max_len_osvr = max(len(vers), _max_len_osvr)

    def __str__(self):
        return self.name

    def __eq__(self, other):
        if self.name != other.name:
            return False
        if self.rpms != other.rpms:
            return False
        if not _wild_eq(self.osname, other.osname):
            return False
        if not _wild_eq(self.osvers, other.osvers):
            return False
        if not _wild_eq(self.machine_id, other.machine_id):
            return False
        return True

    def __gt__(self, other):
        if self.name > other.name:
            return True
        if self.name != other.name:
            return False

        if self.rpms > other.rpms:
            return True
        if self.rpms != other.rpms:
            return False

        if self.osname > other.osname:
            return True
        if not _wild_eq(self.osname, other.osname):
            return False

        if self.osvers > other.osvers:
            return True
        return False

    # Pretend to be a dict...
    def __getitem__(self, key):
        if key not in self.__slots__:
            raise KeyError()
        return getattr(self, key)

    @property
    def osinfo(self):
        return "%s/%s" % (self.osname, self.osvers)

    @property
    def osinfo_small(self):
        if conf_align_osinfo_small:
            vers = self.osvers
            rest = ''
            off = vers.find('.')
            if off != -1:
                rest = vers[off:]
                vers = vers[:off]
            return "%*s/%*s%s" % (_max_len_osnm, self.osname_small,
                                  _max_len_osvr, vers, rest)
        return "%s/%s" % (self.osname_small, self.osvers)

    @property
    def date_tm(self):
        return time.mktime(time.strptime(self.date, "%Y-%m-%d"))


_tm_d = {'d' : 60*60*24, 'h' : 60*60, 'm' : 60, 's' : 1,
         'w' : 60*60*24*7,
         'q' : 60*60*24*7*13}
def parse_duration(seconds):
    if seconds is None:
        return None
    if seconds.isdigit():
        return int(seconds)

    ret = 0
    for mark in ('w', 'd', 'h', 'm', 's'):
        pos = seconds.find(mark)
        if pos == -1:
            continue
        val = seconds[:pos]
        seconds = seconds[pos+1:]
        if not val.isdigit():
            # dbg("!isdigit", val)
            return None
        ret += _tm_d[mark]*int(val)
    if seconds.isdigit():
        ret += int(seconds)
    elif seconds != '':
        # dbg("!empty", seconds)
        return None

    return ret


def _add_dur(dur, ret, nummod, suffix, static=False):
    mod = dur % nummod
    dur = dur // nummod
    if mod > 0 or (static and dur > 0):
        ret.append(suffix)
        if static and dur > 0:
            ret.append("%0*d" % (len(str(nummod)), mod))
        else:
            ret.append(str(mod))
    return dur

def format_duration(seconds, short=False, static=False):
    if seconds is None:
        seconds = 0
    dur = int(seconds)

    ret = []
    dur = _add_dur(dur, ret, 60, "s", static=static)
    dur = _add_dur(dur, ret, 60, "m", static=static)
    if short:
        if dur == 0 and not static:
            return '<1h'
        if dur == 0:
            return '<01h'
        ret = []
    dur = _add_dur(dur, ret, 24, "h", static=static)
    dur = _add_dur(dur, ret,  7, "d", static=static)
    if dur > 0:
        ret.append("w")
        ret.append(str(dur))
    return "".join(reversed(ret))

# Duration in UI for lists/etc.
def _ui_dur(dur):
    return format_duration(dur, short=conf_short_duration, static=True)

def _main_file_recent():
    f1 = os.stat(fname)

    if (int(time.time()) - f1.st_mtime) > (60*60*24):
        return False
    return True

def _files_identical(fn1, fn2):
    f1 = os.stat(fn1)
    f2 = os.stat(fn2)
    if f1.st_size != f2.st_size:
        return False
    if (f1.st_mtime - f2.st_mtime) > 64: # seconds, just a copy
        return False
    return True

def _backup_yesterday_identical():
    if fname_yesterday is None:
        return False

    return _files_identical(fname, fname_yesterday)

def _backup_today_identical():
    if fname_today is None:
        return False

    return _files_identical(fname, fname_today)

_max_len_osnm = 0 # osname_small
_max_len_osvr = 0 # osvers ... upto the first '.'
def line2data(line):
    global _max_len_osnm
    global _max_len_osvr

    name, rpms, uptime, date = line.split(' ', 3)
    osname = "?"
    osvers = "?"
    machine_id = "?"
    boot_id = "?"
    if ' ' in date:
        date, osname, osvers = date.split(' ', 2)
        if ' ' in osvers:
            osvers, machine_id, boot_id = osvers.split(' ', 2)

    rpms = int(rpms)
    uptime = int(uptime)

    return Host(locals())

def lines2datas(lines):
    return (line2data(line) for line in lines)

# Filter datas using name as a filename wildcard match.
def filter_name_datas(datas, names):
    if not names: # Allow everything...
        for data in datas:
            yield data
        return

    for data in datas:
        for name in names:
            if fnmatch.fnmatch(data.name, name):
                break
        else:
            continue
        yield data

# Filter datas using osname/vers/info as a filename wildcard match.
def filter_osname_datas(datas, names):
    if not names: # Allow everything...
        for data in datas:
            yield data
        return

    for data in datas:
        for name in names:
            if fnmatch.fnmatch(data.osinfo, name):
                break
            if fnmatch.fnmatch(data.osinfo_small, name):
                break
            if fnmatch.fnmatch(data.osname, name):
                break
            if fnmatch.fnmatch(data.osname_small, name):
                break
            if fnmatch.fnmatch(data.osvers, name):
                break
            off = data.osvers.find('.')
            if off != -1:
                vers = data.osvers[:off]
                if fnmatch.fnmatch(vers, name):
                    break

        else:
            continue
        yield data

# Filter datas using the date the data is gathered
def filter_age_min_datas(datas, age):
    now = time.time()
    for data in datas:
        if (now - data.date_tm) < age:
            continue
        yield data

# Filter datas using uptime as a minium.
def filter_uptime_min_datas(datas, uptime):
    for data in datas:
        if data.uptime < uptime:
            continue
        yield data

# Filter datas using uptime as a maximum.
def filter_uptime_max_datas(datas, uptime):
    for data in datas:
        if data.uptime > uptime:
            continue
        yield data

# Sub. suffix of DNS names for UI
def _ui_name(name):
    for suffix in conf_suffix_dns_replace:
        if name.endswith(suffix):
            _suffix_dns_replace[suffix] = True
            return name[:-len(suffix)] + conf_suffix_dns_replace[suffix]
    return name

def _ui_osinfo(data):
    if conf_small_osinfo:
        return data.osinfo_small
    return data.osinfo

# Reset the usage after _max_update()
def _reset_ui_name():
    for suffix in sorted(_suffix_dns_replace):
        _suffix_dns_replace[suffix] = False

# Explain if we used any suffix subs.
def _explain_ui_name():
    done = False
    pre = "* NOTE:"
    for suffix in sorted(_suffix_dns_replace):
        if _suffix_dns_replace[suffix]:
            print("%s %12s = %s" % (pre,conf_suffix_dns_replace[suffix],suffix))
            pre = "      :"
            done = True
    if done:
        print("      : Use -vv to show full names.")

def fname2lines(fname):
    return [x.strip() for x in open(fname).readlines()]

def bfname2lines(b):
    return fname2lines(fname + '.' + b)

def _maybe_dynamic_uptime(data):
    """ Only call this for the main file data. """
    if not conf_dynamic_main_uptime:
        return data

    mtime = os.path.getmtime(fname)
    since = int(time.time()) - int(mtime)
    data = list(sorted(data))
    for d1 in data:
        d1.uptime += since
    return data

def fname_datas():
    return _maybe_dynamic_uptime(lines2datas(fname2lines(fname)))


# We don't want to diff against the latest backup, if that backup is identical
def _diff_latest():
    ret = backups[-1] # Most recent
    ident = None
    if fname_today is not None:
        ident = _backup_today_identical()
    elif fname_yesterday is not None:
        ident = _backup_yesterday_identical()
    if ident is not None and ident:
        # Eg. if you just do one update a day, you want to cmp vs.
        # the previous day, not today.
        ret = backups[-2]
    return ret

# Alters the arguments, so we know wtf the filename is.
def fname1(args=[]):
    _dbg_fname1 = False
    if _dbg_fname1: print("JDBG: fname1:", args)
    if args and args[0] not in backups:
        if args[0] in ("diff-latest", "diff-newest"):
            args[0] = _diff_latest()

        if args[0] in ("latest", "newest"):
            args[0] = backups[-1] # Most recent
        if args[0] == "oldest":
            args[0] =  backups[0]
        if args[0] == "today":
            if fname_today is not None:
                args[0] = backup_today
            else:
                args[0] = "main"
        if args[0] == "yesterday" and fname_yesterday is not None:
            if fname_yesterday is not None:
                args[0] = backup_yesterday
            else:
                args[0] = "main"
    if _dbg_fname1: print("JDBG: fname1.2:", args)
    if args and args[0] != "main":
        return lines2datas(bfname2lines(args[0]))
    return fname_datas()

# Has the host been rebooted between these two points.
def host_rebooted(d1, d2):
    # This is easy mode, just compare boot ids
    if d1.boot_id != '?' and d2.boot_id != '?':
        return d1.boot_id != d2.boot_id

    # Now we try to work it out from uptime...
    if d1.date == d2.date and d1.uptime > d2.uptime:
        return True
    # However, we can be looking at old history
    tm1 = d1.date_tm
    tm2 = d2.date_tm
    if tm1 > tm2: # Looking backwards in time...
        return False
    d1up = d1.uptime
    tmdiff = tm2 - tm1
    if tmdiff > conf_tmdiff_fudge:
        d1up += tmdiff - conf_tmdiff_fudge
    return d1up > d2.uptime

_max_len_name = 0
_max_len_rpms = 0 # Number of rpm updates via. _ui_int().
_max_len_upts = 0 # Uptime duration with short=True
_max_len_date = 0 # 2025-08-04 = 4+1+2+1+2
_max_terminal_width = shutil.get_terminal_size().columns
if _max_terminal_width < 20:
    _max_terminal_width = 80
_max_terminal_width -= 14
def _max_update(datas):
    for data in datas:
        _max_update_data(data)

def _max_update_data(data):
    global _max_len_name
    global _max_len_rpms
    global _max_len_upts
    global _max_len_date

    name = _ui_name(data.name)
    if len(name) > _max_len_name:
        _max_len_name = len(name)

    rpms = _ui_int(data.rpms)
    if len(rpms) > _max_len_rpms:
        _max_len_rpms = len(rpms)

    upts_len = len(_ui_dur(data.uptime))
    if upts_len > _max_len_upts:
        _max_len_upts = upts_len

    if len(data.date) > _max_len_date:
        _max_len_date = len(data.date)

def _max_update_correct(prefix):
    global _max_len_name
    global _max_len_rpms
    global _max_len_upts
    global _max_len_date
    mw = _max_terminal_width - len(prefix)
    if _max_len_name + _max_len_rpms + _max_len_upts + _max_len_date < (mw-8):
        _max_len_name += 1
        _max_len_rpms += 1
        _max_len_upts += 1
        _max_len_date += 1

    while _max_len_name + _max_len_rpms + _max_len_upts + _max_len_date >= mw:
        _max_len_name -= 1

def _wild_info_eq(d1, d2):
    if not _wild_eq(d1.osname, d2.osname):
        return False
    if not _wild_eq(d1.osvers, d2.osvers):
        return False
    return True

# Return stats for updates added/deleted between two data sets.
def _diffstats(data1, data2):
    uadd, udel, boot = 0, 0, 0

    data1 = list(sorted(data1))
    data2 = list(sorted(data2))
    while len(data1) > 0 or len(data2) > 0:
        if len(data1) <= 0:
            d2 = data2.pop(0)
            uadd += d2.rpms
            boot += 1
            continue
        if len(data2) <= 0:
            d1 = data1.pop(0)
            udel -= d1.rpms
            continue

        d1 = data1[0]
        d2 = data2[0]

        if d1.name < d2.name:
            udel -= d1.rpms
            data1.pop(0)
            continue

        if d1.name > d2.name:
            uadd += d2.rpms
            data2.pop(0)
            boot += 1
            continue

        if d1 == d2:
            if host_rebooted(d1, d2):
                boot += 1
            data1.pop(0)
            data2.pop(0)
            continue

        if (not _wild_eq(d1.machine_id, d2.machine_id) or
            not _wild_info_eq(d1, d2)):
            boot += 1
            udel -= d1.rpms
            uadd += d2.rpms
            data1.pop(0)
            data2.pop(0)
            continue

        # Now name is eq and osinfo is eq
        if host_rebooted(d1, d2):
            boot += 1
        #  So either new updates arrived, or we installed some and they went
        # down ... alas. we can't tell if both happened.
        if d1.rpms > d2.rpms:
            udel -= d1.rpms - d2.rpms
        if d1.rpms < d2.rpms:
            uadd += d2.rpms - d1.rpms
        data1.pop(0)
        data2.pop(0)

    # diffstat returns...
    return uadd, udel, boot

def _ui_diffstats(data1, data2):
    cmpds = _diffstats(data1.copy(), data2.copy())
    return _ui_int(cmpds[0]), _ui_int(cmpds[1]), _ui_int(cmpds[2])

def _print_diffstats(hostnum, updates, cmpds):
    hostnum = _ui_t_diffstat_hostnum(str(hostnum))
    updates = _ui_t_diffstat_updates(str(updates))
    added = _ui_t_diffstat_added(cmpds[0])
    instl = _ui_t_diffstat_instl(cmpds[1])
    boots = _ui_t_diffstat_boots(cmpds[2])
    print('hosts=%s updates=%s (a=%s i=%s) boots=%s' % (hostnum, updates,
                                                        added, instl, boots))



# This is the real __main__ start ...
def _usage(short=False):
    prog = "updates+uptime"
    if sys.argv:
        prog = os.path.basename(sys.argv[0])
    print("""\
 Usage: %s <cmd>
   Optional arguments:
        --help, -h            Show this help message and exit.
        --verbose, -v         Increase verbosity.
        --conf CONF           Specify configuration.
        --db-dir DB_DIR       Change the path to the files.
        --ansi ANSI           Use ansi terminal codes.
   Cmds:
""" % (prog,), end='')
    if short:
        print("""\
        diff/-u              [backup1] [backup2]
        help
        history
        history-keep         [days]
        hosts/-u             [host*] [host*]...
        info                 [host*] [backup] [backup]...
        list                 [host*] [backup]
        list-n               [host*] [host*]...
        old-list              duration
        oslist               [os*] [backup]
        oslist-n             [os*] [os*]...
        stats                [backup] [host*] [host*]...
        update
        update-host           host
        uptime/-min/-max      duration [backup]
""", end='')
    else:
        # Also see: _cmd_help() below...
        print("""\
        diff                 [backup1] [backup2]
            = See the difference between the current state and backups.
        diff-u               [backup1] [backup2]
            = Shows before/after instead of modified (like diff -u).

        history
            = Show summary of current data, and how it changed over time.
        history-keep         [days]
            = Cleanup old history.

        hosts/-u             [host*] [host*]...
            = See the history of a host(s).
        info                 [host*] [backup] [backup]...
            = See the current state, in long form, can be filtered by name.
        list                 [host*] [backup]
        list-n               [host*] [host*]...
            = See the current state, can be filtered by name.
        old-list              duration
            = See the current state of hosts with data older than duration.
        oslist               [os*] [backup]
        oslist-n             [os*] [os*]...
            = See the current state, can be filtered by OS.

        stats                [backup] [host*] [host*]...
            = Show general stats.

        update
            = Run update-fast, or update-daily if no daily backup.
        update-fast
        update-host           host
            = Create/update the main file, for the specified host(s).
        update-daily
            = Run update-fast and force do a backup for today.
        update-daily-refresh
            = Run update-daily with an empty main file.

        uptime-min            duration [backup]
            = See the current state, can be filtered for uptime >= duration.
        uptime-max            duration [backup]
            = See the current state, can be filtered for uptime <= duration.
""", end='')


def _cmd_history_keep(args):
    keep = args.keep
    while keep < len(backups):
        # We just keep the newest N
        b = backups.pop(0)
        print("Removing:", b)
        fn = fname + '.' + b
        os.unlink(fn)

def _cmd_update(args):
    cmd = args.cmd
    if cmd == "update":
        cmd = "update-flush"
        if not os.path.exists(fname):
            cmd = "update-daily" # This does the sorting etc.
        elif fname_today is None:
            cmd = "update-daily"
        elif False: # -fast and -flush are the same now, so who cares.
            mtime = os.path.getmtime(fname)
            if (int(time.time()) - mtime) > conf_dur_flush_cache:
                cmd = "update-fast"

    if cmd == "update-host":
        os.chdir("/srv/web/infra/ansible/playbooks")
        os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates --limit '" + args.host + "'")
    if cmd == "update-flush": # Get the latest uptime.
        # No need to flush caches now, the new playbook should DTRT.
        os.chdir("/srv/web/infra/ansible/playbooks")
        os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates") # --flush-cache")
    if cmd == "update-fast": # Same as -flush now.
        os.chdir("/srv/web/infra/ansible/playbooks")
        os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates")
    if cmd == "update-daily-refresh": # Also recreate the main file.
        if os.path.exists(fname):
            os.unlink(fname)
        cmd = "update-daily"
    if cmd == "update-daily": # Also create backup file.
        os.chdir("/srv/web/infra/ansible/playbooks")
        os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml")

def _pre_cmd__check_paths():
    # Below here are the query commands, stuff needs to exist at this point.
    if not os.path.exists(fname):
        print(" Error: No main file. Run update sub-command", file=sys.stderr)
        sys.exit(4)
    if not _main_file_recent():
        print(" Warning: Main file is old. Run update sub-command", file=sys.stderr)
    if fname_today is None:
        print(" Warning: History for today does not exist!", file=sys.stderr)
    if fname_yesterday is None:
        print(" Warning: History for yesterday does not exist!", file=sys.stderr)

def _backup_suffix(backup):
    suffix = ''
    if backup == backup_today:
        if _backup_today_identical():
            suffix = ' (' + _ui_t_time("today") + ', is eq)'
        else:
            suffix = ' (today)'
    if backup == backup_yesterday:
        if _backup_yesterday_identical():
            suffix = ' (' + _ui_t_time("yesterday") + ', is eq)'
        else:
            suffix = ' (yesterday)'
    return suffix

def _hist_lengths(hosts=None):
    #  We _could_ open+read+etc each file, just to find out the max updates for
    # all hist ... but len("Updates")+2=9 which means 9,999,999 updates)
    hl = len("Hosts")
    ul = len("Updates")
    rl = len("Boots")
    if conf_fast_width_history:
        ul += 2
    else:
    #  Whatever, it's less memory than holding all history at once if you want
    # to enable it..
        for backup in reversed(backups):
            data = list(sorted(lines2datas(bfname2lines(backup))))
            data = list(filter_name_datas(data, hosts))
            updates = _ui_int(sum(d.rpms for d in data))
            hl = max(hl, len(_ui_int(len(data))))
            ul = max(ul, len(updates))
    return hl, ul, rl

def _cmd_history(args):
    global backups

    last_name = "main"
    last_data = list(sorted(fname_datas()))
    last_suff = ""

    hl, ul, rl = _hist_lengths()

    print('', _ui_t_title("Day", 10), _ui_t_title("Hosts", hl),
            _ui_t_title("Updates", ul),
            '', _ui_t_title("Added", ul), _ui_t_title("Inst.", ul+1), '',
            _ui_t_title("Boots", rl))

    if conf_hist_show > 0 and conf_hist_show < len(backups):
        backups = backups[-conf_hist_show:]

    for backup in reversed(backups):
        data = list(sorted(lines2datas(bfname2lines(backup))))
        updates = sum(d.rpms for d in last_data)
        less_updates = False
        if updates < sum(d.rpms for d in data):
            less_updates = True
        updates = _ui_int(updates)
        ul = max(ul, len(updates))
        updates = "%*s" % (ul, updates)
        if less_updates:
            updates = _ui_t_high(updates)

        cmpds = _ui_diffstats(data, last_data)

        if len(last_data) != len(data):
            nhosts = _ui_t_high("%*s" % (hl, _ui_int(len(last_data))))
        else:
            nhosts = "%*s" % (hl, _ui_int(len(last_data)))
        
        print(' %10s %s %s, %*s %*s, %*s %s' % (last_name,
                nhosts, updates,
                ul, cmpds[0], ul+1, cmpds[1], rl, cmpds[2], last_suff))
        last_name = backup
        last_data = data
        last_suff = _backup_suffix(backup)
    updates = _ui_int(sum(d.rpms for d in last_data))
    print(' %10s %*s %*s %s' % (last_name, hl, _ui_int(len(last_data)),
            ul, updates, last_suff))

def _cli_match_host(args, data):
    if args.hosts:
        hosts = args.hosts[:]
        print("Matching:", ", ".join(hosts))
        data = filter_name_datas(data, hosts)
        data = list(data)
        if not data:
            print("Not host(s) matched:", ", ".join(hosts))
            sys.exit(2)
    return data

def _cmd_stats(args):
    global conf_suffix_dns_replace
    global _max_len_name
    global _max_len_rpms
    global _max_len_upts
    global _max_len_date

    data = fname1([args.hist])
    data = list(_cli_match_host(args, data))
    # Basically we have hosts/updates/uptime and we want 4 tiers of data:
    # 1. All. 2. OS name (Eg. Fedora). 3. OS name+version (Eg. Fedora 42).
    # 4. For updates/uptime a "few" hosts with the biggest numbers.
    osdata = {'hosts' : {}, 'updates' : {}, 'uptimes' : {}, 'vers' : {}}
    updates = 0 # total updates
    most = [] # Tier 4, for updates
    awake = 0 # total uptime
    awakest = [] # Tier 4, for uptime

    conf_suffix_dns_replace = {} # Turn off shortened names for stats...
    for d2 in data:
        # Tidy UI for OS names with only one version...
        if d2.osname not in osdata['vers']:
            osdata['vers'][d2.osname] = set()
        osdata['vers'][d2.osname].add(d2.osinfo)

        # Tier 2/3 hosts...
        if d2.osname not in osdata['hosts']:
            osdata['hosts'][d2.osname] = 0
        osdata['hosts'][d2.osname] += 1

        if d2.osinfo not in osdata['hosts']:
            osdata['hosts'][d2.osinfo] = 0
        osdata['hosts'][d2.osinfo] += 1

        updates += d2.rpms
        # Tier 2/3 updates...
        if d2.osname not in osdata['updates']:
            osdata['updates'][d2.osname] = 0
        osdata['updates'][d2.osname] += d2.rpms

        if d2.osinfo not in osdata['updates']:
            osdata['updates'][d2.osinfo] = 0
        osdata['updates'][d2.osinfo] += d2.rpms
        # Tier 4 updates...
        most.append((d2.rpms, d2.uptime, d2))
        most.sort()
        while len(most) > conf_stat_4_hosts:
            most.pop(0)

        awake += d2.uptime
        # Tier 2/3 uptimes...
        if d2.osname not in osdata['uptimes']:
            osdata['uptimes'][d2.osname] = 0
        osdata['uptimes'][d2.osname] += d2.uptime

        if d2.osinfo not in osdata['uptimes']:
            osdata['uptimes'][d2.osinfo] = 0
        osdata['uptimes'][d2.osinfo] += d2.uptime
        # Tier 4 uptimes...
        awakest.append((d2.uptime, d2.rpms, d2))
        awakest.sort()
        while len(awakest) > conf_stat_4_hosts:
            awakest.pop(0)

    # Print "stats"
    # _max_update(data)
    # Do this by hand...
    _max_len_name = max((len(d.name) for d in data))
    _max_len_rpms = max(len("Updates/h"), len(_ui_int(updates)))
    _max_len_upts = max(len("Uptime/h"), len(_ui_dur(awake)))
    _max_len_date = 0
    _max_update_correct(' ')
    print(_ui_t_title("OS", -16), '', _ui_t_title("Hosts", 6),
            _ui_t_title("Updates", _max_len_rpms), 
            _ui_t_title("Uptime", _max_len_upts),
            _ui_t_title("Updates/h", _max_len_rpms), 
            _ui_t_title("Uptime/h", _max_len_upts))

    print("-" * (16+2+6+2*(1+_max_len_rpms+1+_max_len_upts)))
    nhosts = len(data)
    print("%-16s: %6s %*s %*s %*s %*s" % ("All", _ui_int(len(data)),
            _max_len_rpms, _ui_int(updates), _max_len_upts, _ui_dur(awake),
            _max_len_rpms, _ui_int(updates / nhosts),
            _max_len_upts, _ui_dur(awake / nhosts)))
    subprefix = ''
    subplen   = 14
    max_nhosts_lvl_1 = 0
    max_update_lvl_1 = 0
    max_uptime_lvl_1 = 0
    max_nhosts_lvl_2 = 0
    max_update_lvl_2 = 0
    max_uptime_lvl_2 = 0
    for osi in sorted(osdata['hosts']):
        if '/' not in osi:
            max_nhosts_lvl_1 = max(max_nhosts_lvl_1, osdata['hosts'][osi])
            supd = osdata['updates'][osi] / osdata['hosts'][osi]
            max_update_lvl_1 = max(max_update_lvl_1, supd)
            supt = osdata['uptimes'][osi] / osdata['hosts'][osi]
            max_uptime_lvl_1 = max(max_uptime_lvl_1, supt)

    def _ui_up_d_t(osi, lvl):
            if lvl == 1:
                mhosts = max_nhosts_lvl_1
                mupd   = max_update_lvl_1
                mupt   = max_uptime_lvl_1
                uiosi = "%-14s" % (osi,)
            else:
                mhosts = max_nhosts_lvl_2
                mupd   = max_update_lvl_2
                mupt   = max_uptime_lvl_2
                uiosi = "%-*s" % (subplen, osi)
            nhosts = osdata['hosts'][osi]
            uinhosts = "%6s" % (_ui_int(nhosts),)
            if nhosts >= mhosts:
                uinhosts = _ui_t_high(uinhosts)
            uiupdates = "%*s" % (_max_len_rpms, _ui_int(osdata['updates'][osi]))
            uiuptimes = "%*s" % (_max_len_upts, _ui_dur(osdata['uptimes'][osi]))
            num = osdata['updates'][osi] / nhosts
            uinpdates = "%*s" % (_max_len_rpms, _ui_int(num))
            bigger = False
            suf = ''
            if num >= mupd:
                bigger = True
                suf += ' *'
                # suf += ' Up'
                # uiupdates = _ui_t_high(uiupdates)
                uinpdates = _ui_t_high(uinpdates)
            num = osdata['uptimes'][osi] / nhosts
            uinptimes = "%*s" % (_max_len_upts, _ui_dur(num))
            if num >= mupt:
                bigger = True
                # suf += ' Tm'
                # uiuptimes = _ui_t_high(uiuptimes)
                uinptimes = _ui_t_high(uinptimes)
            if bigger:
                uiosi = _ui_t_high(uiosi)
            if lvl == 1:
                suf = _ui_t_high(suf)
            return uiosi, uinhosts, uiupdates, uiuptimes, uinpdates, uinptimes, suf

    for osi in sorted(osdata['hosts']):
        if '/' not in osi:
            if len(osdata['vers'][osi]) == 1:
                subprefix = ''
                subplen   = 14
                continue
            else: # Have level 2 be local to the group.
                max_nhosts_lvl_2 = 0
                max_update_lvl_2 = 0
                max_uptime_lvl_2 = 0
                for sosi in osdata['vers'][osi]:
                    supd = osdata['updates'][sosi] / osdata['hosts'][sosi]
                    supt = osdata['uptimes'][sosi] / osdata['hosts'][sosi]
                    max_nhosts_lvl_2 = max(max_nhosts_lvl_2, osdata['hosts'][sosi])
                    max_update_lvl_2 = max(max_update_lvl_2, supd)
                    max_uptime_lvl_2 = max(max_uptime_lvl_2, supt)
            subprefix = '  '
            subplen   = 12
            updt = _ui_up_d_t(osi, 1)
            uiosi     = updt[0]
            uinhosts  = updt[1]
            uiupdates = updt[2]
            uiuptimes = updt[3]
            uinpdates = updt[4]
            uinptimes = updt[5]
            print("  %s: %s %s %s %s %s%s" % (uiosi, uinhosts,
                uiupdates, uiuptimes, uinpdates, uinptimes, updt[6]))
        if '/' in osi:
            if subplen == 14: # Hack to say are we level 1
                updt = _ui_up_d_t(osi, 1)
            else:
                updt = _ui_up_d_t(osi, 2)
            uiosi     = updt[0]
            uinhosts  = updt[1]
            uiupdates = updt[2]
            uiuptimes = updt[3]
            uinpdates = updt[4]
            uinptimes = updt[5]
            print("  %s%s: %s %s %s %s %s%s" % (subprefix, uiosi, uinhosts,
                uiupdates, uiuptimes, uinpdates, uinptimes, updt[6]))
    print("-" * (16+2+6+2*(1+_max_len_rpms+1+_max_len_upts)))
    # Redo the lengths, because it's real hostname data now...
    _max_update(data)
    _max_len_date = 0
    _max_update_correct(' ')
    if most:
        # print("")
        print("Hosts with the most", _ui_t_key("Updates") + ":")
    for m in most:
        print("  %-*s %*s %*s %s" % (_max_len_name, m[2],
            _max_len_rpms, _ui_int(m[0]), _max_len_upts, _ui_dur(m[1]),
            _ui_osinfo(m[2])))
    if awakest:
        # print("")
        print("Hosts with the most", _ui_t_key("Uptime") + ":")
    for a in awakest:
        print("  %-*s %*s %*s %s" % (_max_len_name, a[2],
            _max_len_rpms, _ui_int(a[1]), _max_len_upts, _ui_dur(a[0]),
            _ui_osinfo(a[2])))
    _explain_ui_name()

def _print_info(hosts, data):
    fhosts = []
    for x in data:
        for host in hosts:
            if not fnmatch.fnmatch(x.name, host):
                continue
            fhosts.append(x)
            break
    if not fhosts:
        print("Not host(s) matched:", host)
        sys.exit(2)
    for host in fhosts:
        print("Host:", host.name)
        print("        OS:", host.osinfo)
        print("   Updates:", _ui_int(host.rpms))
        print("    Uptime:", format_duration(host.uptime)) # !ui_dur
        print("   Checked:", _ui_date(host))
        if conf_info_machine_ids:
            print("   Machine:", host.machine_id)
            print("      Boot:", host.boot_id)

def _cmd_info(args):
    hists = ['main']
    hosts = conf_important_hosts.copy()
    if args.host:
        # print("JDBG:", args.host)
        hosts = [args.host]
    if False and args.all:
        for b in backups:
            print("History:", b)
            _print_info(hosts, lines2datas(bfname2lines(b)))
    if args.hists:
        hists = args.hists[:]
    for hist in hists:
        if hist != "main": # One or more historical files...
            print("History:", hist)
        else:
            print("Main:")
        _print_info(hosts, fname1([hist]))

#  If save=True, and we haven't output anything then save the line as we might
# not do anything. After something has gone out save=True does nothing.
_prnt_line_saved = []
def _print_line_add(line):
    global _prnt_line_saved
    if _prnt_line_saved is None:
        _prnt_line_saved = []
    _prnt_line_saved.append(line)
def _print_line_reset():
    global _prnt_line_saved
    ret = _prnt_line_saved is not None
    _prnt_line_saved = []
    return ret
def _print_line(prefix, data, high='', save=False, prev=None):
    global _prnt_line_saved
    if prev is not None:
        prev = prev.date
    uiname = "%-*s" % (_max_len_name, _ui_name(data.name))
    if high:
        uiname = _ui_t_ansi(uiname, high)
    line = "%s%s %*s %*s %s %s" % (prefix,
          uiname,
          _max_len_rpms, _ui_int(data.rpms),
          _max_len_upts, _ui_dur(data.uptime),
          _ui_date(data, align=_max_len_date, prev=prev), _ui_osinfo(data))
    if save and _prnt_line_saved is not None:
        _prnt_line_saved.append(line)
        return
    if _prnt_line_saved is not None:
        for oline in _prnt_line_saved:
            print(oline)
        _prnt_line_saved = None
    print(line)

def _print_lines(prefix, data, explain=True):
    pd1 = None
    for d1 in data:
        _print_line(prefix, d1, prev=pd1)
        pd1 = d1
    if explain:
        _explain_ui_name()

# -n variants match multiple things, but only allow looking at current data
def _cmd_list(args):
    # FIXME: Ideally argparse would do this for us :(
    hists = []
    hosts = []
    osnames = []
    if hasattr(args, 'hists'):
        hists = args.hists[:]
    if hasattr(args, 'hosts'):
        hosts = args.hosts[:]
    if hasattr(args, 'osnames'):
        osnames = args.osnames[:]
    if hasattr(args, 'host') and args.host is not None:
        hosts += [args.host]
    if hasattr(args, 'osname') and args.osname is not None:
        osnames += [args.osname]

    data = fname1(hists)
    if hasattr(args, 'dateage'):
        data = list(filter_age_min_datas(data, args.dateage))
    data = list(filter_name_datas(data, hosts))
    data = list(filter_osname_datas(data, osnames))
    _max_update(data)
    _max_update_correct('')
    print(_ui_t_title("Host", -_max_len_name), 
            _ui_t_title("*", _max_len_rpms), 
            _ui_t_title("Up", _max_len_upts),
            _ui_t_title("Date", _max_len_date), _ui_t_title("OS"))
    _print_lines('', data)

def _cmd_uptime(args):
    age = 0
    if args.dur > 0:
        age = args.dur

    data = fname1(args.hists[:])
    if cmd == "uptime-max":
        data = list(filter_uptime_max_datas(data, age))
    else:
        data = list(filter_uptime_min_datas(data, age))
    _max_update(data)
    _max_update_correct('')
    _print_lines('', data)

def _diff_hosts(data1, data2, show_both=False, show_utf8=True, skip_eq=False):
    pdata = None
    while len(data1) > 0 or len(data2) > 0:
        if len(data1) <= 0:
            _print_line('+', data2[0], prev=pdata)
            pdata = data2.pop(0)
            continue
        if len(data2) <= 0:
            _print_line('-', data1[0], prev=pdata)
            pdata = data1.pop(0)
            continue

        d1 = data1[0]
        d2 = data2[0]

        if d1.name < d2.name:
            _print_line('-', d1, prev=pdata)
            pdata = data1.pop(0)
            continue

        if d1.name > d2.name:
            _print_line('+', d2, prev=pdata)
            pdata = data2.pop(0)
            continue

        # d1.name == d2.name; so both are going now
        data1.pop(0)
        data2.pop(0)

        # Name, rpms, and OSname/OSvers are the same
        if d1 == d2:
            if show_utf8 and conf_utf8 and host_rebooted(d1, d2):
                u_ed_or_up = _conf_utf8_boot_ed
                h_ed_or_up = conf_term_host_boot_ed
                if not d2.rpms:
                    u_ed_or_up = _conf_utf8_boot_up
                    h_ed_or_up = conf_term_host_boot_up
                _print_line(u_ed_or_up, d2, high=h_ed_or_up, prev=pdata)
                pdata = d2
                continue
            _print_line(' ', d2, save=skip_eq, prev=pdata)
            pdata = d2
            continue

        # Something changed, see what and set utf8 prefix and highlight
        if False: pass
        elif not _wild_eq(d1.machine_id, d2.machine_id):
            utf8, high = _conf_utf8_diff_hw, conf_term_host_diff_hw
        elif not _wild_info_eq(d1, d2):
            utf8, high = _conf_utf8_diff_os, conf_term_host_diff_os
        elif host_rebooted(d1, d2) and d1.rpms > d2.rpms:
            utf8, high = _conf_utf8_boot_up, conf_term_host_boot_up
        elif host_rebooted(d1, d2):
            utf8, high = _conf_utf8_boot_ed, conf_term_host_boot_ed
        elif d1.rpms > d2.rpms:
            utf8, high = _conf_utf8_less_up, conf_term_host_less_up
        else: # d1.rpms < d2.rpms:
            utf8, high = _conf_utf8_more_up, conf_term_host_more_up

        if not conf_utf8:
            utf8 = '+'

        # Something about host changed, show old/new...
        if show_both:
            _print_line('-', d1, prev=pdata)
            _print_line(utf8, d2, high=high, prev=pdata)
            pdata = d2
            continue

        # Something changed, but we only show the new data...
        if not conf_utf8:
            utf8 = '!'
        _print_line(utf8, d2, high=high, prev=pdata)
        pdata = d2

def _cmd_diff(args):
    hists = args.hists[:]
    if len(hists) < 1:
        hists = ["diff-latest", 'main']
    if len(hists) < 2:
        hists += ['main']
    data1 = fname1(hists)
    fn1 = hists.pop(0)
    data2 = fname1(hists)
    fn2 = hists.pop(0)
    print(_ui_t_cmd("diff %s %s" % (fn1, fn2)), file=sys.stderr)
    data1 = list(sorted(data1))
    data2 = list(sorted(data2))
    hosts = _ui_int(len(data2))
    updates = _ui_int(sum(d.rpms for d in data2))
    ul = len(updates)
    cmpds = _ui_diffstats(data1, data2)
    _max_update(data1)
    _max_update(data2)
    _max_update_correct(' ')

    _diff_hosts(data1, data2, show_both=cmd == "diff-u", show_utf8=cmd == "diff")

    _print_diffstats(hosts, updates, cmpds)
    _explain_ui_name()

# Like diff/history mixed, but for specific hosts...
def _cmd_host(args):
    global conf_host_skip_eq

    hosts = conf_important_hosts.copy()
    if args.hosts:
        hosts = args.hosts

    print("Hosts history:")
    last_name = "main"
    last_data = list(sorted(filter_name_datas(fname_datas(), hosts)))

    _max_update(last_data)
    if not conf_fast_width_history:
        for backup in reversed(backups):
            data = filter_name_datas(lines2datas(bfname2lines(backup)), hosts)
            data = list(sorted(data))
            _max_update(data)
    _max_update_correct(' ')

    if conf_utf8:
        sep = conf_host_arrow_utf8
    else:
        sep = conf_host_arrow_asci
    done = False
    skipped_num = 0
    thostnum = 0
    for backup in reversed(backups):
        data = filter_name_datas(lines2datas(bfname2lines(backup)), hosts)
        data = list(sorted(data))

        if done and skipped_num < 1:
            print("")
        done = True
        skipped = sep + ' '
        if skipped_num >= 1:
            # Use "skip(s)" so the alignment works out, for 1-9 skips.
            skipped = "%s %s skip(s) %s " % (sep, skipped_num, sep)

        _print_line_add("Host diff: %s %s%s" % (backup, skipped,
                                                _ui_t_time(last_name)))
        if conf_fast_width_history:
            _max_update(data)
            _max_update_correct(' ')
        if skipped and backup == backups[0]:
            conf_host_skip_eq = False
        _diff_hosts(data.copy(), last_data.copy(), show_both=cmd.endswith("-u"),
                    skip_eq=conf_host_skip_eq)
        lines_output = not _print_line_reset()
        hostnum = len(last_data)
        if lines_output and conf_host_show_diffstat_hostnum <= hostnum:
            cmpds = _ui_diffstats(data, last_data)
            updates = _ui_int(sum(d.rpms for d in last_data))
            _print_diffstats(hostnum, updates, cmpds)
            thostnum += 1

        if not lines_output:
            skipped_num += 1
            continue

        last_name = backup
        last_data = data
        skipped_num = 0
        thostnum += 2

        if conf_host_end_total_hostnum > 0:
            thostnum += hostnum
            if conf_host_end_total_hostnum <= thostnum:
                sys.exit(0) # Don't want to output "host data".

    if done:
        print("")
    print("Host data: %s" % (_ui_t_time(last_name),), file=sys.stderr)
    _print_lines(' ', last_data)


def _cmdline_arg_ansi(oval):
    val = oval.lower()
    if val in ("yes", "on", "1", "always"):
        return True
    if val in ("no", "off", "0", "never"):
        return False
    if val in ("automatic", "?", "tty", "auto"):
        return None
    raise argparse.ArgumentTypeError(f"{oval} is not valid: always/never/auto")

def _cmdline_arg_duration(oval):
    val = parse_duration(oval)
    if val is None:
        raise argparse.ArgumentTypeError(f"{oval} is not a duration")
    return val

def _cmdline_arg_positive_integer(oval):
    try:
        val = int(oval)
    except:
        val = -1
    if val <= 0:
        raise argparse.ArgumentTypeError(f"{oval} is not a positive integer")
    return val

def _cmdline_arg_hist(oval):
    #  The big problem here is that this happens before conf_* is fully loaded
    # so we can't trust backups[] yet. So defer everything until fname1().
    names = ("main", "latest", "newest", "diff-latest", "diff-newest", "oldest",
             "today", "yesterday")
    if oval in names:
        return oval

    if oval == "today": # Special ... might be nice to error on these, sigh...
        if fname_today is not None:
            return backup_today
        raise argparse.ArgumentTypeError(f"No history file for today")
    if oval == "yesterday" and fname_yesterday is not None:
        if fname_yesterday is not None:
            return backup_yesterday
        raise argparse.ArgumentTypeError(f"No history file for yesterday")

    if oval in backups:
        return oval

    msg = f"{oval} is not a history file"
    msg += "\n  History:", ", ".join([] + names + backups)
    raise argparse.ArgumentTypeError(msg)


def _cmd_help(args):
    prog = "updates+uptime"
    if sys.argv:
        prog = os.path.basename(sys.argv[0])
    if not args.hcmd:
        _usage()
    elif args.hcmd in ("diff", "diff-u"):
        print(f"""\
 Usage: {prog} diff [backup1] [backup2]
 
 See the difference between the current state and backups.
 The -u variant shows before/after instead of modified.
 
 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} yesterday
     {prog} {args.hcmd} 2025-08-16 main
 """, end='')
    elif args.hcmd in ("history", "hist"):
        print(f"""\
 Usage: {prog} {args.hcmd}
 
 Show summary of current data, and how it changed over time.
 
 Kind of like: git log --pretty=oneline --abbrev-commit --decorate

 Eg. {prog} {args.hcmd}
""", end='')
    elif args.hcmd == "history-keep":
        print(f"""\
 Usage: {prog} {args.hcmd} [days]

 Cleanup old history, older than the given number of days.
 The main file and today's history have to be kept, which happens
if you pass "1".

 Logrotate, for history.
 
 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} 32
""", end='')
    elif args.hcmd in ("host", "hosts", "host-u", "hosts-u"):
        print(f"""\
 Usage: {prog} {args.hcmd}
 
 See the history of a host(s). A cross between looking at diff and history.
 The -u variants show before/after instead of modified.

 Easiest way to see how hosts have changed over time, the more history the
better for this. Kind of like git blame, but instead of lines it's events.

 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} 'batcave*'
     {prog} {args.hcmd} 'batcave*' 'noc*'
""", end='')
    elif args.hcmd in ("information", "info"):
        print(f"""\
 Usage: {prog} {args.hcmd} [host*] [backup] [backup]...
 
 See the current state, in long form, can be filtered by name.

 If you want to compare things by hand, use this.

 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} 'batcave*'
     {prog} {args.hcmd} 'noc*' main yesterday
""", end='')
    elif args.hcmd in ("list",):
        print(f"""\
 Usage: {prog} {args.hcmd} [host*] [backup]
 
 See the current state of the hosts, can be filtered by name.
 
 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} 'batcave*'
     {prog} {args.hcmd} 'noc*' yesterday
""", end='')
    elif args.hcmd in ("list-n",):
        print(f"""\
 Usage: {prog} {args.hcmd} [host*] [host*]...
 
 See the current state of the hosts, can be filtered by name.
 
 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} 'batcave*'
     {prog} {args.hcmd} 'batcave*' 'noc*'
""", end='')
    elif args.hcmd in ("old-list", ):
        print(f"""\
 Usage: {prog} {args.hcmd} duration
 
 See the current state of hosts, with data older than duration.

 Easiest way to see what hosts we aren't getting data for.

 Eg. {prog} {args.hcmd} 2d
""", end='')
    elif args.hcmd in ("oslist",):
        print(f"""\
 Usage: {prog} {args.hcmd} [os*] [backup]
 
 See the current state, can be filtered by OS.
 
 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} RedHat
     {prog} {args.hcmd} 10 yesterday
""", end='')
    elif args.hcmd in ("oslist-n",):
        print(f"""\
 Usage: {prog} {args.hcmd} [os*] [os*]...
 
 See the current state, can be filtered by OS.
 
 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} RedHat
     {prog} {args.hcmd} F 10
""", end='')
    elif args.hcmd in ("statistics", "stats"):
        print(f"""\
 Usage: {prog} {args.hcmd} [backup] [host*] [host*]...
 
 Show general statistics.

 Easiest way to see the current state of the hosts.
 
 Eg. {prog} {args.hcmd}
     {prog} {args.hcmd} yesterday
     {prog} {args.hcmd} newest '*.stg.*' '*-test.*'
""", end='')
    elif args.hcmd in ("update",):
        print(f"""\
 Usage: {prog} {args.hcmd}
 
 Run update-fast, or update-daily if no daily backup.

 Easiest way to update, from cron or cmdline after you change things. DTRT.

 Eg. {prog} {args.hcmd}
""", end='')
    elif args.hcmd in ("update-fast",):
        print(f"""\
 Usage: {prog} {args.hcmd}
 
 Update the data for the hosts, in the main file (creating it if needed).
 
 Eg. {prog} {args.hcmd}
""", end='')
    elif args.hcmd in ("update-host",):
        print(f"""\
 Usage: {prog} {args.hcmd} host*
 
 Run update-fast, only for the specified host(s).
 
 Eg. {prog} {args.hcmd} bat\*
""", end='')
    elif args.hcmd in ("update-daily",):
        print(f"""\
 Usage: {prog} {args.hcmd}
 
 Run update-fast and force do a backup for today.
 
 Eg. {prog} {args.hcmd}
""", end='')
    elif args.hcmd in ("update-daily-refresh",):
        print(f"""\
 Usage: {prog} {args.hcmd}
 
 Delete the current file, then run update and also force do a backup for today.

 Easiest way to refresh the current history for today.

 Eg. {prog} {args.hcmd}
""", end='')
    elif args.hcmd in ("uptime", "uptime-min"):
        print(f"""\
 Usage: {prog} {args.hcmd} duration [backup]
 
 See the current state, can be filtered for uptime >= duration.

 Easy way to see what has been rebooted recently.

 Eg. {prog} {args.hcmd} 32h
     {prog} {args.hcmd} 1d yesterday
""", end='')
    elif args.hcmd in ("uptime-max",):
        print(f"""\
 Usage: {prog} {args.hcmd} duration [backup]
 
 See the current state, can be filtered for uptime <= duration.

 Easy way to see what hasn't been rebooted recently.

 Eg. {prog} {args.hcmd} 26w
     {prog} {args.hcmd} 4w4d yesterday
""", end='')
    else:
        print(" Unknown command:", args.hcmd)
        _usage()

def _main():
    global conf_ansi_terminal
    global conf_path
    global cmd

    _user_conf()

    parser = argparse.ArgumentParser(add_help=False)
    parser.add_argument('--verbose', '-v', action='count', default=0)
    parser.add_argument("--conf", action='append', default=[])
    parser.add_argument("--db-dir")
    parser.add_argument("--ansi", type=_cmdline_arg_ansi,
                        help="Use ansi terminal codes")
    parser.add_argument("--colour", type=_cmdline_arg_ansi, dest='ansi',
                        help=argparse.SUPPRESS)
    parser.add_argument("--color", type=_cmdline_arg_ansi, dest='ansi',
                        help=argparse.SUPPRESS)
    parser.add_argument('-h', '--help', action='store_true',
                        help='Show this help message')

    # We do this here so that `$0 -v stats -v` works.
    margs, args = parser.parse_known_args()

    if margs.help:
        _usage(short=True)
        sys.exit(0)

    subparsers = parser.add_subparsers(dest="cmd")

    cmd = subparsers.add_parser("help")
    cmd.add_argument("hcmd", nargs='?', help="cmd to get help for")
    cmd.set_defaults(func=_cmd_help)

    def __defs(func):
        cmd.set_defaults(func=func)

    # parser.add_argument('rest', nargs='*', type=int)

    # HIDDEN commands...
    cmd = subparsers.add_parser("dur2secs", help=argparse.SUPPRESS)
    cmd.add_argument("dur", type=_cmdline_arg_duration, help="duration")
    __defs(func=lambda x: print("secs:", x.dur))

    cmd = subparsers.add_parser("secs2dur", help=argparse.SUPPRESS)
    cmd.add_argument("secs", type=int, help="seconds")
    __defs(func=lambda x: print("dur:", _ui_dur(x.secs)))

    cmd = subparsers.add_parser("int2num", help=argparse.SUPPRESS)
    cmd.add_argument("num", type=int, help="int")
    __defs(func=lambda x: print("num:", _ui_int(x.num)))

    # -- Start of the real commands...

    # diff/diff-u commands
    cmd = subparsers.add_parser("diff", aliases=['diff-u'], help="diff")
    cmd.add_argument("hists", nargs='*', type=_cmdline_arg_hist, help="history file")
    __defs(func=_cmd_diff)

    # hosts/hosts-u commands
    als = ['host', 'hosts-u', 'host-u']
    hlp = "show history data about specific hosts"
    cmd = subparsers.add_parser("hosts", aliases=als, help=hlp)
    cmd.add_argument("hosts", nargs='*', help="wildcard hostname(s)")
    __defs(func=_cmd_host)

    # history command
    cmd = subparsers.add_parser("history", aliases=['hist'], help="show history")
    __defs(func=_cmd_history)

    # history-keep command
    cmd = subparsers.add_parser("history-keep", help="remove old files")
    hlp = "number of history files to keep"
    cmd.add_argument("keep", nargs='?', default=8, 
                     type=_cmdline_arg_positive_integer, help=hlp)
    __defs(func=_cmd_history_keep)

    # info command
    hlp = "show host information"
    cmd = subparsers.add_parser("information", aliases=['info'], help=hlp)
    cmd.add_argument("host", nargs='?', help="wildcard hostname")
    cmd.add_argument("hists", nargs='*', type=_cmdline_arg_hist, help="history file")
    __defs(func=_cmd_info)

    # list/list-n/old-list/olist/oslist/oslist-n commands
    cmd = subparsers.add_parser("list", help="list hosts")
    cmd.add_argument("host", nargs='?', help="wildcard hostname")
    cmd.add_argument("hists", nargs='*', type=_cmdline_arg_hist, help="history file")
    __defs(func=_cmd_list)

    cmd = subparsers.add_parser("list-n", help="list hosts")
    cmd.add_argument("hosts", nargs='*', help="wildcard hostname(s)")
    __defs(func=_cmd_list)

    cmd = subparsers.add_parser("old-list", aliases=['olist'],help="list hosts")
    cmd.add_argument("dateage", type=_cmdline_arg_duration,
                     help="data age minimum duration")
    __defs(func=_cmd_list)

    cmd = subparsers.add_parser("oslist", help="list hosts")
    cmd.add_argument("osname", nargs='?', help="wildcard OSname")
    cmd.add_argument("hists", nargs='*', type=_cmdline_arg_hist, help="history file")
    __defs(func=_cmd_list)

    cmd = subparsers.add_parser("oslist-n", help="list hosts")
    cmd.add_argument("osnames", nargs='*', help="wildcard OSname(s)")
    __defs(func=_cmd_list)

    # stats commands
    hlp = "show stats information"
    cmd = subparsers.add_parser("statistics", aliases=['stats'], help=hlp)
    cmd.add_argument("hist", nargs='?', type=_cmdline_arg_hist, default="main",
                      help="history file")
    cmd.add_argument("hosts", nargs='*', help="wildcard hostname(s)")
    __defs(func=_cmd_stats)

    # update commands
    als = ['update-daily','update-daily-refresh', 'update-fast', 'update-flush']
    cmd = subparsers.add_parser("update", aliases=als, help="update DB")
    __defs(func=_cmd_update)

    cmd = subparsers.add_parser("update-host", help="update DB for hosts")
    cmd.add_argument("host", help="wildcard hostname")
    __defs(func=_cmd_update)

    # uptime/uptime-min/uptime-max commands
    cmd = subparsers.add_parser("uptime-max", help="list hosts")
    cmd.add_argument("dur", type=_cmdline_arg_duration,
                     help="uptime maximum duration")
    __defs(func=_cmd_uptime)

    als = ['uptime']
    cmd = subparsers.add_parser("uptime-min", help="list hosts", aliases=als)
    cmd.add_argument("dur", type=_cmdline_arg_duration,
                     help="uptime minimum duration")
    __defs(func=_cmd_uptime)

    #  Need to setup backup[] for cmd line validation ... but conf can change
    # so just validate format? And revalidate later?
    _pre_cmd__setup()

    # Parse the above options/cmds
    args = parser.parse_args(args)

    if margs.db_dir:
        conf_path = margs.db_dir

    for line in margs.conf:
        _user_conf_line(line)

    if margs.ansi is not None:
        conf_ansi_terminal = margs.ansi

    # Setup based on the config.
    _pre_cmd__setup()

    _pre_cmd__verbose(margs)

    _pre_cmd__check_paths()

    # Run the actual command.
    if not hasattr(args, "func"):
        cmd = "diff"
        args.hists = []
        _cmd_diff(args)
    else:
        cmd = args.cmd
        args.func(args)


if __name__ == "__main__":
    _main()