mirror of
https://pagure.io/fedora-infra/ansible.git
synced 2026-02-02 20:59:02 +08:00
playbooks/updates-uptimes: Add distro. to data and script to view data.
Signed-off-by: James Antill <james@and.org>
This commit is contained in:
James Antill
@@ -31,7 +31,7 @@
|
||||
- name: Generate the Upgrade+uptime report
|
||||
ansible.builtin.lineinfile:
|
||||
regexp: '^{{inventory_hostname}} '
|
||||
line: "{{inventory_hostname}} {{pkgoutput.results|length}} {{ansible_uptime_seconds}} {{ansible_date_time['date']}}"
|
||||
line: "{{inventory_hostname}} {{pkgoutput.results|length}} {{ansible_uptime_seconds}} {{ansible_date_time['date']}} {{ansible_distribution}} {{ansible_distribution_version}}"
|
||||
path: /var/log/ansible-list-updates-uptime.txt
|
||||
create: yes
|
||||
delegate_to: localhost
|
||||
|
||||
403
scripts/updates-uptime-cmd.py
Executable file
403
scripts/updates-uptime-cmd.py
Executable file
@@ -0,0 +1,403 @@
|
||||
#! /usr/bin/python3
|
||||
|
||||
# Create/view a "txt" file using the ansible playbook
|
||||
# "generate-updates-uptimes-per-host-file.yml" which records the number of rpms
|
||||
# available to be upgraded for ansible hosts, and the uptime of those hosts.
|
||||
# This is very helpful when doing upgrade+reboot runs, as we can easily see
|
||||
# what has/hasn't been upgraded and/or rebooted.
|
||||
|
||||
# $0 update = create the file and/or do backups
|
||||
# $0 diff [x] [y] = see the difference between the current state and backups
|
||||
# $0 uptime [x] = see the current state, can be filtered for uptime >= x
|
||||
# $0 info [x] = see the current state, in long form, can be filtered by name
|
||||
# $0 host [x] = see the current state of a host(s), can be filtered by name
|
||||
# $0 list [x] = see the current state, can be filtered by name
|
||||
# $0 backups = see backups
|
||||
# $0 backups-keep = clenaup old backups
|
||||
# $0 stats [x] = see stats, can specify a backup
|
||||
|
||||
# Examples:
|
||||
# $0 update ... run it daily or so as root.
|
||||
# $0 diff ... see what changed.
|
||||
# $0 list '*.stg.*' ... see what staging looks like.
|
||||
# $0 list '*copr*' ... see what copr looks like.
|
||||
# $0 backups-keep 4 ... keep four days of backups (including today)
|
||||
# $0 uptime 1d ... see what hasn't been rebooted in the last 24 hours.
|
||||
# $0 uptime 25w ... see what hasn't been rebooted in too damn long
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import fnmatch
|
||||
import glob
|
||||
import time
|
||||
|
||||
path = "/var/log/"
|
||||
fname = path + "ansible-list-updates-uptime.txt"
|
||||
fname_today = fname + '.' + time.strftime("%Y-%m-%d")
|
||||
|
||||
backups = sorted(x.removeprefix(fname + '.') for x in glob.glob(fname + '.*'))
|
||||
|
||||
cmd = "diff"
|
||||
if len(sys.argv) >= 2:
|
||||
if sys.argv[1] in ("backups", "backups-keep", "diff", "diff-u", "help", "host", "info", "list", "stats", "update", "update-fast", "update-daily", "uptime",):
|
||||
cmd = sys.argv.pop(1)
|
||||
|
||||
_tm_d = {'d' : 60*60*24, 'h' : 60*60, 'm' : 60, 's' : 1,
|
||||
'w' : 60*60*24*7,
|
||||
'q' : 60*60*24*7*13}
|
||||
def parse_duration(seconds):
|
||||
if seconds is None:
|
||||
return None
|
||||
if seconds.isdigit():
|
||||
return int(seconds)
|
||||
|
||||
ret = 0
|
||||
for mark in ('w', 'd', 'h', 'm', 's'):
|
||||
pos = seconds.find(mark)
|
||||
if pos == -1:
|
||||
continue
|
||||
val = seconds[:pos]
|
||||
seconds = seconds[pos+1:]
|
||||
if not val.isdigit():
|
||||
# dbg("!isdigit", val)
|
||||
return None
|
||||
ret += _tm_d[mark]*int(val)
|
||||
if seconds.isdigit():
|
||||
ret += int(seconds)
|
||||
elif seconds != '':
|
||||
# dbg("!empty", seconds)
|
||||
return None
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def _add_dur(dur, ret, nummod, suffix, static=False):
|
||||
mod = dur % nummod
|
||||
dur = dur // nummod
|
||||
if mod > 0 or (static and dur > 0):
|
||||
ret.append(suffix)
|
||||
if static and dur > 0:
|
||||
ret.append("%0*d" % (len(str(nummod)), mod))
|
||||
else:
|
||||
ret.append(str(mod))
|
||||
return dur
|
||||
|
||||
def format_duration(seconds, static=False):
|
||||
if seconds is None:
|
||||
seconds = 0
|
||||
dur = int(seconds)
|
||||
|
||||
ret = []
|
||||
dur = _add_dur(dur, ret, 60, "s", static=static)
|
||||
dur = _add_dur(dur, ret, 60, "m", static=static)
|
||||
dur = _add_dur(dur, ret, 24, "h", static=static)
|
||||
dur = _add_dur(dur, ret, 7, "d", static=static)
|
||||
if dur > 0:
|
||||
ret.append("w")
|
||||
ret.append(str(dur))
|
||||
return "".join(reversed(ret))
|
||||
|
||||
|
||||
cmp_arg = False
|
||||
cmp = None
|
||||
def _cmp_arg():
|
||||
global cmp
|
||||
global cmp_arg
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
cmp = backups[-1] # Most recent
|
||||
elif sys.argv[1] not in backups:
|
||||
_usage()
|
||||
print("Backups:", ", ".join(backups))
|
||||
sys.exit(1)
|
||||
else:
|
||||
cmp = sys.argv[1]
|
||||
cmp_arg = True
|
||||
|
||||
def line2data(line):
|
||||
name, rpms, uptime, date = line.split(' ', 3)
|
||||
osname = "Unknown"
|
||||
osvers = "0"
|
||||
osinfo = "Unknown/?"
|
||||
if ' ' in date:
|
||||
date, osname, osvers = date.split(' ', 2)
|
||||
osinfo = osname + '/' + osvers
|
||||
if osname == 'CentOS':
|
||||
osinfo = 'EL/' + osvers
|
||||
if osname == 'Redhat':
|
||||
osinfo = 'EL/' + osvers
|
||||
if osname == 'Fedora':
|
||||
osinfo = 'F/' + osvers
|
||||
|
||||
updates = name + ' ' + rpms
|
||||
|
||||
rpms = int(rpms)
|
||||
uptime = int(uptime)
|
||||
|
||||
return locals()
|
||||
|
||||
def fname2lines(fname):
|
||||
return [x.strip() for x in open(fname).readlines()]
|
||||
|
||||
def bfname2lines(b):
|
||||
return fname2lines(fname + '.' + b)
|
||||
|
||||
def fname1():
|
||||
if cmp_arg:
|
||||
return bfname2lines(cmp)
|
||||
return fname2lines(fname)
|
||||
|
||||
def _usage():
|
||||
prog = "updates+uptime"
|
||||
if sys.argv:
|
||||
prog = os.path.basename(sys.argv[0])
|
||||
pl = " " * len(prog)
|
||||
print("""
|
||||
Usage: %s <cmd>
|
||||
Cmds:
|
||||
help
|
||||
= This message.
|
||||
|
||||
backups
|
||||
= Show backups.
|
||||
backups-trim [days]
|
||||
= Cleanup old backups.
|
||||
|
||||
diff [backup1] [backup2]
|
||||
diff-u [backup1] [backup2]
|
||||
= See the difference between the current state and backups.
|
||||
|
||||
host [host*] [backup]
|
||||
= See the current state of a host(s), can be filtered by name.
|
||||
info [host*] [backup]
|
||||
= See the current state, in long form, can be filtered by name.
|
||||
list [host*] [backup]
|
||||
= See the current state, can be filtered by name.
|
||||
|
||||
stats [backup]
|
||||
= Show stats.
|
||||
|
||||
update
|
||||
= Create the file and/or do backups.
|
||||
|
||||
uptime [duration] [backup]
|
||||
= See the current state, can be filtered for uptime >= duration.
|
||||
""" % (prog,))
|
||||
|
||||
if cmd == "help":
|
||||
_usage()
|
||||
|
||||
if cmd == "backups":
|
||||
print("Backups:")
|
||||
for backup in backups:
|
||||
print('', backup, len(bfname2lines(backup)))
|
||||
|
||||
if cmd == "backups-keep":
|
||||
keep = 8
|
||||
if len(sys.argv) >= 2:
|
||||
keep = int(sys.argv.pop(1))
|
||||
if keep <= 0:
|
||||
_usage()
|
||||
sys.exit(1)
|
||||
while keep < len(backups):
|
||||
# We just keep the newest N
|
||||
fn = fname + '.' + backups.pop()
|
||||
os.unlink(fn)
|
||||
|
||||
if cmd == "update":
|
||||
if not os.path.exists(fname):
|
||||
cmd = "update-daily" # This does the sorting etc.
|
||||
elif not os.path.exists(fname_today):
|
||||
cmd = "update-daily"
|
||||
else:
|
||||
mtime = os.path.getmtime(fname)
|
||||
if (time.now() - mtime) <= (60*60*8): # 8 hours.
|
||||
cmd = "update-fast"
|
||||
|
||||
if cmd == "update": # Get the latest uptime.
|
||||
os.chdir("/srv/web/infra/ansible/playbooks")
|
||||
os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates --flush-cache")
|
||||
if cmd == "update-fast": # Use ansible FACT cache for uptime.
|
||||
os.chdir("/srv/web/infra/ansible/playbooks")
|
||||
os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml -t updates")
|
||||
if cmd == "update-daily": # Also create backup file.
|
||||
os.chdir("/srv/web/infra/ansible/playbooks")
|
||||
os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml --flush-cache")
|
||||
|
||||
if cmd == "stats":
|
||||
_cmp_arg()
|
||||
data = fname1()
|
||||
osinfo = {}
|
||||
print("Hosts:", len(data))
|
||||
most = 0, None
|
||||
updates = 0
|
||||
awake = 0
|
||||
awakest = 0, None
|
||||
for line in data:
|
||||
d2 = line2data(line)
|
||||
|
||||
if d2['osname'] not in osinfo:
|
||||
osinfo[d2['osname']] = 0
|
||||
osinfo[d2['osname']] += 1
|
||||
if d2['osinfo'] not in osinfo:
|
||||
osinfo[d2['osinfo']] = 0
|
||||
osinfo[d2['osinfo']] += 1
|
||||
updates += d2['rpms']
|
||||
if d2['rpms'] >= most[0]:
|
||||
most = d2['rpms'], d2['name']
|
||||
|
||||
awake += d2['uptime']
|
||||
if d2['uptime'] >= awakest[0]:
|
||||
awakest = d2['uptime'], d2['name']
|
||||
for osi in sorted(osinfo):
|
||||
print(" Hosts:", osi, osinfo[osi])
|
||||
print("Updates:", updates)
|
||||
print(" Most:", most[1], most[0])
|
||||
print("Uptime:", format_duration(awake))
|
||||
print(" Most:", awakest[1], format_duration(awakest[0]))
|
||||
|
||||
def _print_info(host, lines):
|
||||
hosts = []
|
||||
for x in lines:
|
||||
x = line2data(x)
|
||||
if fnmatch.fnmatch(x['name'], host):
|
||||
hosts.append(x)
|
||||
if not hosts:
|
||||
print("Not found host:", host)
|
||||
sys.exit(2)
|
||||
for host in hosts:
|
||||
print("Host:", host['name'])
|
||||
print(" OS:", host['osinfo'], host['osname'], host['osvers'])
|
||||
print(" Rpms:", host['rpms'])
|
||||
print(" Uptime:", format_duration(host['uptime']))
|
||||
print(" Date:", host['date'])
|
||||
|
||||
if cmd in ("host", "info"):
|
||||
if cmd == "host":
|
||||
host = "batcave*"
|
||||
else:
|
||||
host = "*"
|
||||
if len(sys.argv) >= 2:
|
||||
host = sys.argv.pop(1)
|
||||
if len(sys.argv) >= 2 and sys.argv[1] == "all":
|
||||
for b in backups:
|
||||
print("Backup:", b)
|
||||
_print_info(bfname2lines(b))
|
||||
sys.argv = [sys.argv[0]]
|
||||
print("Main:")
|
||||
_cmp_arg()
|
||||
_print_info(host, fname1())
|
||||
|
||||
_max_len_name = 0
|
||||
_max_len_rpms = 0
|
||||
_max_len_updur = 0
|
||||
_max_len_date = 0 # 4+1+2+1+2
|
||||
def _max_update(lines):
|
||||
global _max_len_name
|
||||
global _max_len_rpms
|
||||
global _max_len_updur
|
||||
global _max_len_date
|
||||
|
||||
for line in lines:
|
||||
data = line2data(line)
|
||||
if len(data['name']) > _max_len_name:
|
||||
_max_len_name = len(data['name'])
|
||||
if len(str(data['rpms'])) > _max_len_rpms:
|
||||
_max_len_rpms = len(str(data['rpms']))
|
||||
updur_len = len(format_duration(data['uptime'], static=True))
|
||||
if updur_len > _max_len_updur:
|
||||
_max_len_updur = updur_len
|
||||
if len(data['date']) > _max_len_date:
|
||||
_max_len_date = len(data['date'])
|
||||
|
||||
def _print_line(prefix, data):
|
||||
print("%s%-*s %*u %*s %*s %s" % (prefix,
|
||||
_max_len_name, data['name'],
|
||||
_max_len_rpms, data['rpms'],
|
||||
_max_len_updur, format_duration(data['uptime'], static=True),
|
||||
_max_len_date, data['date'], data['osinfo']))
|
||||
|
||||
if cmd == "list":
|
||||
host = "*"
|
||||
if len(sys.argv) >= 2:
|
||||
host = sys.argv.pop(1)
|
||||
|
||||
_cmp_arg()
|
||||
lines = fname1()
|
||||
_max_update(lines)
|
||||
for line in lines:
|
||||
d1 = line2data(line)
|
||||
if not fnmatch.fnmatch(d1['name'], host):
|
||||
continue
|
||||
_print_line('', d1)
|
||||
|
||||
if cmd == "uptime":
|
||||
age = 0
|
||||
if len(sys.argv) >= 2:
|
||||
age = parse_duration(sys.argv.pop(1))
|
||||
|
||||
_cmp_arg()
|
||||
lines = fname1()
|
||||
_max_update(lines)
|
||||
for line in lines:
|
||||
d1 = line2data(line)
|
||||
if age > d1['uptime']:
|
||||
continue
|
||||
_print_line('', d1)
|
||||
|
||||
if cmd in ("diff", "diff-u"):
|
||||
_cmp_arg()
|
||||
fn1 = fname + '.' + cmp
|
||||
fn2 = fname
|
||||
data1 = fname2lines(fn1)
|
||||
if len(sys.argv) >= 2 and sys.argv[1] in backups:
|
||||
# Doing a diff. between two backups...
|
||||
fn2 = fname + '.' + sys.argv[1]
|
||||
data2 = fname2lines(fn2)
|
||||
print("diff %s %s" % (fn1, fn2), file=sys.stderr)
|
||||
_max_update(data1)
|
||||
_max_update(data2)
|
||||
while len(data1) > 0 or len(data2) > 0:
|
||||
if len(data1) <= 0:
|
||||
_print_line('+', line2data(data2[0]))
|
||||
data2.pop(0)
|
||||
continue
|
||||
if len(data2) <= 0:
|
||||
_print_line('-', line2data(data1[0]))
|
||||
data1.pop(0)
|
||||
continue
|
||||
|
||||
d1 = line2data(data1[0])
|
||||
d2 = line2data(data2[0])
|
||||
|
||||
if d1['name'] < d2['name']:
|
||||
_print_line('-', d1)
|
||||
data1.pop(0)
|
||||
continue
|
||||
|
||||
if d1['name'] > d2['name']:
|
||||
_print_line('+', d2)
|
||||
data2.pop(0)
|
||||
continue
|
||||
|
||||
if d1['updates'] == d2['updates']:
|
||||
_print_line(' ', d2)
|
||||
data1.pop(0)
|
||||
data2.pop(0)
|
||||
continue
|
||||
|
||||
if cmd == "diff-u":
|
||||
_print_line('-', d1)
|
||||
data1.pop(0)
|
||||
_print_line('+', d2)
|
||||
data2.pop(0)
|
||||
continue
|
||||
|
||||
# diff
|
||||
data1.pop(0)
|
||||
_print_line('!', d2)
|
||||
data2.pop(0)
|
||||
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user