#!/usr/bin/python3 -I
# omv-loadreport - report the processes behind high load, for Monit.
#
# Meant to be run by a Monit "check program". It samples the system twice,
# a few seconds apart, and checks the 1/5 min load average and the CPU
# usage (user/system/wait) against the given thresholds. If any threshold
# is exceeded it prints a short report of the top CPU and I/O consumers and
# the processes blocked in uninterruptible sleep (D state), then exits 1.
# Monit puts that output into $DESCRIPTION, so it lands in the normal
# monitoring email. Exit 0 means everything is within limits.
#
# Use --always to print the report regardless (handy for testing).
#
# License: GPL-3.0-or-later

import argparse
import json
import os
import re
import sys
import time

CLK_TCK = os.sysconf("SC_CLK_TCK")
NUM_CPUS = os.cpu_count() or 1
DOCKER_ROOT = "/var/lib/docker/containers"


def read(path):
    try:
        with open(path, "rb") as fh:
            return fh.read().decode("utf-8", "replace")
    except OSError:
        return None


def cpu_totals():
    # cpu  user nice system idle iowait irq softirq steal ...
    fields = [int(x) for x in read("/proc/stat").splitlines()[0].split()[1:]]
    fields += [0] * (8 - len(fields))
    user, nice, system, idle, iowait, irq, softirq, steal = fields[:8]
    return {
        "user": user + nice,
        "system": system + irq + softirq,
        "wait": iowait,
        "total": sum(fields[:8]),
    }


def snapshot():
    procs = {}
    for pid in os.listdir("/proc"):
        if not pid.isdigit():
            continue
        stat = read(f"/proc/{pid}/stat")
        if not stat:
            continue
        # comm may contain spaces and parens; it ends at the last ')'.
        rpar = stat.rfind(")")
        comm = stat[stat.find("(") + 1:rpar]
        rest = stat[rpar + 2:].split()
        state = rest[0]
        ticks = int(rest[11]) + int(rest[12])  # utime + stime
        rbytes = wbytes = 0
        io = read(f"/proc/{pid}/io")
        if io:
            for line in io.splitlines():
                key, _, val = line.partition(":")
                if key == "read_bytes":
                    rbytes = int(val)
                elif key == "write_bytes":
                    wbytes = int(val)
        procs[int(pid)] = (comm, state, ticks, rbytes, wbytes)
    return procs


_container_names = {}


def container_of(pid):
    cg = read(f"/proc/{pid}/cgroup") or ""
    m = re.search(r"(?:docker[-/]|/docker/)([0-9a-f]{64})", cg)
    if not m:
        return None
    cid = m.group(1)
    if cid not in _container_names:
        name = cid[:12]
        cfg = read(f"{DOCKER_ROOT}/{cid}/config.v2.json")
        if cfg:
            try:
                name = json.loads(cfg).get("Name", name).lstrip("/") or name
            except ValueError:
                pass
        _container_names[cid] = name
    return _container_names[cid]


def label(pid, comm):
    cmd = (read(f"/proc/{pid}/cmdline") or "").replace("\0", " ").strip()
    text = cmd or f"[{comm}]"
    if len(text) > 60:
        text = text[:57] + "..."
    ctr = container_of(pid)
    return f"{text} (container: {ctr})" if ctr else text


def human(rate):
    for unit in ("B", "K", "M", "G"):
        if rate < 1024 or unit == "G":
            return f"{rate:.0f}{unit}/s" if unit == "B" else f"{rate:.1f}{unit}/s"
        rate /= 1024


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--load1", type=float, help="1 min loadavg limit")
    ap.add_argument("--load5", type=float, help="5 min loadavg limit")
    ap.add_argument("--cpu-user", type=float, help="CPU user %% limit")
    ap.add_argument("--cpu-system", type=float, help="CPU system %% limit")
    ap.add_argument("--cpu-wait", type=float, help="CPU iowait %% limit")
    ap.add_argument("--interval", type=float, default=3.0,
                    help="sampling interval in seconds (default 3)")
    ap.add_argument("--top", type=int, default=5,
                    help="processes per list (default 5)")
    ap.add_argument("--always", action="store_true",
                    help="print the report even if no limit is exceeded")
    args = ap.parse_args()

    c0, p0 = cpu_totals(), snapshot()
    t0 = time.monotonic()
    time.sleep(args.interval)
    c1, p1 = cpu_totals(), snapshot()
    elapsed = time.monotonic() - t0

    dt = max(c1["total"] - c0["total"], 1)
    cpu = {k: 100.0 * (c1[k] - c0[k]) / dt for k in ("user", "system", "wait")}
    load = [float(x) for x in read("/proc/loadavg").split()[:3]]

    reasons = []
    if args.load1 is not None and load[0] > args.load1:
        reasons.append(f"loadavg(1min) {load[0]:.2f} > {args.load1:g}")
    if args.load5 is not None and load[1] > args.load5:
        reasons.append(f"loadavg(5min) {load[1]:.2f} > {args.load5:g}")
    for key in ("user", "system", "wait"):
        limit = getattr(args, f"cpu_{key}")
        if limit is not None and cpu[key] > limit:
            reasons.append(f"cpu {key} {cpu[key]:.1f}% > {limit:g}%")

    if not reasons and not args.always:
        return 0

    rows = []
    for pid, (comm, state, ticks, rb, wb) in p1.items():
        prev = p0.get(pid)
        if prev is None:
            # Started during the sample: count from zero.
            prev = (comm, state, 0, 0, 0)
        cpu_pct = 100.0 * (ticks - prev[2]) / CLK_TCK / elapsed
        rrate = max(rb - prev[3], 0) / elapsed
        wrate = max(wb - prev[4], 0) / elapsed
        rows.append((pid, comm, state, cpu_pct, rrate, wrate))

    out = []
    out.append("; ".join(reasons) if reasons else "report requested (--always)")
    out.append(
        f"load {load[0]:.2f} {load[1]:.2f} {load[2]:.2f} on {NUM_CPUS} CPUs | "
        f"cpu user {cpu['user']:.0f}% sys {cpu['system']:.0f}% "
        f"iowait {cpu['wait']:.0f}% (over {elapsed:.0f}s)"
    )

    top_cpu = [r for r in sorted(rows, key=lambda r: -r[3]) if r[3] >= 0.5]
    out.append("")
    out.append("Top CPU (% of one core):")
    for pid, comm, _, pct, _, _ in top_cpu[:args.top]:
        out.append(f"  {pct:6.1f}%  {pid:>7}  {label(pid, comm)}")
    if not top_cpu:
        out.append("  (nothing busy)")

    top_io = [r for r in sorted(rows, key=lambda r: -(r[4] + r[5]))
              if r[4] + r[5] > 0]
    out.append("")
    out.append("Top disk I/O (read / write):")
    for pid, comm, _, _, rr, wr in top_io[:args.top]:
        out.append(f"  {human(rr):>9} / {human(wr):<9} {pid:>7}  {label(pid, comm)}")
    if not top_io:
        out.append("  (no disk I/O)")

    blocked = [r for r in rows if r[2] == "D"]
    if blocked:
        out.append("")
        out.append(f"Blocked on I/O (D state, counts toward load): {len(blocked)}")
        for pid, comm, *_ in blocked[:args.top]:
            out.append(f"  {pid:>7}  {label(pid, comm)}")

    print("\n".join(out))
    return 1 if reasons else 0


if __name__ == "__main__":
    sys.exit(main())
