From 3c01596c382f9a87693cda80de0dff60c196e7b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Etxaniz?= Date: Mon, 9 Nov 2026 19:19:04 +0000 Subject: [PATCH] add restart signals to other machines --- README.md | 4 ++-- main.py | 55 +++++++++++++++++++++++++++++++++++++-------------- schedule.yaml | 2 +- signals.yml | 30 +++++++++++++++++++++++++--- 4 files changed, 70 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index ee559c8..d47800f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,6 @@ One row per session of the host in the custom table `host_session` (schema 0.0.1). The hub adds the gateway as `spoke_id`. -The first run after a boot opens a row for the current session: its start time, the same value as end time (the mark of an open session), everything else 0. In the same run it closes the previous session: it finds that session in the history of the uptime signal (63) before the boot (last sample is the end, end minus that sample's uptime is the start), reads CPU, RAM and disk at one minute, and fills that session's open row with the end time, duration, means, maxima and coverage (share of minutes with a CPU sample). A previous session with no row, such as the one running when the job was first installed, gets a new full row. Every later run in the same boot sees the current row and stops after one query. +The first run after a boot opens a row for the current session (start time equal to end time, everything else 0) and closes the previous one from the history of the host's uptime signal: CPU, RAM and disk at one minute, with coverage as the share of minutes with a CPU sample. Later runs in the same boot stop after one query. -Runs every minute. The current boot comes from `/proc/uptime`. Signal ids are deb13-kde's in the home core. Needs only `requests` in the image. +One script for every gateway. `signals.yml` lists the cpu, ram, disk and uptime signals of every host; the script asks the history of each `-uptime` until one answers 200 (a gateway answers 403 for signals it does not own) and uses that host's four signals. A new host needs its four signals added to `signals.yml` and a new tag. Runs every minute; needs only `requests` in the image. \ No newline at end of file diff --git a/main.py b/main.py index 6357d54..d82490a 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ import os +import re import time import requests @@ -6,7 +7,7 @@ import requests SIDECAR_URL = os.getenv("SIDECAR_URL", "http://sidecar:80") DB_URL = os.getenv("DB_CONTROLLER_URL", "http://sidecar:8081") TABLE = "host_session" -UPTIME, CPU, RAM, DISK = 63, 21, 22, 23 +SIGNALS_FILE = "/app/signals.yml" MINUTE_MS = 60000 HOUR_MS = 60 * MINUTE_MS LOOKBACK_MS = 60 * 24 * HOUR_MS @@ -18,7 +19,29 @@ def read_history(signals, start_ms, end_ms, resolution): path = f"/api/history/?signals={names}&start={int(start_ms)}&end={int(end_ms)}&resolution={resolution}" response = requests.get(SIDECAR_URL + path, timeout=(5, 80)) response.raise_for_status() - return {item["signalId"]: item for item in response.json()["data"]["series"]} + return {item["name"]: item for item in response.json()["data"]["series"]} + + +def signal_hosts(): + with open(SIGNALS_FILE) as file: + names = re.findall(r"name:\s*'([^']+)'", file.read()) + return sorted(name[: -len("-uptime")] for name in names if name.endswith("-uptime")) + + +def owns(host, now_ms): + path = f"/api/history/?signals={host}-uptime&start={int(now_ms - MINUTE_MS)}&end={int(now_ms)}&resolution=1min" + status = requests.get(SIDECAR_URL + path, timeout=(5, 80)).status_code + if status not in (200, 403): + raise RuntimeError(f"history probe for {host} answered {status}") + return status == 200 + + +def own_host(): + now_ms = time.time() * 1000 + host = next((name for name in signal_hosts() if owns(name, now_ms)), None) + if host is None: + raise RuntimeError("no uptime signal of this gateway is listed in signals.yml") + return host def samples(series): @@ -39,14 +62,15 @@ def current_boot_ms(): return int(time.time() * 1000 - uptime_s * 1000) -def previous_session(boot_ms): - hours = last_samples(UPTIME, boot_ms - LOOKBACK_MS, boot_ms, "1hr") +def previous_session(host, boot_ms): + uptime = f"{host}-uptime" + hours = last_samples(uptime, boot_ms - LOOKBACK_MS, boot_ms, "1hr") if not hours: return None - minutes = last_samples(UPTIME, hours[-1][0], min(hours[-1][0] + HOUR_MS, boot_ms), "1min") + minutes = last_samples(uptime, hours[-1][0], min(hours[-1][0] + HOUR_MS, boot_ms), "1min") if not minutes: return None - seconds = last_samples(UPTIME, minutes[-1][0], min(minutes[-1][0] + MINUTE_MS, boot_ms), "1s") + seconds = last_samples(uptime, minutes[-1][0], min(minutes[-1][0] + MINUTE_MS, boot_ms), "1s") end_ms, uptime_s = (seconds or minutes)[-1] return int(end_ms - uptime_s * 1000), int(end_ms) @@ -88,21 +112,22 @@ def coverage(series, start_ms, end_ms): return min(1.0, present / total) -def build_row(start_ms, end_ms): - history = read_history([CPU, RAM, DISK], start_ms, end_ms, "1min") +def build_row(host, start_ms, end_ms): + cpu, ram, disk = f"{host}-cpu", f"{host}-ram", f"{host}-disk" + history = read_history([cpu, ram, disk], start_ms, end_ms, "1min") row = {"start_time": start_ms, "end_time": end_ms, "duration_s": (end_ms - start_ms) // 1000} - for prefix, signal in (("cpu", CPU), ("ram", RAM), ("disk", DISK)): + for prefix, signal in (("cpu", cpu), ("ram", ram), ("disk", disk)): row[f"{prefix}_mean"], row[f"{prefix}_max"] = mean_max(history[signal], start_ms, end_ms) - row["coverage"] = coverage(history[CPU], start_ms, end_ms) + row["coverage"] = coverage(history[cpu], start_ms, end_ms) return row -def close_previous(rows, boot_ms): - session = previous_session(boot_ms) +def close_previous(host, rows, boot_ms): + session = previous_session(host, boot_ms) if session is None: print("no previous session in the uptime history", flush=True) return - row = build_row(*session) + row = build_row(host, *session) stored = find_row(rows, session[0]) if stored is None: print("previous session stored", db_call("POST", f"/api/db/{TABLE}/", row), row, flush=True) @@ -116,9 +141,9 @@ def main(): rows = stored_rows() if find_row(rows, boot_ms) is not None: return - close_previous(rows, boot_ms) + close_previous(own_host(), rows, boot_ms) print("current session opened", db_call("POST", f"/api/db/{TABLE}/", open_row(boot_ms)), flush=True) if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/schedule.yaml b/schedule.yaml index cdcae6a..4b4f0a9 100644 --- a/schedule.yaml +++ b/schedule.yaml @@ -9,4 +9,4 @@ command: ["python", "-u", "/app/main.py"] maxCpuCores: 0.25 maxMemoryMb: 128 autoremove: false -logOption: 1 +logOption: 1 \ No newline at end of file diff --git a/signals.yml b/signals.yml index 34d287c..7319824 100644 --- a/signals.yml +++ b/signals.yml @@ -1,4 +1,28 @@ -- { id: 21, name: 'deb13kde-cpu', read: 1, write: 0 } -- { id: 22, name: 'deb13kde-ram', read: 1, write: 0 } -- { id: 23, name: 'deb13kde-disk', read: 1, write: 0 } +- { id: 21, name: 'deb13kde-cpu', read: 1, write: 0 } +- { id: 22, name: 'deb13kde-ram', read: 1, write: 0 } +- { id: 23, name: 'deb13kde-disk', read: 1, write: 0 } - { id: 63, name: 'deb13kde-uptime', read: 1, write: 0 } +- { id: 1, name: 'rbx-01-cpu', read: 1, write: 0 } +- { id: 2, name: 'rbx-01-ram', read: 1, write: 0 } +- { id: 3, name: 'rbx-01-disk', read: 1, write: 0 } +- { id: 64, name: 'rbx-01-uptime', read: 1, write: 0 } +- { id: 6, name: 'fr-01-cpu', read: 1, write: 0 } +- { id: 7, name: 'fr-01-ram', read: 1, write: 0 } +- { id: 8, name: 'fr-01-disk', read: 1, write: 0 } +- { id: 65, name: 'fr-01-uptime', read: 1, write: 0 } +- { id: 11, name: 'sb-01-cpu', read: 1, write: 0 } +- { id: 12, name: 'sb-01-ram', read: 1, write: 0 } +- { id: 13, name: 'sb-01-disk', read: 1, write: 0 } +- { id: 66, name: 'sb-01-uptime', read: 1, write: 0 } +- { id: 16, name: 'grab-01-cpu', read: 1, write: 0 } +- { id: 17, name: 'grab-01-ram', read: 1, write: 0 } +- { id: 18, name: 'grab-01-disk', read: 1, write: 0 } +- { id: 67, name: 'grab-01-uptime', read: 1, write: 0 } +- { id: 26, name: 'pi102-cpu', read: 1, write: 0 } +- { id: 27, name: 'pi102-ram', read: 1, write: 0 } +- { id: 28, name: 'pi102-disk', read: 1, write: 0 } +- { id: 68, name: 'pi102-uptime', read: 1, write: 0 } +- { id: 32, name: 'pi103-cpu', read: 1, write: 0 } +- { id: 33, name: 'pi103-ram', read: 1, write: 0 } +- { id: 34, name: 'pi103-disk', read: 1, write: 0 } +- { id: 69, name: 'pi103-uptime', read: 1, write: 0 }