add restart signals to other machines

README.md

@@ -2,6 +2,6 @@
 
 One row per session of the host in the custom table `host_session` (schema 0.0.1). The hub adds the gateway as `spoke_id`.
 
-The first run after a boot opens a row for the current session: its start time, the same value as end time (the mark of an open session), everything else 0. In the same run it closes the previous session: it finds that session in the history of the uptime signal (63) before the boot (last sample is the end, end minus that sample's uptime is the start), reads CPU, RAM and disk at one minute, and fills that session's open row with the end time, duration, means, maxima and coverage (share of minutes with a CPU sample). A previous session with no row, such as the one running when the job was first installed, gets a new full row. Every later run in the same boot sees the current row and stops after one query.
+The first run after a boot opens a row for the current session (start time equal to end time, everything else 0) and closes the previous one from the history of the host's uptime signal: CPU, RAM and disk at one minute, with coverage as the share of minutes with a CPU sample. Later runs in the same boot stop after one query.
 
-Runs every minute. The current boot comes from `/proc/uptime`. Signal ids are deb13-kde's in the home core. Needs only `requests` in the image.
+One script for every gateway. `signals.yml` lists the cpu, ram, disk and uptime signals of every host; the script asks the history of each `<host>-uptime` until one answers 200 (a gateway answers 403 for signals it does not own) and uses that host's four signals. A new host needs its four signals added to `signals.yml` and a new tag. Runs every minute; needs only `requests` in the image.

main.py

@@ -1,4 +1,5 @@
 import os
+import re
 import time
 
 import requests
@@ -6,7 +7,7 @@ import requests
 SIDECAR_URL = os.getenv("SIDECAR_URL", "http://sidecar:80")
 DB_URL = os.getenv("DB_CONTROLLER_URL", "http://sidecar:8081")
 TABLE = "host_session"
-UPTIME, CPU, RAM, DISK = 63, 21, 22, 23
+SIGNALS_FILE = "/app/signals.yml"
 MINUTE_MS = 60000
 HOUR_MS = 60 * MINUTE_MS
 LOOKBACK_MS = 60 * 24 * HOUR_MS
@@ -18,7 +19,29 @@ def read_history(signals, start_ms, end_ms, resolution):
     path = f"/api/history/?signals={names}&start={int(start_ms)}&end={int(end_ms)}&resolution={resolution}"
     response = requests.get(SIDECAR_URL + path, timeout=(5, 80))
     response.raise_for_status()
-    return {item["signalId"]: item for item in response.json()["data"]["series"]}
+    return {item["name"]: item for item in response.json()["data"]["series"]}
+
+
+def signal_hosts():
+    with open(SIGNALS_FILE) as file:
+        names = re.findall(r"name:\s*'([^']+)'", file.read())
+    return sorted(name[: -len("-uptime")] for name in names if name.endswith("-uptime"))
+
+
+def owns(host, now_ms):
+    path = f"/api/history/?signals={host}-uptime&start={int(now_ms - MINUTE_MS)}&end={int(now_ms)}&resolution=1min"
+    status = requests.get(SIDECAR_URL + path, timeout=(5, 80)).status_code
+    if status not in (200, 403):
+        raise RuntimeError(f"history probe for {host} answered {status}")
+    return status == 200
+
+
+def own_host():
+    now_ms = time.time() * 1000
+    host = next((name for name in signal_hosts() if owns(name, now_ms)), None)
+    if host is None:
+        raise RuntimeError("no uptime signal of this gateway is listed in signals.yml")
+    return host
 
 
 def samples(series):
@@ -39,14 +62,15 @@ def current_boot_ms():
     return int(time.time() * 1000 - uptime_s * 1000)
 
 
-def previous_session(boot_ms):
-    hours = last_samples(UPTIME, boot_ms - LOOKBACK_MS, boot_ms, "1hr")
+def previous_session(host, boot_ms):
+    uptime = f"{host}-uptime"
+    hours = last_samples(uptime, boot_ms - LOOKBACK_MS, boot_ms, "1hr")
     if not hours:
         return None
-    minutes = last_samples(UPTIME, hours[-1][0], min(hours[-1][0] + HOUR_MS, boot_ms), "1min")
+    minutes = last_samples(uptime, hours[-1][0], min(hours[-1][0] + HOUR_MS, boot_ms), "1min")
     if not minutes:
         return None
-    seconds = last_samples(UPTIME, minutes[-1][0], min(minutes[-1][0] + MINUTE_MS, boot_ms), "1s")
+    seconds = last_samples(uptime, minutes[-1][0], min(minutes[-1][0] + MINUTE_MS, boot_ms), "1s")
     end_ms, uptime_s = (seconds or minutes)[-1]
     return int(end_ms - uptime_s * 1000), int(end_ms)
 
@@ -88,21 +112,22 @@ def coverage(series, start_ms, end_ms):
     return min(1.0, present / total)
 
 
-def build_row(start_ms, end_ms):
-    history = read_history([CPU, RAM, DISK], start_ms, end_ms, "1min")
+def build_row(host, start_ms, end_ms):
+    cpu, ram, disk = f"{host}-cpu", f"{host}-ram", f"{host}-disk"
+    history = read_history([cpu, ram, disk], start_ms, end_ms, "1min")
     row = {"start_time": start_ms, "end_time": end_ms, "duration_s": (end_ms - start_ms) // 1000}
-    for prefix, signal in (("cpu", CPU), ("ram", RAM), ("disk", DISK)):
+    for prefix, signal in (("cpu", cpu), ("ram", ram), ("disk", disk)):
         row[f"{prefix}_mean"], row[f"{prefix}_max"] = mean_max(history[signal], start_ms, end_ms)
-    row["coverage"] = coverage(history[CPU], start_ms, end_ms)
+    row["coverage"] = coverage(history[cpu], start_ms, end_ms)
     return row
 
 
-def close_previous(rows, boot_ms):
-    session = previous_session(boot_ms)
+def close_previous(host, rows, boot_ms):
+    session = previous_session(host, boot_ms)
     if session is None:
         print("no previous session in the uptime history", flush=True)
         return
-    row = build_row(*session)
+    row = build_row(host, *session)
     stored = find_row(rows, session[0])
     if stored is None:
         print("previous session stored", db_call("POST", f"/api/db/{TABLE}/", row), row, flush=True)
@@ -116,9 +141,9 @@ def main():
     rows = stored_rows()
     if find_row(rows, boot_ms) is not None:
         return
-    close_previous(rows, boot_ms)
+    close_previous(own_host(), rows, boot_ms)
     print("current session opened", db_call("POST", f"/api/db/{TABLE}/", open_row(boot_ms)), flush=True)
 
 
 if __name__ == "__main__":
-    main()
+    main()

schedule.yaml

@@ -9,4 +9,4 @@ command: ["python", "-u", "/app/main.py"]
 maxCpuCores: 0.25
 maxMemoryMb: 128
 autoremove: false
-logOption: 1
+logOption: 1

signals.yml

@@ -1,4 +1,28 @@
-- { id: 21, name: 'deb13kde-cpu',    read: 1, write: 0 }
-- { id: 22, name: 'deb13kde-ram',    read: 1, write: 0 }
-- { id: 23, name: 'deb13kde-disk',   read: 1, write: 0 }
+- { id: 21, name: 'deb13kde-cpu', read: 1, write: 0 }
+- { id: 22, name: 'deb13kde-ram', read: 1, write: 0 }
+- { id: 23, name: 'deb13kde-disk', read: 1, write: 0 }
 - { id: 63, name: 'deb13kde-uptime', read: 1, write: 0 }
+- { id: 1, name: 'rbx-01-cpu', read: 1, write: 0 }
+- { id: 2, name: 'rbx-01-ram', read: 1, write: 0 }
+- { id: 3, name: 'rbx-01-disk', read: 1, write: 0 }
+- { id: 64, name: 'rbx-01-uptime', read: 1, write: 0 }
+- { id: 6, name: 'fr-01-cpu', read: 1, write: 0 }
+- { id: 7, name: 'fr-01-ram', read: 1, write: 0 }
+- { id: 8, name: 'fr-01-disk', read: 1, write: 0 }
+- { id: 65, name: 'fr-01-uptime', read: 1, write: 0 }
+- { id: 11, name: 'sb-01-cpu', read: 1, write: 0 }
+- { id: 12, name: 'sb-01-ram', read: 1, write: 0 }
+- { id: 13, name: 'sb-01-disk', read: 1, write: 0 }
+- { id: 66, name: 'sb-01-uptime', read: 1, write: 0 }
+- { id: 16, name: 'grab-01-cpu', read: 1, write: 0 }
+- { id: 17, name: 'grab-01-ram', read: 1, write: 0 }
+- { id: 18, name: 'grab-01-disk', read: 1, write: 0 }
+- { id: 67, name: 'grab-01-uptime', read: 1, write: 0 }
+- { id: 26, name: 'pi102-cpu', read: 1, write: 0 }
+- { id: 27, name: 'pi102-ram', read: 1, write: 0 }
+- { id: 28, name: 'pi102-disk', read: 1, write: 0 }
+- { id: 68, name: 'pi102-uptime', read: 1, write: 0 }
+- { id: 32, name: 'pi103-cpu', read: 1, write: 0 }
+- { id: 33, name: 'pi103-ram', read: 1, write: 0 }
+- { id: 34, name: 'pi103-disk', read: 1, write: 0 }
+- { id: 69, name: 'pi103-uptime', read: 1, write: 0 }