infra/host_session
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add restart signals to other machines

Browse Source
This commit is contained in:
Iñigo Etxaniz
2026-11-09 19:19:04 +00:00
parent eff8ea4dfe
commit 3c01596c38
4 changed files with 70 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -2,6 +2,6 @@
One row per session of the host in the custom table `host_session` (schema 0.0.1). The hub adds the gateway as `spoke_id`.
The first run after a boot opens a row for the current session: its start time, the same value as end time (the mark of an open session), everything else 0. In the same run it closes the previous session: it finds that session in the history of the uptime signal (63) before the boot (last sample is the end, end minus that sample's uptime is the start), reads CPU, RAM and disk at one minute, and fills that session's open row with the end time, duration, means, maxima and coverage (share of minutes with a CPU sample). A previous session with no row, such as the one running when the job was first installed, gets a new full row. Every later run in the same boot sees the current row and stops after one query.
The first run after a boot opens a row for the current session (start time equal to end time, everything else 0) and closes the previous one from the history of the host's uptime signal: CPU, RAM and disk at one minute, with coverage as the share of minutes with a CPU sample. Later runs in the same boot stop after one query.
Runs every minute. The current boot comes from `/proc/uptime`. Signal ids are deb13-kde's in the home core. Needs only `requests` in the image.
One script for every gateway. `signals.yml` lists the cpu, ram, disk and uptime signals of every host; the script asks the history of each `<host>-uptime` until one answers 200 (a gateway answers 403 for signals it does not own) and uses that host's four signals. A new host needs its four signals added to `signals.yml` and a new tag. Runs every minute; needs only `requests` in the image.
main.py
+40 -15
View File
@@ -1,4 +1,5 @@
import os
import re
import time
import requests
@@ -6,7 +7,7 @@ import requests
SIDECAR_URL = os.getenv("SIDECAR_URL", "http://sidecar:80")
DB_URL = os.getenv("DB_CONTROLLER_URL", "http://sidecar:8081")
TABLE = "host_session"
UPTIME, CPU, RAM, DISK = 63, 21, 22, 23
SIGNALS_FILE = "/app/signals.yml"
MINUTE_MS = 60000
HOUR_MS = 60 * MINUTE_MS
LOOKBACK_MS = 60 * 24 * HOUR_MS
@@ -18,7 +19,29 @@ def read_history(signals, start_ms, end_ms, resolution):
path = f"/api/history/?signals={names}&start={int(start_ms)}&end={int(end_ms)}&resolution={resolution}"
response = requests.get(SIDECAR_URL + path, timeout=(5, 80))
response.raise_for_status()
return {item["signalId"]: item for item in response.json()["data"]["series"]}
return {item["name"]: item for item in response.json()["data"]["series"]}
def signal_hosts():
with open(SIGNALS_FILE) as file:
names = re.findall(r"name:\s*'([^']+)'", file.read())
return sorted(name[: -len("-uptime")] for name in names if name.endswith("-uptime"))
def owns(host, now_ms):
path = f"/api/history/?signals={host}-uptime&start={int(now_ms - MINUTE_MS)}&end={int(now_ms)}&resolution=1min"
status = requests.get(SIDECAR_URL + path, timeout=(5, 80)).status_code
if status not in (200, 403):
raise RuntimeError(f"history probe for {host} answered {status}")
return status == 200
def own_host():
now_ms = time.time() * 1000
host = next((name for name in signal_hosts() if owns(name, now_ms)), None)
if host is None:
raise RuntimeError("no uptime signal of this gateway is listed in signals.yml")
return host
def samples(series):
@@ -39,14 +62,15 @@ def current_boot_ms():
return int(time.time() * 1000 - uptime_s * 1000)
def previous_session(boot_ms):
hours = last_samples(UPTIME, boot_ms - LOOKBACK_MS, boot_ms, "1hr")
def previous_session(host, boot_ms):
uptime = f"{host}-uptime"
hours = last_samples(uptime, boot_ms - LOOKBACK_MS, boot_ms, "1hr")
if not hours:
return None
minutes = last_samples(UPTIME, hours[-1][0], min(hours[-1][0] + HOUR_MS, boot_ms), "1min")
minutes = last_samples(uptime, hours[-1][0], min(hours[-1][0] + HOUR_MS, boot_ms), "1min")
if not minutes:
return None
seconds = last_samples(UPTIME, minutes[-1][0], min(minutes[-1][0] + MINUTE_MS, boot_ms), "1s")
seconds = last_samples(uptime, minutes[-1][0], min(minutes[-1][0] + MINUTE_MS, boot_ms), "1s")
end_ms, uptime_s = (seconds or minutes)[-1]
return int(end_ms - uptime_s * 1000), int(end_ms)
@@ -88,21 +112,22 @@ def coverage(series, start_ms, end_ms):
return min(1.0, present / total)
def build_row(start_ms, end_ms):
history = read_history([CPU, RAM, DISK], start_ms, end_ms, "1min")
def build_row(host, start_ms, end_ms):
cpu, ram, disk = f"{host}-cpu", f"{host}-ram", f"{host}-disk"
history = read_history([cpu, ram, disk], start_ms, end_ms, "1min")
row = {"start_time": start_ms, "end_time": end_ms, "duration_s": (end_ms - start_ms) // 1000}
for prefix, signal in (("cpu", CPU), ("ram", RAM), ("disk", DISK)):
for prefix, signal in (("cpu", cpu), ("ram", ram), ("disk", disk)):
row[f"{prefix}_mean"], row[f"{prefix}_max"] = mean_max(history[signal], start_ms, end_ms)
row["coverage"] = coverage(history[CPU], start_ms, end_ms)
row["coverage"] = coverage(history[cpu], start_ms, end_ms)
return row
def close_previous(rows, boot_ms):
session = previous_session(boot_ms)
def close_previous(host, rows, boot_ms):
session = previous_session(host, boot_ms)
if session is None:
print("no previous session in the uptime history", flush=True)
return
row = build_row(*session)
row = build_row(host, *session)
stored = find_row(rows, session[0])
if stored is None:
print("previous session stored", db_call("POST", f"/api/db/{TABLE}/", row), row, flush=True)
@@ -116,9 +141,9 @@ def main():
rows = stored_rows()
if find_row(rows, boot_ms) is not None:
return
close_previous(rows, boot_ms)
close_previous(own_host(), rows, boot_ms)
print("current session opened", db_call("POST", f"/api/db/{TABLE}/", open_row(boot_ms)), flush=True)
if __name__ == "__main__":
main()
main()
schedule.yaml
+1 -1
View File
@@ -9,4 +9,4 @@ command: ["python", "-u", "/app/main.py"]
maxCpuCores: 0.25
maxMemoryMb: 128
autoremove: false
logOption: 1
logOption: 1
signals.yml
+27 -3
View File
@@ -1,4 +1,28 @@
- { id: 21, name: 'deb13kde-cpu', read: 1, write: 0 }
- { id: 22, name: 'deb13kde-ram', read: 1, write: 0 }
- { id: 23, name: 'deb13kde-disk', read: 1, write: 0 }
- { id: 21, name: 'deb13kde-cpu', read: 1, write: 0 }
- { id: 22, name: 'deb13kde-ram', read: 1, write: 0 }
- { id: 23, name: 'deb13kde-disk', read: 1, write: 0 }
- { id: 63, name: 'deb13kde-uptime', read: 1, write: 0 }
- { id: 1, name: 'rbx-01-cpu', read: 1, write: 0 }
- { id: 2, name: 'rbx-01-ram', read: 1, write: 0 }
- { id: 3, name: 'rbx-01-disk', read: 1, write: 0 }
- { id: 64, name: 'rbx-01-uptime', read: 1, write: 0 }
- { id: 6, name: 'fr-01-cpu', read: 1, write: 0 }
- { id: 7, name: 'fr-01-ram', read: 1, write: 0 }
- { id: 8, name: 'fr-01-disk', read: 1, write: 0 }
- { id: 65, name: 'fr-01-uptime', read: 1, write: 0 }
- { id: 11, name: 'sb-01-cpu', read: 1, write: 0 }
- { id: 12, name: 'sb-01-ram', read: 1, write: 0 }
- { id: 13, name: 'sb-01-disk', read: 1, write: 0 }
- { id: 66, name: 'sb-01-uptime', read: 1, write: 0 }
- { id: 16, name: 'grab-01-cpu', read: 1, write: 0 }
- { id: 17, name: 'grab-01-ram', read: 1, write: 0 }
- { id: 18, name: 'grab-01-disk', read: 1, write: 0 }
- { id: 67, name: 'grab-01-uptime', read: 1, write: 0 }
- { id: 26, name: 'pi102-cpu', read: 1, write: 0 }
- { id: 27, name: 'pi102-ram', read: 1, write: 0 }
- { id: 28, name: 'pi102-disk', read: 1, write: 0 }
- { id: 68, name: 'pi102-uptime', read: 1, write: 0 }
- { id: 32, name: 'pi103-cpu', read: 1, write: 0 }
- { id: 33, name: 'pi103-ram', read: 1, write: 0 }
- { id: 34, name: 'pi103-disk', read: 1, write: 0 }
- { id: 69, name: 'pi103-uptime', read: 1, write: 0 }