set maximum failures
This commit is contained in:
Iñigo Etxaniz
@@ -1,5 +1,7 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
@@ -12,6 +14,22 @@ MINUTE_MS = 60000
|
||||
HOUR_MS = 60 * MINUTE_MS
|
||||
LOOKBACK_MS = 60 * 24 * HOUR_MS
|
||||
MATCH_MS = 2 * MINUTE_MS
|
||||
EXIT_RETRYABLE = 1
|
||||
EXIT_PERMANENT = 2
|
||||
REFUSED_STATUS_MIN = 400
|
||||
REFUSED_STATUS_MAX = 499
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DbRefused(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def log(level, msg, **fields):
|
||||
print(json.dumps({"level": level, "msg": msg, **fields}, default=str), flush=True)
|
||||
|
||||
|
||||
def read_history(signals, start_ms, end_ms, resolution):
|
||||
@@ -23,9 +41,15 @@ def read_history(signals, start_ms, end_ms, resolution):
|
||||
|
||||
|
||||
def signal_hosts():
|
||||
with open(SIGNALS_FILE) as file:
|
||||
names = re.findall(r"name:\s*'([^']+)'", file.read())
|
||||
return sorted(name[: -len("-uptime")] for name in names if name.endswith("-uptime"))
|
||||
try:
|
||||
with open(SIGNALS_FILE) as file:
|
||||
names = re.findall(r"name:\s*'([^']+)'", file.read())
|
||||
except OSError as error:
|
||||
raise ConfigError(f"cannot read {SIGNALS_FILE}: {error}")
|
||||
hosts = sorted(name[: -len("-uptime")] for name in names if name.endswith("-uptime"))
|
||||
if not hosts:
|
||||
raise ConfigError(f"{SIGNALS_FILE} lists no <host>-uptime signal")
|
||||
return hosts
|
||||
|
||||
|
||||
def owns(host, now_ms):
|
||||
@@ -40,7 +64,7 @@ def own_host():
|
||||
now_ms = time.time() * 1000
|
||||
host = next((name for name in signal_hosts() if owns(name, now_ms)), None)
|
||||
if host is None:
|
||||
raise RuntimeError("no uptime signal of this gateway is listed in signals.yml")
|
||||
raise ConfigError("no uptime signal of this gateway is listed in signals.yml")
|
||||
return host
|
||||
|
||||
|
||||
@@ -62,26 +86,51 @@ def current_boot_ms():
|
||||
return int(time.time() * 1000 - uptime_s * 1000)
|
||||
|
||||
|
||||
def belongs_to_boot(time_ms, uptime_s, boot_ms):
|
||||
return abs(time_ms - uptime_s * 1000 - boot_ms) < MATCH_MS
|
||||
|
||||
|
||||
def recent_hours(uptime, boot_ms):
|
||||
hours = last_samples(uptime, boot_ms - LOOKBACK_MS, boot_ms, "1hr")
|
||||
return [time_ms for time_ms, _ in reversed(hours[-2:])]
|
||||
|
||||
|
||||
def last_sample_of_an_earlier_boot(uptime, hour_ms, boot_ms):
|
||||
window_end = min(hour_ms + 2 * HOUR_MS, boot_ms + MATCH_MS)
|
||||
found = last_samples(uptime, hour_ms, window_end, "1min")
|
||||
earlier = [(t, v) for t, v in found if not belongs_to_boot(t, v, boot_ms)]
|
||||
return earlier[-1] if earlier else None
|
||||
|
||||
|
||||
def previous_session(host, boot_ms):
|
||||
uptime = f"{host}-uptime"
|
||||
hours = last_samples(uptime, boot_ms - LOOKBACK_MS, boot_ms, "1hr")
|
||||
if not hours:
|
||||
return None
|
||||
minutes = last_samples(uptime, hours[-1][0], min(hours[-1][0] + HOUR_MS, boot_ms), "1min")
|
||||
if not minutes:
|
||||
return None
|
||||
seconds = last_samples(uptime, minutes[-1][0], min(minutes[-1][0] + MINUTE_MS, boot_ms), "1s")
|
||||
end_ms, uptime_s = (seconds or minutes)[-1]
|
||||
return int(end_ms - uptime_s * 1000), int(end_ms)
|
||||
for hour_ms in recent_hours(uptime, boot_ms):
|
||||
last = last_sample_of_an_earlier_boot(uptime, hour_ms, boot_ms)
|
||||
if last is not None:
|
||||
end_ms, uptime_s = last
|
||||
return int(end_ms - uptime_s * 1000), int(end_ms)
|
||||
return None
|
||||
|
||||
|
||||
def is_distinct_session(session, boot_ms):
|
||||
start_ms, end_ms = session
|
||||
return end_ms > start_ms and abs(start_ms - boot_ms) >= MATCH_MS
|
||||
|
||||
|
||||
def read_db_answer(body):
|
||||
status = body.get("statusCode", 200)
|
||||
if status == 200:
|
||||
return body.get("data")
|
||||
message = f"db-controller {status}: {body.get('errorMessage')}"
|
||||
if REFUSED_STATUS_MIN <= status <= REFUSED_STATUS_MAX:
|
||||
raise DbRefused(message)
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def db_call(method, path, payload):
|
||||
response = requests.request(method, DB_URL + path, json=payload, timeout=(5, 30))
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
if body.get("statusCode", 200) != 200:
|
||||
raise RuntimeError(f"db-controller {body.get('statusCode')}: {body.get('errorMessage')}")
|
||||
return body.get("data")
|
||||
return read_db_answer(response.json())
|
||||
|
||||
|
||||
def stored_rows():
|
||||
@@ -102,7 +151,7 @@ def open_row(start_ms):
|
||||
def mean_max(series, start_ms, end_ms):
|
||||
values = [v for t, v in samples(series) if start_ms <= t < end_ms]
|
||||
if not values:
|
||||
return None, None
|
||||
return 0, 0
|
||||
return sum(values) / len(values), max(values)
|
||||
|
||||
|
||||
@@ -122,18 +171,37 @@ def build_row(host, start_ms, end_ms):
|
||||
return row
|
||||
|
||||
|
||||
def close_previous(host, rows, boot_ms):
|
||||
session = previous_session(host, boot_ms)
|
||||
if session is None:
|
||||
print("no previous session in the uptime history", flush=True)
|
||||
def store_previous_session(host, rows, session):
|
||||
stored = find_row(rows, session[0])
|
||||
if stored is not None and stored["end_time"] != stored["start_time"]:
|
||||
log("INFO", "previous session already stored", id=stored["id"], start=session[0], end=session[1])
|
||||
return
|
||||
row = build_row(host, *session)
|
||||
stored = find_row(rows, session[0])
|
||||
if stored is None:
|
||||
print("previous session stored", db_call("POST", f"/api/db/{TABLE}/", row), row, flush=True)
|
||||
elif stored["end_time"] == stored["start_time"]:
|
||||
db_call("PUT", f"/api/db/{TABLE}/{stored['id']}/", row)
|
||||
print("previous session closed", stored["id"], row, flush=True)
|
||||
created = db_call("POST", f"/api/db/{TABLE}/", row)
|
||||
log("INFO", "previous session stored", id=created["id"], start=session[0], end=session[1], coverage=row["coverage"])
|
||||
return
|
||||
db_call("PUT", f"/api/db/{TABLE}/{stored['id']}/", row)
|
||||
log("INFO", "previous session closed", id=stored["id"], start=session[0], end=session[1], coverage=row["coverage"])
|
||||
|
||||
|
||||
def close_previous_session(host, rows, boot_ms):
|
||||
session = previous_session(host, boot_ms)
|
||||
if session is None:
|
||||
log("INFO", "no previous session in the uptime history")
|
||||
return
|
||||
if not is_distinct_session(session, boot_ms):
|
||||
log("WARN", "previous session ignored, it is the current boot", start=session[0], end=session[1], boot=boot_ms)
|
||||
return
|
||||
try:
|
||||
store_previous_session(host, rows, session)
|
||||
except DbRefused as error:
|
||||
log("ERROR", "previous session not stored, the database refused it", start=session[0], end=session[1], error=str(error))
|
||||
|
||||
|
||||
def open_current_session(boot_ms):
|
||||
created = db_call("POST", f"/api/db/{TABLE}/", open_row(boot_ms))
|
||||
log("INFO", "current session opened", id=created["id"], start=boot_ms)
|
||||
|
||||
|
||||
def main():
|
||||
@@ -141,9 +209,21 @@ def main():
|
||||
rows = stored_rows()
|
||||
if find_row(rows, boot_ms) is not None:
|
||||
return
|
||||
close_previous(own_host(), rows, boot_ms)
|
||||
print("current session opened", db_call("POST", f"/api/db/{TABLE}/", open_row(boot_ms)), flush=True)
|
||||
close_previous_session(own_host(), rows, boot_ms)
|
||||
open_current_session(boot_ms)
|
||||
|
||||
|
||||
def run():
|
||||
try:
|
||||
main()
|
||||
except (ConfigError, DbRefused) as error:
|
||||
log("ERROR", "this run can not succeed until the script or the schema changes", error=str(error))
|
||||
return EXIT_PERMANENT
|
||||
except Exception as error:
|
||||
log("ERROR", "run failed, it is tried again", error=repr(error))
|
||||
return EXIT_RETRYABLE
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
sys.exit(run())
|
||||
Reference in New Issue
Block a user