**Considering modifications to collector**

I need to modify the collector to handle gzipped rotated logs and consider reading from them. A daily summary in a JSON format could be useful, including metrics like hours and sessions per device. I should also think about backfilling existing logs and possibly using an API, but maybe just a summary file will suffice. Implementing end-to-end validation is important, and I might need to use a Python script for easier modifications. Overall, there's a lot to think about regarding retention and aggregation of logs.

edit

import concurrent.futures as cf
import ipaddress
import json
import os
import re
import socket
import ssl
import subprocess
import sys
import time
import urllib.request

import concurrent.futures as cf
import gzip
import ipaddress
import json
import os
import re
import socket
import ssl
import subprocess
import sys
import time
import urllib.request

INVENTORY_PATH = Path(os.environ.get("WIFI_PRESENCE_INVENTORY", str(LOG_DIR / "inventory.json")))
SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_SUMMARY", "/var/www/wifi-tracker/summary.json"))
PING_WORKERS = int(os.environ.get("WIFI_PRESENCE_PING_WORKERS", "32"))
HOSTNAME_TIMEOUT = float(os.environ.get("WIFI_PRESENCE_HOSTNAME_TIMEOUT", "0.35"))
SUMMARY_TAIL_LINES = int(os.environ.get("WIFI_PRESENCE_SUMMARY_TAIL_LINES", "20000"))
ROTATE_MAX_BYTES = int(os.environ.get("WIFI_PRESENCE_ROTATE_MAX_BYTES", str(4 * 1024 * 1024)))
ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "5"))
OFFLINE_GAP_MINUTES = int(os.environ.get("WIFI_PRESENCE_OFFLINE_GAP_MINUTES", "5"))

INVENTORY_PATH = Path(os.environ.get("WIFI_PRESENCE_INVENTORY", str(LOG_DIR / "inventory.json")))
DAILY_SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_DAILY_SUMMARY", str(LOG_DIR / "daily-summary.json")))
SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_SUMMARY", "/var/www/wifi-tracker/summary.json"))
PING_WORKERS = int(os.environ.get("WIFI_PRESENCE_PING_WORKERS", "32"))
HOSTNAME_TIMEOUT = float(os.environ.get("WIFI_PRESENCE_HOSTNAME_TIMEOUT", "0.35"))
SUMMARY_TAIL_LINES = int(os.environ.get("WIFI_PRESENCE_SUMMARY_TAIL_LINES", "20000"))
ROTATE_MAX_BYTES = int(os.environ.get("WIFI_PRESENCE_ROTATE_MAX_BYTES", str(4 * 1024 * 1024)))
ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "30"))
OFFLINE_GAP_MINUTES = int(os.environ.get("WIFI_PRESENCE_OFFLINE_GAP_MINUTES", "5"))

    if not INVENTORY_PATH.exists():
        INVENTORY_PATH.write_text('{"devices": {}}\n')
    if not DAILY_SUMMARY_PATH.exists():
        DAILY_SUMMARY_PATH.write_text('{"updated_at": null, "devices": {}}\n')

def rotate_logs() -> None:
    if not LOG_PATH.exists() or LOG_PATH.stat().st_size < ROTATE_MAX_BYTES:
        return
    ts = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
    rotated = LOG_DIR / f"activity-{ts}.jsonl"
    LOG_PATH.replace(rotated)
    old = sorted(LOG_DIR.glob("activity-*.jsonl"))
    for path in old[:-ROTATE_KEEP]:
        try:
            path.unlink()
        except FileNotFoundError:
            pass

def rotate_logs() -> None:
    if not LOG_PATH.exists() or LOG_PATH.stat().st_size < ROTATE_MAX_BYTES:
        return
    ts = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
    rotated = LOG_DIR / f"activity-{ts}.jsonl"
    compressed = LOG_DIR / f"activity-{ts}.jsonl.gz"
    LOG_PATH.replace(rotated)
    with rotated.open("rb") as src, gzip.open(compressed, "wb", compresslevel=6) as dst:
        dst.writelines(src)
    rotated.unlink(missing_ok=True)
    old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
    for path in old[:-ROTATE_KEEP]:
        try:
            path.unlink()
        except FileNotFoundError:
            pass

def load_inventory() -> Dict:
    return load_json(INVENTORY_PATH, {"devices": {}})

def save_inventory(data: Dict):
    write_json_atomic(INVENTORY_PATH, data)

def load_inventory() -> Dict:
    return load_json(INVENTORY_PATH, {"devices": {}})

def load_daily_summary() -> Dict:
    return load_json(DAILY_SUMMARY_PATH, {"updated_at": None, "devices": {}})

def save_inventory(data: Dict):
    write_json_atomic(INVENTORY_PATH, data)

def save_daily_summary(data: Dict):
    write_json_atomic(DAILY_SUMMARY_PATH, data)

def hydrate_inventory_from_events(inventory: Dict, aliases: Dict):
    inv_devices = inventory.setdefault("devices", {})
    try:
        lines = LOG_PATH.read_text().splitlines()[-SUMMARY_TAIL_LINES:]
    except Exception:
        return inventory
    for line in lines:
        try:
            ev = json.loads(line)
        except Exception:
            continue
        dev = ev.get("device") or {}
        mac = (dev.get("mac") or "").lower()
        if not mac:
            continue
        prev_inv = inv_devices.get(mac, {})
        merged = dict(prev_inv)
        merged.update({k: v for k, v in dev.items() if v is not None})
        merged["alias"] = aliases.get(mac, merged.get("alias"))
        merged["display_name"] = merged.get("alias") or merged.get("hostname") or merged.get("mac") or merged.get("ip")
        merged["first_seen_ts"] = prev_inv.get("first_seen_ts") or ev.get("ts") or dev.get("first_seen_ts")
        merged["last_seen_ts"] = ev.get("ts") or dev.get("last_seen_ts") or prev_inv.get("last_seen_ts")
        inv_devices[mac] = merged
    return inventory

def event_files() -> List[Path]:
    return sorted(list(LOG_DIR.glob("activity-*.jsonl.gz")) + list(LOG_DIR.glob("activity-*.jsonl")) + ([LOG_PATH] if LOG_PATH.exists() else []))

def iter_events(paths: List[Path]):
    for path in paths:
        opener = gzip.open if path.suffix == ".gz" else open
        try:
            with opener(path, "rt") as fh:
                for line in fh:
                    try:
                        yield json.loads(line)
                    except Exception:
                        continue
        except FileNotFoundError:
            continue

def read_recent_events(limit: int = SUMMARY_TAIL_LINES) -> List[Dict]:
    if not LOG_PATH.exists():
        return []
    try:
        lines = LOG_PATH.read_text().splitlines()[-limit:]
    except Exception:
        return []
    events = []
    for line in lines:
        try:
            events.append(json.loads(line))
        except Exception:
            continue
    return events

def hydrate_inventory_from_events(inventory: Dict, aliases: Dict):
    inv_devices = inventory.setdefault("devices", {})
    for ev in read_recent_events():
        dev = ev.get("device") or {}
        mac = (dev.get("mac") or "").lower()
        if not mac:
            continue
        prev_inv = inv_devices.get(mac, {})
        merged = dict(prev_inv)
        merged.update({k: v for k, v in dev.items() if v is not None})
        merged["alias"] = aliases.get(mac, merged.get("alias"))
        merged["display_name"] = merged.get("alias") or merged.get("hostname") or merged.get("mac") or merged.get("ip")
        merged["first_seen_ts"] = prev_inv.get("first_seen_ts") or ev.get("ts") or dev.get("first_seen_ts")
        merged["last_seen_ts"] = ev.get("ts") or dev.get("last_seen_ts") or prev_inv.get("last_seen_ts")
        inv_devices[mac] = merged
    return inventory

def build_summary(devices: List[Dict], state: Dict, inventory: Dict, iface: str, gw: str, cidr: str, router: Dict, aliases: Dict, alert_prefs: Dict) -> Dict:
    now_ts = now()
    now_dt = parse_ts(now_ts)
    active_total = len([d for d in devices if d.get("online")])
    named = sum(1 for d in devices if d.get("hostname"))
    gateway_mac = next((d.get("mac") for d in devices if "gateway" in d.get("roles", [])), None)
    vendors = {}
    timeline = []
    alert_events = []
    try:
        lines = LOG_PATH.read_text().splitlines()[-SUMMARY_TAIL_LINES:]
    except Exception:
        lines = []
    events = []
    for line in lines:
        try:
            events.append(json.loads(line))
        except Exception:
            continue

def update_daily_summary(all_events: List[Dict], aliases: Dict, inventory: Dict) -> Dict:
    out = {"updated_at": now(), "devices": {}}
    grouped = defaultdict(list)
    for ev in all_events:
        if ev.get("event") != "device_snapshot":
            continue
        dev = ev.get("device") or {}
        mac = (dev.get("mac") or "").lower()
        ts = ev.get("ts")
        if not mac or not ts:
            continue
        grouped[mac].append(parse_ts(ts))
    inv_devices = inventory.get("devices") or {}
    for mac, times in grouped.items():
        times.sort()
        alias = aliases.get(mac) or (inv_devices.get(mac) or {}).get("alias") or (inv_devices.get(mac) or {}).get("display_name") or mac
        sessions = build_sessions([{"event": "device_snapshot", "device": {"mac": mac}, "ts": t.strftime("%Y-%m-%dT%H:%M:%SZ")} for t in times], mac)
        by_day = defaultdict(lambda: {"hours": 0.0, "sessions": 0, "first_seen_ts": None, "last_seen_ts": None})
        for session in sessions:
            start = session["start"]
            end = session["end"] + timedelta(minutes=2)
            first_day = start.strftime("%Y-%m-%d")
            by_day[first_day]["sessions"] += 1
            cursor = start
            while cursor < end:
                day_start = cursor.replace(hour=0, minute=0, second=0, microsecond=0)
                nxt = min(day_start + timedelta(days=1), end)
                key = day_start.strftime("%Y-%m-%d")
                row = by_day[key]
                row["hours"] += max((nxt - cursor).total_seconds() / 3600, 0)
                ts_start = cursor.strftime("%Y-%m-%dT%H:%M:%SZ")
                ts_end = nxt.strftime("%Y-%m-%dT%H:%M:%SZ")
                row["first_seen_ts"] = min(filter(None, [row["first_seen_ts"], ts_start])) if row["first_seen_ts"] else ts_start
                row["last_seen_ts"] = max(filter(None, [row["last_seen_ts"], ts_end])) if row["last_seen_ts"] else ts_end
                cursor = nxt
        out["devices"][mac] = {
            "alias": alias,
            "first_seen_ts": times[0].strftime("%Y-%m-%dT%H:%M:%SZ"),
            "last_seen_ts": times[-1].strftime("%Y-%m-%dT%H:%M:%SZ"),
            "days": {day: {**vals, "hours": round(vals["hours"], 2)} for day, vals in sorted(by_day.items())},
        }
    save_daily_summary(out)
    return out

def build_summary(devices: List[Dict], state: Dict, inventory: Dict, iface: str, gw: str, cidr: str, router: Dict, aliases: Dict, alert_prefs: Dict, daily_summary: Dict) -> Dict:
    now_ts = now()
    now_dt = parse_ts(now_ts)
    active_total = len([d for d in devices if d.get("online")])
    named = sum(1 for d in devices if d.get("hostname"))
    gateway_mac = next((d.get("mac") for d in devices if "gateway" in d.get("roles", [])), None)
    vendors = {}
    timeline = []
    alert_events = []
    events = read_recent_events()

    summary = {
        "updated_at": now_ts,
        "iface": iface,
        "gateway": gw,
        "cidr": cidr,
        "router": router,
        "totals": {
            "devices_now": active_total,
            "named_devices": named,
            "known_devices": len((inventory.get("devices") or {})),
            "aliased_devices": len([k for k, v in aliases.items() if v]),
        },
        "aliases": aliases,
        "devices": devices,
        "vendors": [{"name": k, "count": v} for k, v in sorted(vendors.items(), key=lambda x: (-x[1], x[0]))],
        "recent_events": timeline[-100:],
        "recent_alerts": alert_events[-100:],
        "highlights": {"gateway_mac": gateway_mac},
    }

    summary = {
        "updated_at": now_ts,
        "iface": iface,
        "gateway": gw,
        "cidr": cidr,
        "router": router,
        "totals": {
            "devices_now": active_total,
            "named_devices": named,
            "known_devices": len((inventory.get("devices") or {})),
            "aliased_devices": len([k for k, v in aliases.items() if v]),
        },
        "aliases": aliases,
        "devices": devices,
        "vendors": [{"name": k, "count": v} for k, v in sorted(vendors.items(), key=lambda x: (-x[1], x[0]))],
        "recent_events": timeline[-100:],
        "recent_alerts": alert_events[-100:],
        "daily_summary": daily_summary,
        "retention": {
            "rotate_max_bytes": ROTATE_MAX_BYTES,
            "rotate_keep": ROTATE_KEEP,
            "rotated_compression": "gzip",
        },
        "highlights": {"gateway_mac": gateway_mac},
    }

    save_state(new_state)
    save_inventory(inventory)
    inventory_devices = list(inv_devices.values())
    inventory_devices.sort(key=lambda d: (not d.get("online", False), tuple(int(x) for x in d.get("ip", "255.255.255.255").split(".")) if "." in d.get("ip", "") else (255,255,255,255)))
    build_summary(inventory_devices, new_state, inventory, iface, gw, cidr, router, aliases, alert_prefs)

    log_event({"event": "cycle_end", "iface": iface, "gateway": gw, "seen_devices": len(current), "duration_sec": round(time.time() - started, 2)})

    save_state(new_state)
    save_inventory(inventory)
    all_events = list(iter_events(event_files()))
    daily_summary = update_daily_summary(all_events, aliases, inventory)
    inventory_devices = list(inv_devices.values())
    inventory_devices.sort(key=lambda d: (not d.get("online", False), tuple(int(x) for x in d.get("ip", "255.255.255.255").split(".")) if "." in d.get("ip", "") else (255,255,255,255)))
    build_summary(inventory_devices, new_state, inventory, iface, gw, cidr, router, aliases, alert_prefs, daily_summary)

    log_event({"event": "cycle_end", "iface": iface, "gateway": gw, "seen_devices": len(current), "duration_sec": round(time.time() - started, 2)})


Successfully replaced 9 block(s) in /home/sebas/pi-config/bin/wifi-presence-log.

   1 #!/usr/bin/env python3
   2 import concurrent.futures as cf
+  3 import gzip
   3 import ipaddress
   4 import json
   5 import os
   6 import re
     ...
  21 ALIASES_PATH = Path(os.environ.get("WIFI_PRESENCE_ALIASES", str(LOG_DIR / "aliases.json")))
  22 FINGERPRINTS_PATH = Path(os.environ.get("WIFI_PRESENCE_FINGERPRINTS", str(LOG_DIR / "fingerprints.json")))
  23 ALERT_PREFS_PATH = Path(os.environ.get("WIFI_PRESENCE_ALERT_PREFS", str(LOG_DIR / "alert-prefs.json")))
  24 INVENTORY_PATH = Path(os.environ.get("WIFI_PRESENCE_INVENTORY", str(LOG_DIR / "inventory.json")))
+ 26 DAILY_SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_DAILY_SUMMARY", str(LOG_DIR / "daily-summary.json")))
  25 SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_SUMMARY", "/var/www/wifi-tracker/summary.json"))
  26 PING_WORKERS = int(os.environ.get("WIFI_PRESENCE_PING_WORKERS", "32"))
  27 HOSTNAME_TIMEOUT = float(os.environ.get("WIFI_PRESENCE_HOSTNAME_TIMEOUT", "0.35"))
  28 SUMMARY_TAIL_LINES = int(os.environ.get("WIFI_PRESENCE_SUMMARY_TAIL_LINES", "20000"))
  29 ROTATE_MAX_BYTES = int(os.environ.get("WIFI_PRESENCE_ROTATE_MAX_BYTES", str(4 * 1024 * 1024)))
- 30 ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "5"))
+ 32 ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "30"))
  31 OFFLINE_GAP_MINUTES = int(os.environ.get("WIFI_PRESENCE_OFFLINE_GAP_MINUTES", "5"))
  32
  33
  34 def sh(cmd: str) -> str:
     ...
  55     if not ALERT_PREFS_PATH.exists():
  56         ALERT_PREFS_PATH.write_text('{"new_device_alerts": true, "devices": {}}\n')
  57     if not INVENTORY_PATH.exists():
  58         INVENTORY_PATH.write_text('{"devices": {}}\n')
+ 61     if not DAILY_SUMMARY_PATH.exists():
+ 62         DAILY_SUMMARY_PATH.write_text('{"updated_at": null, "devices": {}}\n')
  59
  60
  61 def rotate_logs() -> None:
  62     if not LOG_PATH.exists() or LOG_PATH.stat().st_size < ROTATE_MAX_BYTES:
  63         return
  64     ts = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
  65     rotated = LOG_DIR / f"activity-{ts}.jsonl"
+ 70     compressed = LOG_DIR / f"activity-{ts}.jsonl.gz"
  66     LOG_PATH.replace(rotated)
- 67     old = sorted(LOG_DIR.glob("activity-*.jsonl"))
+ 72     with rotated.open("rb") as src, gzip.open(compressed, "wb", compresslevel=6) as dst:
+ 73         dst.writelines(src)
+ 74     rotated.unlink(missing_ok=True)
+ 75     old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
  68     for path in old[:-ROTATE_KEEP]:
  69         try:
  70             path.unlink()
  71         except FileNotFoundError:
     ...
  99 def load_inventory() -> Dict:
 100     return load_json(INVENTORY_PATH, {"devices": {}})
 101
 102
+111 def load_daily_summary() -> Dict:
+112     return load_json(DAILY_SUMMARY_PATH, {"updated_at": None, "devices": {}})
+113
+114
 103 def save_inventory(data: Dict):
 104     write_json_atomic(INVENTORY_PATH, data)
 105
 106
+119 def save_daily_summary(data: Dict):
+120     write_json_atomic(DAILY_SUMMARY_PATH, data)
+121
+122
 107 def save_state(state: Dict) -> None:
 108     tmp = STATE_PATH.with_suffix(".tmp")
 109     tmp.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
 110     tmp.replace(STATE_PATH)
     ...
 537         "sessions": len(sessions),
 538     }
 539
 540
-541 def hydrate_inventory_from_events(inventory: Dict, aliases: Dict):
-542     inv_devices = inventory.setdefault("devices", {})
+557 def event_files() -> List[Path]:
+558     return sorted(list(LOG_DIR.glob("activity-*.jsonl.gz")) + list(LOG_DIR.glob("activity-*.jsonl")) + ([LOG_PATH] if LOG_PATH.exists() else []))
+559
+560
+561 def iter_events(paths: List[Path]):
+562     for path in paths:
+563         opener = gzip.open if path.suffix == ".gz" else open
+564         try:
+565             with opener(path, "rt") as fh:
+566                 for line in fh:
+567                     try:
+568                         yield json.loads(line)
+569                     except Exception:
+570                         continue
+571         except FileNotFoundError:
+572             continue
+573
+574
+575 def read_recent_events(limit: int = SUMMARY_TAIL_LINES) -> List[Dict]:
+576     if not LOG_PATH.exists():
+577         return []
 543     try:
-544         lines = LOG_PATH.read_text().splitlines()[-SUMMARY_TAIL_LINES:]
+579         lines = LOG_PATH.read_text().splitlines()[-limit:]
 545     except Exception:
-546         return inventory
+581         return []
+582     events = []
 547     for line in lines:
 548         try:
-549             ev = json.loads(line)
+585             events.append(json.loads(line))
 550         except Exception:
 551             continue
+588     return events
+589
+590
+591 def hydrate_inventory_from_events(inventory: Dict, aliases: Dict):
+592     inv_devices = inventory.setdefault("devices", {})
+593     for ev in read_recent_events():
 552         dev = ev.get("device") or {}
 553         mac = (dev.get("mac") or "").lower()
 554         if not mac:
 555             continue
     ...
 563         inv_devices[mac] = merged
 564     return inventory
 565
 566
-567 def build_summary(devices: List[Dict], state: Dict, inventory: Dict, iface: str, gw: str, cidr: str, router: Dict, aliases: Dict, alert_prefs: Dict) -> Dict:
+609 def update_daily_summary(all_events: List[Dict], aliases: Dict, inventory: Dict) -> Dict:
+610     out = {"updated_at": now(), "devices": {}}
+611     grouped = defaultdict(list)
+612     for ev in all_events:
+613         if ev.get("event") != "device_snapshot":
+614             continue
+615         dev = ev.get("device") or {}
+616         mac = (dev.get("mac") or "").lower()
+617         ts = ev.get("ts")
+618         if not mac or not ts:
+619             continue
+620         grouped[mac].append(parse_ts(ts))
+621     inv_devices = inventory.get("devices") or {}
+622     for mac, times in grouped.items():
+623         times.sort()
+624         alias = aliases.get(mac) or (inv_devices.get(mac) or {}).get("alias") or (inv_devices.get(mac) or {}).get("display_name") or mac
+625         sessions = build_sessions([{"event": "device_snapshot", "device": {"mac": mac}, "ts": t.strftime("%Y-%m-%dT%H:%M:%SZ")} for t in times], mac)
+626         by_day = defaultdict(lambda: {"hours": 0.0, "sessions": 0, "first_seen_ts": None, "last_seen_ts": None})
+627         for session in sessions:
+628             start = session["start"]
+629             end = session["end"] + timedelta(minutes=2)
+630             first_day = start.strftime("%Y-%m-%d")
+631             by_day[first_day]["sessions"] += 1
+632             cursor = start
+633             while cursor < end:
+634                 day_start = cursor.replace(hour=0, minute=0, second=0, microsecond=0)
+635                 nxt = min(day_start + timedelta(days=1), end)
+636                 key = day_start.strftime("%Y-%m-%d")
+637                 row = by_day[key]
+638                 row["hours"] += max((nxt - cursor).total_seconds() / 3600, 0)
+639                 ts_start = cursor.strftime("%Y-%m-%dT%H:%M:%SZ")
+640                 ts_end = nxt.strftime("%Y-%m-%dT%H:%M:%SZ")
+641                 row["first_seen_ts"] = min(filter(None, [row["first_seen_ts"], ts_start])) if row["first_seen_ts"] else ts_start
+642                 row["last_seen_ts"] = max(filter(None, [row["last_seen_ts"], ts_end])) if row["last_seen_ts"] else ts_end
+643                 cursor = nxt
+644         out["devices"][mac] = {
+645             "alias": alias,
+646             "first_seen_ts": times[0].strftime("%Y-%m-%dT%H:%M:%SZ"),
+647             "last_seen_ts": times[-1].strftime("%Y-%m-%dT%H:%M:%SZ"),
+648             "days": {day: {**vals, "hours": round(vals["hours"], 2)} for day, vals in sorted(by_day.items())},
+649         }
+650     save_daily_summary(out)
+651     return out
+652
+653
+654 def build_summary(devices: List[Dict], state: Dict, inventory: Dict, iface: str, gw: str, cidr: str, router: Dict, aliases: Dict, alert_prefs: Dict, daily_summary: Dict) -> Dict:
 568     now_ts = now()
 569     now_dt = parse_ts(now_ts)
 570     active_total = len([d for d in devices if d.get("online")])
 571     named = sum(1 for d in devices if d.get("hostname"))
 572     gateway_mac = next((d.get("mac") for d in devices if "gateway" in d.get("roles", [])), None)
 573     vendors = {}
 574     timeline = []
 575     alert_events = []
-576     try:
-577         lines = LOG_PATH.read_text().splitlines()[-SUMMARY_TAIL_LINES:]
-578     except Exception:
-579         lines = []
-580     events = []
-581     for line in lines:
-582         try:
-583             events.append(json.loads(line))
-584         except Exception:
-585             continue
+663     events = read_recent_events()
 586     for ev in events:
 587         et = ev.get("event")
 588         ts = ev.get("ts")
 589         dev = ev.get("device") or {}
     ...
 628         "devices": devices,
 629         "vendors": [{"name": k, "count": v} for k, v in sorted(vendors.items(), key=lambda x: (-x[1], x[0]))],
 630         "recent_events": timeline[-100:],
 631         "recent_alerts": alert_events[-100:],
+710         "daily_summary": daily_summary,
+711         "retention": {
+712             "rotate_max_bytes": ROTATE_MAX_BYTES,
+713             "rotate_keep": ROTATE_KEEP,
+714             "rotated_compression": "gzip",
+715         },
 632         "highlights": {"gateway_mac": gateway_mac},
 633     }
 634     write_json_atomic(SUMMARY_PATH, summary)
 635     return summary
     ...
 712             merged["last_seen_ts"] = dev.get("last_seen_ts")
 713         inv_devices[mac] = merged
 714     save_state(new_state)
 715     save_inventory(inventory)
+800     all_events = list(iter_events(event_files()))
+801     daily_summary = update_daily_summary(all_events, aliases, inventory)
 716     inventory_devices = list(inv_devices.values())
 717     inventory_devices.sort(key=lambda d: (not d.get("online", False), tuple(int(x) for x in d.get("ip", "255.255.255.255").split(".")) if "." in d.get("ip", "") else (255,255,255,255)))
-718     build_summary(inventory_devices, new_state, inventory, iface, gw, cidr, router, aliases, alert_prefs)
+804     build_summary(inventory_devices, new_state, inventory, iface, gw, cidr, router, aliases, alert_prefs, daily_summary)
 719
 720     log_event({"event": "cycle_end", "iface": iface, "gateway": gw, "seen_devices": len(current), "duration_sec": round(time.time() - started, 2)})
 721     return 0
 722
     ...

--- /home/sebas/pi-config/bin/wifi-presence-log
+++ /home/sebas/pi-config/bin/wifi-presence-log
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import concurrent.futures as cf
+import gzip
 import ipaddress
 import json
 import os
 import re
@@ -21,14 +22,15 @@
 ALIASES_PATH = Path(os.environ.get("WIFI_PRESENCE_ALIASES", str(LOG_DIR / "aliases.json")))
 FINGERPRINTS_PATH = Path(os.environ.get("WIFI_PRESENCE_FINGERPRINTS", str(LOG_DIR / "fingerprints.json")))
 ALERT_PREFS_PATH = Path(os.environ.get("WIFI_PRESENCE_ALERT_PREFS", str(LOG_DIR / "alert-prefs.json")))
 INVENTORY_PATH = Path(os.environ.get("WIFI_PRESENCE_INVENTORY", str(LOG_DIR / "inventory.json")))
+DAILY_SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_DAILY_SUMMARY", str(LOG_DIR / "daily-summary.json")))
 SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_SUMMARY", "/var/www/wifi-tracker/summary.json"))
 PING_WORKERS = int(os.environ.get("WIFI_PRESENCE_PING_WORKERS", "32"))
 HOSTNAME_TIMEOUT = float(os.environ.get("WIFI_PRESENCE_HOSTNAME_TIMEOUT", "0.35"))
 SUMMARY_TAIL_LINES = int(os.environ.get("WIFI_PRESENCE_SUMMARY_TAIL_LINES", "20000"))
 ROTATE_MAX_BYTES = int(os.environ.get("WIFI_PRESENCE_ROTATE_MAX_BYTES", str(4 * 1024 * 1024)))
-ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "5"))
+ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "30"))
 OFFLINE_GAP_MINUTES = int(os.environ.get("WIFI_PRESENCE_OFFLINE_GAP_MINUTES", "5"))

 def sh(cmd: str) -> str:
@@ -55,17 +57,23 @@
     if not ALERT_PREFS_PATH.exists():
         ALERT_PREFS_PATH.write_text('{"new_device_alerts": true, "devices": {}}\n')
     if not INVENTORY_PATH.exists():
         INVENTORY_PATH.write_text('{"devices": {}}\n')
+    if not DAILY_SUMMARY_PATH.exists():
+        DAILY_SUMMARY_PATH.write_text('{"updated_at": null, "devices": {}}\n')

 def rotate_logs() -> None:
     if not LOG_PATH.exists() or LOG_PATH.stat().st_size < ROTATE_MAX_BYTES:
         return
     ts = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
     rotated = LOG_DIR / f"activity-{ts}.jsonl"
+    compressed = LOG_DIR / f"activity-{ts}.jsonl.gz"
     LOG_PATH.replace(rotated)
-    old = sorted(LOG_DIR.glob("activity-*.jsonl"))
+    with rotated.open("rb") as src, gzip.open(compressed, "wb", compresslevel=6) as dst:
+        dst.writelines(src)
+    rotated.unlink(missing_ok=True)
+    old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
     for path in old[:-ROTATE_KEEP]:
         try:
             path.unlink()
         except FileNotFoundError:
@@ -99,12 +107,20 @@
 def load_inventory() -> Dict:
     return load_json(INVENTORY_PATH, {"devices": {}})

+def load_daily_summary() -> Dict:
+    return load_json(DAILY_SUMMARY_PATH, {"updated_at": None, "devices": {}})
+
+
 def save_inventory(data: Dict):
     write_json_atomic(INVENTORY_PATH, data)

+def save_daily_summary(data: Dict):
+    write_json_atomic(DAILY_SUMMARY_PATH, data)
+
+
 def save_state(state: Dict) -> None:
     tmp = STATE_PATH.with_suffix(".tmp")
     tmp.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
     tmp.replace(STATE_PATH)
@@ -537,19 +553,45 @@
         "sessions": len(sessions),
     }

-def hydrate_inventory_from_events(inventory: Dict, aliases: Dict):
-    inv_devices = inventory.setdefault("devices", {})
+def event_files() -> List[Path]:
+    return sorted(list(LOG_DIR.glob("activity-*.jsonl.gz")) + list(LOG_DIR.glob("activity-*.jsonl")) + ([LOG_PATH] if LOG_PATH.exists() else []))
+
+
+def iter_events(paths: List[Path]):
+    for path in paths:
+        opener = gzip.open if path.suffix == ".gz" else open
+        try:
+            with opener(path, "rt") as fh:
+                for line in fh:
+                    try:
+                        yield json.loads(line)
+                    except Exception:
+                        continue
+        except FileNotFoundError:
+            continue
+
+
+def read_recent_events(limit: int = SUMMARY_TAIL_LINES) -> List[Dict]:
+    if not LOG_PATH.exists():
+        return []
     try:
-        lines = LOG_PATH.read_text().splitlines()[-SUMMARY_TAIL_LINES:]
+        lines = LOG_PATH.read_text().splitlines()[-limit:]
     except Exception:
-        return inventory
+        return []
+    events = []
     for line in lines:
         try:
-            ev = json.loads(line)
+            events.append(json.loads(line))
         except Exception:
             continue
+    return events
+
+
+def hydrate_inventory_from_events(inventory: Dict, aliases: Dict):
+    inv_devices = inventory.setdefault("devices", {})
+    for ev in read_recent_events():
         dev = ev.get("device") or {}
         mac = (dev.get("mac") or "").lower()
         if not mac:
             continue
@@ -563,27 +605,63 @@
         inv_devices[mac] = merged
     return inventory

-def build_summary(devices: List[Dict], state: Dict, inventory: Dict, iface: str, gw: str, cidr: str, router: Dict, aliases: Dict, alert_prefs: Dict) -> Dict:
+def update_daily_summary(all_events: List[Dict], aliases: Dict, inventory: Dict) -> Dict:
+    out = {"updated_at": now(), "devices": {}}
+    grouped = defaultdict(list)
+    for ev in all_events:
+        if ev.get("event") != "device_snapshot":
+            continue
+        dev = ev.get("device") or {}
+        mac = (dev.get("mac") or "").lower()
+        ts = ev.get("ts")
+        if not mac or not ts:
+            continue
+        grouped[mac].append(parse_ts(ts))
+    inv_devices = inventory.get("devices") or {}
+    for mac, times in grouped.items():
+        times.sort()
+        alias = aliases.get(mac) or (inv_devices.get(mac) or {}).get("alias") or (inv_devices.get(mac) or {}).get("display_name") or mac
+        sessions = build_sessions([{"event": "device_snapshot", "device": {"mac": mac}, "ts": t.strftime("%Y-%m-%dT%H:%M:%SZ")} for t in times], mac)
+        by_day = defaultdict(lambda: {"hours": 0.0, "sessions": 0, "first_seen_ts": None, "last_seen_ts": None})
+        for session in sessions:
+            start = session["start"]
+            end = session["end"] + timedelta(minutes=2)
+            first_day = start.strftime("%Y-%m-%d")
+            by_day[first_day]["sessions"] += 1
+            cursor = start
+            while cursor < end:
+                day_start = cursor.replace(hour=0, minute=0, second=0, microsecond=0)
+                nxt = min(day_start + timedelta(days=1), end)
+                key = day_start.strftime("%Y-%m-%d")
+                row = by_day[key]
+                row["hours"] += max((nxt - cursor).total_seconds() / 3600, 0)
+                ts_start = cursor.strftime("%Y-%m-%dT%H:%M:%SZ")
+                ts_end = nxt.strftime("%Y-%m-%dT%H:%M:%SZ")
+                row["first_seen_ts"] = min(filter(None, [row["first_seen_ts"], ts_start])) if row["first_seen_ts"] else ts_start
+                row["last_seen_ts"] = max(filter(None, [row["last_seen_ts"], ts_end])) if row["last_seen_ts"] else ts_end
+                cursor = nxt
+        out["devices"][mac] = {
+            "alias": alias,
+            "first_seen_ts": times[0].strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "last_seen_ts": times[-1].strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "days": {day: {**vals, "hours": round(vals["hours"], 2)} for day, vals in sorted(by_day.items())},
+        }
+    save_daily_summary(out)
+    return out
+
+
+def build_summary(devices: List[Dict], state: Dict, inventory: Dict, iface: str, gw: str, cidr: str, router: Dict, aliases: Dict, alert_prefs: Dict, daily_summary: Dict) -> Dict:
     now_ts = now()
     now_dt = parse_ts(now_ts)
     active_total = len([d for d in devices if d.get("online")])
     named = sum(1 for d in devices if d.get("hostname"))
     gateway_mac = next((d.get("mac") for d in devices if "gateway" in d.get("roles", [])), None)
     vendors = {}
     timeline = []
     alert_events = []
-    try:
-        lines = LOG_PATH.read_text().splitlines()[-SUMMARY_TAIL_LINES:]
-    except Exception:
-        lines = []
-    events = []
-    for line in lines:
-        try:
-            events.append(json.loads(line))
-        except Exception:
-            continue
+    events = read_recent_events()
     for ev in events:
         et = ev.get("event")
         ts = ev.get("ts")
         dev = ev.get("device") or {}
@@ -628,8 +706,14 @@
         "devices": devices,
         "vendors": [{"name": k, "count": v} for k, v in sorted(vendors.items(), key=lambda x: (-x[1], x[0]))],
         "recent_events": timeline[-100:],
         "recent_alerts": alert_events[-100:],
+        "daily_summary": daily_summary,
+        "retention": {
+            "rotate_max_bytes": ROTATE_MAX_BYTES,
+            "rotate_keep": ROTATE_KEEP,
+            "rotated_compression": "gzip",
+        },
         "highlights": {"gateway_mac": gateway_mac},
     }
     write_json_atomic(SUMMARY_PATH, summary)
     return summary
@@ -712,11 +796,13 @@
             merged["last_seen_ts"] = dev.get("last_seen_ts")
         inv_devices[mac] = merged
     save_state(new_state)
     save_inventory(inventory)
+    all_events = list(iter_events(event_files()))
+    daily_summary = update_daily_summary(all_events, aliases, inventory)
     inventory_devices = list(inv_devices.values())
     inventory_devices.sort(key=lambda d: (not d.get("online", False), tuple(int(x) for x in d.get("ip", "255.255.255.255").split(".")) if "." in d.get("ip", "") else (255,255,255,255)))
-    build_summary(inventory_devices, new_state, inventory, iface, gw, cidr, router, aliases, alert_prefs)
+    build_summary(inventory_devices, new_state, inventory, iface, gw, cidr, router, aliases, alert_prefs, daily_summary)

     log_event({"event": "cycle_end", "iface": iface, "gateway": gw, "seen_devices": len(current), "duration_sec": round(time.time() - started, 2)})
     return 0


**Verifying syntax issues**

I think I need to check the syntax, especially regarding `min(filter(None...`, since it might be invalid if there's only one argument; that could be fine for some cases. But using `min` on strings can be tricky, right? It might be costly to build sessions from synthetic events. Okay, I should also look at the edited sections around the `update_daily_summary` to catch any potential issues. Let's see what I find there!


#!/usr/bin/env python3
import concurrent.futures as cf
import gzip
import ipaddress
import json
import os
import re
import socket
import ssl
import subprocess
import sys
import time
import urllib.request
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Dict, List

LOG_DIR = Path(os.environ.get("WIFI_PRESENCE_LOG_DIR", "/home/sebas/runtime/agent-logs/wifi-presence"))
STATE_PATH = Path(os.environ.get("WIFI_PRESENCE_STATE", str(LOG_DIR / "state.json")))
LOG_PATH = Path(os.environ.get("WIFI_PRESENCE_LOG", str(LOG_DIR / "activity.jsonl")))
ALIASES_PATH = Path(os.environ.get("WIFI_PRESENCE_ALIASES", str(LOG_DIR / "aliases.json")))
FINGERPRINTS_PATH = Path(os.environ.get("WIFI_PRESENCE_FINGERPRINTS", str(LOG_DIR / "fingerprints.json")))
ALERT_PREFS_PATH = Path(os.environ.get("WIFI_PRESENCE_ALERT_PREFS", str(LOG_DIR / "alert-prefs.json")))
INVENTORY_PATH = Path(os.environ.get("WIFI_PRESENCE_INVENTORY", str(LOG_DIR / "inventory.json")))
DAILY_SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_DAILY_SUMMARY", str(LOG_DIR / "daily-summary.json")))
SUMMARY_PATH = Path(os.environ.get("WIFI_PRESENCE_SUMMARY", "/var/www/wifi-tracker/summary.json"))
PING_WORKERS = int(os.environ.get("WIFI_PRESENCE_PING_WORKERS", "32"))
HOSTNAME_TIMEOUT = float(os.environ.get("WIFI_PRESENCE_HOSTNAME_TIMEOUT", "0.35"))
SUMMARY_TAIL_LINES = int(os.environ.get("WIFI_PRESENCE_SUMMARY_TAIL_LINES", "20000"))
ROTATE_MAX_BYTES = int(os.environ.get("WIFI_PRESENCE_ROTATE_MAX_BYTES", str(4 * 1024 * 1024)))
ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "30"))
OFFLINE_GAP_MINUTES = int(os.environ.get("WIFI_PRESENCE_OFFLINE_GAP_MINUTES", "5"))

def sh(cmd: str) -> str:
    return subprocess.check_output(cmd, shell=True, text=True, stderr=subprocess.DEVNULL).strip()

def now() -> str:
    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())

def parse_ts(ts: str) -> datetime:
    return datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)

def ensure_paths() -> None:
    LOG_DIR.mkdir(parents=True, exist_ok=True)
    SUMMARY_PATH.parent.mkdir(parents=True, exist_ok=True)
    if not STATE_PATH.exists():
        STATE_PATH.write_text('{"devices": {}}\n')
    if not ALIASES_PATH.exists():
        ALIASES_PATH.write_text('{}\n')
    if not FINGERPRINTS_PATH.exists():
        FINGERPRINTS_PATH.write_text('{}\n')
    if not ALERT_PREFS_PATH.exists():
        ALERT_PREFS_PATH.write_text('{"new_device_alerts": true, "devices": {}}\n')
    if not INVENTORY_PATH.exists():
        INVENTORY_PATH.write_text('{"devices": {}}\n')
    if not DAILY_SUMMARY_PATH.exists():
        DAILY_SUMMARY_PATH.write_text('{"updated_at": null, "devices": {}}\n')

def rotate_logs() -> None:
    if not LOG_PATH.exists() or LOG_PATH.stat().st_size < ROTATE_MAX_BYTES:
        return
    ts = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
    rotated = LOG_DIR / f"activity-{ts}.jsonl"
    compressed = LOG_DIR / f"activity-{ts}.jsonl.gz"
    LOG_PATH.replace(rotated)
    with rotated.open("rb") as src, gzip.open(compressed, "wb", compresslevel=6) as dst:
        dst.writelines(src)
    rotated.unlink(missing_ok=True)
    old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
    for path in old[:-ROTATE_KEEP]:
        try:
            path.unlink()
        except FileNotFoundError:
            pass

def load_json(path: Path, default):
    try:
        return json.loads(path.read_text())
    except Exception:
        return default

def load_state() -> Dict:
    return load_json(STATE_PATH, {"devices": {}})

def load_aliases() -> Dict:
    data = load_json(ALIASES_PATH, {})
    return {k.lower(): v for k, v in data.items() if isinstance(v, str)}

def load_fingerprints() -> Dict:
    return load_json(FINGERPRINTS_PATH, {})

def load_alert_prefs() -> Dict:
    return load_json(ALERT_PREFS_PATH, {"new_device_alerts": True, "devices": {}})

def load_inventory() -> Dict:
    return load_json(INVENTORY_PATH, {"devices": {}})

def load_daily_summary() -> Dict:
    return load_json(DAILY_SUMMARY_PATH, {"updated_at": None, "devices": {}})

def save_inventory(data: Dict):
    write_json_atomic(INVENTORY_PATH, data)

def save_daily_summary(data: Dict):
    write_json_atomic(DAILY_SUMMARY_PATH, data)

def save_state(state: Dict) -> None:
    tmp = STATE_PATH.with_suffix(".tmp")
    tmp.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
    tmp.replace(STATE_PATH)

def write_json_atomic(path: Path, data) -> None:
    tmp = path.with_suffix(".tmp")
    tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n")
    tmp.replace(path)

def log_event(event: Dict) -> None:
    rotate_logs()
    event.setdefault("ts", now())
    with LOG_PATH.open("a") as f:
        f.write(json.dumps(event, sort_keys=True) + "\n")

def default_gateway() -> str:
    return sh("ip route | awk '/default/ {print $3; exit}'")

def default_iface() -> str:
    return sh("ip route | awk '/default/ {print $5; exit}'")

def iface_cidr(iface: str) -> str:
    return sh(f"ip -o -f inet addr show dev {iface} | awk '{{print $4; exit}}'")

def network_cidr(cidr: str) -> str:
    return str(ipaddress.ip_interface(cidr).network)

def local_ip(iface: str) -> str:
    out = sh(f"ip -o -4 addr show dev {iface}")
    return out.split()[3].split("/")[0]

def ping_host(ip: str) -> None:
    subprocess.run(["ping", "-c", "1", "-W", "1", ip], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

def sweep_network(cidr: str, self_ip: str) -> None:
    net = ipaddress.ip_network(cidr, strict=False)
    hosts: List[str] = [str(h) for h in net.hosts() if str(h) != self_ip]
    with cf.ThreadPoolExecutor(max_workers=PING_WORKERS) as ex:
        list(ex.map(ping_host, hosts))

def arp_entries(iface: str) -> List[Dict]:
    out = sh(f"ip neigh show dev {iface} || true")
    rows = []
    for line in out.splitlines():
        parts = line.split()
        if len(parts) < 4:
            continue
        row = {"ip": parts[0], "mac": None, "state": parts[-1], "iface": iface}
        if "lladdr" in parts:
            i = parts.index("lladdr")
            if i + 1 < len(parts):
                row["mac"] = parts[i + 1].lower()
        rows.append(row)
    return rows

def reverse_name(ip: str):
    try:
        socket.setdefaulttimeout(HOSTNAME_TIMEOUT)
        name, _, _ = socket.gethostbyaddr(ip)
        return name
    except Exception:
        return None

def probe_router(gateway: str) -> Dict:
    out = {"gateway": gateway}
    try:
        with urllib.request.urlopen(f"http://{gateway}/", timeout=4) as r:
            html = r.read(65536).decode("utf-8", "ignore")
        if "Huawei" in html:
            out["vendor"] = "Huawei"
        m = re.search(r"ProductName\s*=\s*'([^']+)'", html)
        if m:
            out["model"] = m.group(1).replace("\\x2d", "-")
    except Exception as e:
        out["probe_error"] = str(e)
    return out

def tcp_open(ip: str, port: int, timeout: float = 0.5) -> bool:
    s = socket.socket()
    s.settimeout(timeout)
    try:
        s.connect((ip, port))
        return True
    except Exception:

[592 more lines in file. Use offset=221 to continue.]


    }

def build_sessions(events: List[Dict], mac: str) -> List[Dict]:
    mac = mac.lower()
    snapshots = []
    for ev in events:
        if ev.get("event") != "device_snapshot":
            continue
        dev = ev.get("device") or {}
        if (dev.get("mac") or "").lower() != mac:
            continue
        ts = ev.get("ts")
        if ts:
            snapshots.append(parse_ts(ts))
    snapshots.sort()
    if not snapshots:
        return []
    sessions = []
    start = prev = snapshots[0]
    gap = timedelta(minutes=OFFLINE_GAP_MINUTES)
    for dt in snapshots[1:]:
        if dt - prev > gap:
            sessions.append({"start": start, "end": prev})
            start = dt
        prev = dt
    sessions.append({"start": start, "end": prev})
    return sessions

def fill_bucket_series(start: datetime, count: int, step: timedelta, key_fmt: str, values: Dict[str, float]) -> List[Dict]:
    out = []
    for i in range(count):
        dt = start + step * i
        key = dt.strftime(key_fmt)
        out.append({"bucket": key, "count": round(values.get(key, 0), 2)})
    return out

def summarize_presence_timeline(events: List[Dict], mac: str, now_dt: datetime, hours: int = 24, step_minutes: int = 2, offline_misses: int = 3) -> Dict:
    start = (now_dt - timedelta(hours=hours)).replace(second=0, microsecond=0)
    slots = []
    observed = set()
    for ev in events:
        if ev.get("event") != "device_snapshot":
            continue
        dev = ev.get("device") or {}
        if (dev.get("mac") or "").lower() != mac.lower():
            continue
        ts = ev.get("ts")
        if not ts:
            continue
        dt = parse_ts(ts)
        if dt < start or dt > now_dt + timedelta(minutes=step_minutes):
            continue
        minute = (dt.minute // step_minutes) * step_minutes
        bucket = dt.replace(minute=minute, second=0, microsecond=0)
        observed.add(bucket)
    cursor = start.replace(minute=(start.minute // step_minutes) * step_minutes, second=0, microsecond=0)
    state = False
    miss_count = offline_misses
    while cursor <= now_dt:
        seen = cursor in observed
        if seen:
            state = True
            miss_count = 0
        else:
            if state:
                miss_count += 1
                if miss_count >= offline_misses:
                    state = False
        slots.append({"ts": cursor.strftime("%Y-%m-%dT%H:%M:%SZ"), "observed": seen, "online": state})
        cursor += timedelta(minutes=step_minutes)
    segments = []
    seg_start = None
    for slot in slots:
        if slot["online"] and seg_start is None:
            seg_start = slot["ts"]
        if not slot["online"] and seg_start is not None:
            segments.append({"start": seg_start, "end": slot["ts"]})
            seg_start = None
    if seg_start is not None and slots:
        segments.append({"start": seg_start, "end": slots[-1]["ts"]})
    return {"start": start.strftime("%Y-%m-%dT%H:%M:%SZ"), "end": now_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), "step_minutes": step_minutes, "offline_misses": offline_misses, "segments": segments, "slots": slots}

def summarize_activity(events: List[Dict], mac: str, now_dt: datetime) -> Dict:
    hour_values = defaultdict(float)
    day_values = defaultdict(float)
    week_values = defaultdict(float)
    hour_start = (now_dt - timedelta(hours=23)).replace(minute=0, second=0, microsecond=0)
    day_start = (now_dt - timedelta(days=13)).replace(hour=0, minute=0, second=0, microsecond=0)
    week_anchor = (now_dt - timedelta(days=now_dt.weekday())).replace(hour=0, minute=0, second=0, microsecond=0)
    week_start = week_anchor - timedelta(weeks=11)
    sessions = build_sessions(events, mac)

    for session in sessions:
        start = session["start"]
        end = session["end"] + timedelta(minutes=2)
        cursor = max(start, hour_start)
        while cursor < end:
            nxt = min((cursor.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)), end)
            key = cursor.replace(minute=0, second=0, microsecond=0).strftime("%Y-%m-%dT%H:00Z")
            hour_values[key] += max((nxt - cursor).total_seconds() / 3600, 0)
            cursor = nxt
        cursor = max(start, day_start)
        while cursor < end:
            nxt = min((cursor.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1)), end)
            key = cursor.strftime("%Y-%m-%d")
            day_values[key] += max((nxt - cursor).total_seconds() / 3600, 0)
            cursor = nxt
        cursor = max(start, week_start)
        while cursor < end:
            wk = (cursor - timedelta(days=cursor.weekday())).replace(hour=0, minute=0, second=0, microsecond=0)
            nxt = min(wk + timedelta(weeks=1), end)
            key = wk.strftime("%Y-%m-%d")
            week_values[key] += max((nxt - cursor).total_seconds() / 3600, 0)
            cursor = nxt

    return {
        "hour": fill_bucket_series(hour_start, 24, timedelta(hours=1), "%Y-%m-%dT%H:00Z", hour_values),
        "day": fill_bucket_series(day_start, 14, timedelta(days=1), "%Y-%m-%d", day_values),
        "week": fill_bucket_series(week_start, 12, timedelta(weeks=1), "%Y-%m-%d", week_values),
        "sessions": len(sessions),
    }

def event_files() -> List[Path]:
    return sorted(list(LOG_DIR.glob("activity-*.jsonl.gz")) + list(LOG_DIR.glob("activity-*.jsonl")) + ([LOG_PATH] if LOG_PATH.exists() else []))

def iter_events(paths: List[Path]):
    for path in paths:
        opener = gzip.open if path.suffix == ".gz" else open
        try:
            with opener(path, "rt") as fh:
                for line in fh:
                    try:
                        yield json.loads(line)
                    except Exception:
                        continue
        except FileNotFoundError:
            continue

def read_recent_events(limit: int = SUMMARY_TAIL_LINES) -> List[Dict]:
    if not LOG_PATH.exists():
        return []
    try:
        lines = LOG_PATH.read_text().splitlines()[-limit:]
    except Exception:
        return []
    events = []
    for line in lines:
        try:
            events.append(json.loads(line))
        except Exception:
            continue
    return events

def hydrate_inventory_from_events(inventory: Dict, aliases: Dict):
    inv_devices = inventory.setdefault("devices", {})
    for ev in read_recent_events():
        dev = ev.get("device") or {}
        mac = (dev.get("mac") or "").lower()
        if not mac:
            continue
        prev_inv = inv_devices.get(mac, {})
        merged = dict(prev_inv)
        merged.update({k: v for k, v in dev.items() if v is not None})
        merged["alias"] = aliases.get(mac, merged.get("alias"))
        merged["display_name"] = merged.get("alias") or merged.get("hostname") or merged.get("mac") or merged.get("ip")
        merged["first_seen_ts"] = prev_inv.get("first_seen_ts") or ev.get("ts") or dev.get("first_seen_ts")
        merged["last_seen_ts"] = ev.get("ts") or dev.get("last_seen_ts") or prev_inv.get("last_seen_ts")
        inv_devices[mac] = merged
    return inventory

def update_daily_summary(all_events: List[Dict], aliases: Dict, inventory: Dict) -> Dict:
    out = {"updated_at": now(), "devices": {}}
    grouped = defaultdict(list)
    for ev in all_events:
        if ev.get("event") != "device_snapshot":
            continue
        dev = ev.get("device") or {}
        mac = (dev.get("mac") or "").lower()
        ts = ev.get("ts")
        if not mac or not ts:
            continue
        grouped[mac].append(parse_ts(ts))
    inv_devices = inventory.get("devices") or {}
    for mac, times in grouped.items():
        times.sort()
        alias = aliases.get(mac) or (inv_devices.get(mac) or {}).get("alias") or (inv_devices.get(mac) or {}).get("display_name") or mac
        sessions = build_sessions([{"event": "device_snapshot", "device": {"mac": mac}, "ts": t.strftime("%Y-%m-%dT%H:%M:%SZ")} for t in times], mac)
        by_day = defaultdict(lambda: {"hours": 0.0, "sessions": 0, "first_seen_ts": None, "last_seen_ts": None})
        for session in sessions:
            start = session["start"]
            end = session["end"] + timedelta(minutes=2)
            first_day = start.strftime("%Y-%m-%d")
            by_day[first_day]["sessions"] += 1
            cursor = start
            while cursor < end:
                day_start = cursor.replace(hour=0, minute=0, second=0, microsecond=0)
                nxt = min(day_start + timedelta(days=1), end)
                key = day_start.strftime("%Y-%m-%d")
                row = by_day[key]
                row["hours"] += max((nxt - cursor).total_seconds() / 3600, 0)
                ts_start = cursor.strftime("%Y-%m-%dT%H:%M:%SZ")
                ts_end = nxt.strftime("%Y-%m-%dT%H:%M:%SZ")
                row["first_seen_ts"] = min(filter(None, [row["first_seen_ts"], ts_start])) if row["first_seen_ts"] else ts_start
                row["last_seen_ts"] = max(filter(None, [row["last_seen_ts"], ts_end])) if row["last_seen_ts"] else ts_end
                cursor = nxt
        out["devices"][mac] = {
            "alias": alias,
            "first_seen_ts": times[0].strftime("%Y-%m-%dT%H:%M:%SZ"),
            "last_seen_ts": times[-1].strftime("%Y-%m-%dT%H:%M:%SZ"),
            "days": {day: {**vals, "hours": round(vals["hours"], 2)} for day, vals in sorted(by_day.items())},
        }
    save_daily_summary(out)
    return out

def build_summary(devices: List[Dict], state: Dict, inventory: Dict, iface: str, gw: str, cidr: str, router: Dict, aliases: Dict, alert_prefs: Dict, daily_summary: Dict) -> Dict:
    now_ts = now()
    now_dt = parse_ts(now_ts)
    active_total = len([d for d in devices if d.get("online")])
    named = sum(1 for d in devices if d.get("hostname"))
    gateway_mac = next((d.get("mac") for d in devices if "gateway" in d.get("roles", [])), None)
    vendors = {}
    timeline = []
    alert_events = []
    events = read_recent_events()
    for ev in events:
        et = ev.get("event")
        ts = ev.get("ts")
        dev = ev.get("device") or {}
        mac = (dev.get("mac") or "").lower() or None
        if et in {"device_online", "device_offline", "new_device_detected"} and ts:
            inv = (inventory.get("devices") or {}).get(mac or "", {})
            row = {
                "ts": ts,
                "event": et,
                "ip": dev.get("ip") or inv.get("ip"),
                "mac": mac,
                "alias": aliases.get(mac) if mac else inv.get("alias"),
                "display_name": aliases.get(mac) if mac else None,
            }
            row["display_name"] = row["alias"] or inv.get("display_name") or dev.get("display_name") or dev.get("hostname") or mac or row.get("ip")
            timeline.append(row)
            enabled = alert_prefs.get("devices", {}).get(mac, {}).get("alerts_enabled", True) if mac else True
            if et == 'new_device_detected':
                if alert_prefs.get("new_device_alerts", True):
                    alert_events.append(row)
            elif et in {'device_online', 'device_offline'} and enabled:
                alert_events.append(row)
    for d in devices:
        prefix = d.get("oui_prefix") or "unknown"
        vendors[prefix] = vendors.get(prefix, 0) + 1

[123 more lines in file. Use offset=690 to continue.]


**Clarifying session logic**

I’m thinking I might need to use row["last_seen_ts"] to track sessions, especially if they cross over midnight. It looks like the session count should only increment on the first day of the session, which seems acceptable. For the daily summary, I want to count session starts for each day. I should also update the documentation to mention gzip and the daily summary. It feels like I’m on the right path!

- rotated raw events: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl`
- state: `/home/sebas/runtime/agent-logs/wifi-presence/state.json`

- rotated raw events: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl.gz`
- legacy uncompressed rotates: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl`
- state: `/home/sebas/runtime/agent-logs/wifi-presence/state.json`

- inventory ever-seen: `/home/sebas/runtime/agent-logs/wifi-presence/inventory.json`
- summary: `/var/www/wifi-tracker/summary.json`

- inventory ever-seen: `/home/sebas/runtime/agent-logs/wifi-presence/inventory.json`
- persistent daily rollup: `/home/sebas/runtime/agent-logs/wifi-presence/daily-summary.json`
- summary: `/var/www/wifi-tracker/summary.json`

The raw JSONL log rotates automatically when it grows past the configured size threshold.
Old rotated files are kept in a small rolling set.

The raw JSONL log rotates automatically when it grows past the configured size threshold.
Rotated logs are gzip-compressed and the default keep window is larger now (`ROTATE_KEEP=30`).

The collector also writes `daily-summary.json`, a durable per-device per-day rollup with hours, sessions, first seen, and last seen. That file is rebuilt from retained raw logs and survives even when old fine-grained snapshots age out.

Recent UI/API activity still reads from the live tail for speed; long-range history should use `daily-summary.json`.


Successfully replaced 3 block(s) in /home/sebas/pi-config/docs/wifi-tracker.md.

    ...
 10 ## Pieces
 11
 12 - collector: `/home/sebas/pi-config/bin/wifi-presence-log`
 13 - raw events: `/home/sebas/runtime/agent-logs/wifi-presence/activity.jsonl`
-14 - rotated raw events: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl`
+14 - rotated raw events: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl.gz`
+15 - legacy uncompressed rotates: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl`
 15 - state: `/home/sebas/runtime/agent-logs/wifi-presence/state.json`
 16 - aliases: `/home/sebas/runtime/agent-logs/wifi-presence/aliases.json`
 17 - fingerprints/evidence: `/home/sebas/runtime/agent-logs/wifi-presence/fingerprints.json`
 18 - alert prefs: `/home/sebas/runtime/agent-logs/wifi-presence/alert-prefs.json`
 19 - inventory ever-seen: `/home/sebas/runtime/agent-logs/wifi-presence/inventory.json`
+21 - persistent daily rollup: `/home/sebas/runtime/agent-logs/wifi-presence/daily-summary.json`
 20 - summary: `/var/www/wifi-tracker/summary.json`
 21 - UI: `/var/www/wifi-tracker/index.html`
 22 - API: `/home/sebas/pi-config/bin/wifi-tracker-api.py`
 23
    ...
 75
 76 ## Retention
 77
 78 The raw JSONL log rotates automatically when it grows past the configured size threshold.
-79 Old rotated files are kept in a small rolling set.
+81 Rotated logs are gzip-compressed and the default keep window is larger now (`ROTATE_KEEP=30`).
 80
+83 The collector also writes `daily-summary.json`, a durable per-device per-day rollup with hours, sessions, first seen, and last seen. That file is rebuilt from retained raw logs and survives even when old fine-grained snapshots age out.
+84
+85 Recent UI/API activity still reads from the live tail for speed; long-range history should use `daily-summary.json`.
+86
 81 Alert toggles are stored in `alert-prefs.json`.
 82 The UI can enable or disable alerts per device without removing the device from the inventory.

--- /home/sebas/pi-config/docs/wifi-tracker.md
+++ /home/sebas/pi-config/docs/wifi-tracker.md
@@ -10,14 +10,16 @@
 ## Pieces

 - collector: `/home/sebas/pi-config/bin/wifi-presence-log`
 - raw events: `/home/sebas/runtime/agent-logs/wifi-presence/activity.jsonl`
-- rotated raw events: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl`
+- rotated raw events: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl.gz`
+- legacy uncompressed rotates: `/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl`
 - state: `/home/sebas/runtime/agent-logs/wifi-presence/state.json`
 - aliases: `/home/sebas/runtime/agent-logs/wifi-presence/aliases.json`
 - fingerprints/evidence: `/home/sebas/runtime/agent-logs/wifi-presence/fingerprints.json`
 - alert prefs: `/home/sebas/runtime/agent-logs/wifi-presence/alert-prefs.json`
 - inventory ever-seen: `/home/sebas/runtime/agent-logs/wifi-presence/inventory.json`
+- persistent daily rollup: `/home/sebas/runtime/agent-logs/wifi-presence/daily-summary.json`
 - summary: `/var/www/wifi-tracker/summary.json`
 - UI: `/var/www/wifi-tracker/index.html`
 - API: `/home/sebas/pi-config/bin/wifi-tracker-api.py`

@@ -75,8 +77,12 @@

 ## Retention

 The raw JSONL log rotates automatically when it grows past the configured size threshold.
-Old rotated files are kept in a small rolling set.
+Rotated logs are gzip-compressed and the default keep window is larger now (`ROTATE_KEEP=30`).

+The collector also writes `daily-summary.json`, a durable per-device per-day rollup with hours, sessions, first seen, and last seen. That file is rebuilt from retained raw logs and survives even when old fine-grained snapshots age out.
+
+Recent UI/API activity still reads from the live tail for speed; long-range history should use `daily-summary.json`.
+
 Alert toggles are stored in `alert-prefs.json`.
 The UI can enable or disable alerts per device without removing the device from the inventory.


**Validating script syntax**

I'm considering validating my script's syntax first, to ensure everything is in order. I think using bash for this purpose makes sense, and I might want to utilize `py_compile` to check the Python code. After that, executing the script could be the next solid step. It feels important to get this right by following the correct order for validation and execution! Let’s make sure everything runs smoothly.

bash

python -m py_compile /home/sebas/pi-config/bin/wifi-presence-log

python - <<'PY'
import gzip,glob,os
for f in sorted(glob.glob('/home/sebas/runtime/agent-logs/wifi-presence/activity-*.jsonl'))[:2]:
    print(os.path.basename(f))
PY


(no output)


activity-20260526-160119.jsonl
activity-20260528-213316.jsonl


**Considering execution strategy**

I need to run the collector, which might take some time but should be doable. I’m wondering if I should prepare a backup plan, or just execute the script once and see how it goes. It’s important to keep everything simple and efficient, but I can’t help but think about possible issues that might arise. Let’s see how this plays out!


find /home/sebas/runtime/agent-logs/wifi-presence -maxdepth 1 -type f | sort | sed 's#^#/##' | xargs -I{} stat -c '%n | size=%s | mtime=%y' {} | grep 'activity-\|daily-summary\|activity.jsonl'

python - <<'PY'
import json
p='/var/www/wifi-tracker/summary.json'
obj=json.load(open(p))
print(obj['retention'])
print('daily devices', len(obj.get('daily_summary',{}).get('devices',{})))
print('sample', list(obj.get('daily_summary',{}).get('devices',{}).items())[:1])
PY


//home/sebas/runtime/agent-logs/wifi-presence/activity-20260526-160119.jsonl | size=4194428 | mtime=2026-05-26 13:01:19.988554171 -0300
//home/sebas/runtime/agent-logs/wifi-presence/activity-20260528-213316.jsonl | size=4194420 | mtime=2026-05-28 18:33:06.193631676 -0300
//home/sebas/runtime/agent-logs/wifi-presence/activity-20260531-101721.jsonl | size=4194545 | mtime=2026-05-31 07:17:20.936282326 -0300
//home/sebas/runtime/agent-logs/wifi-presence/activity-20260602-185215.jsonl | size=4194357 | mtime=2026-06-02 15:52:04.750346827 -0300
//home/sebas/runtime/agent-logs/wifi-presence/activity-20260605-041624.jsonl | size=4194360 | mtime=2026-06-05 01:16:23.440143937 -0300
//home/sebas/runtime/agent-logs/wifi-presence/activity.jsonl | size=3169961 | mtime=2026-06-06 18:44:49.925119195 -0300
//home/sebas/runtime/agent-logs/wifi-presence/daily-summary.json | size=20993 | mtime=2026-06-06 18:44:49.405116082 -0300


{
  "devices": {
    "12:6e:0c:42:eb:c9": {
      "alias": "Macbook sebas",
      "days": {
        "2026-06-06": {
          "first_seen_ts": "2026-06-06T17:05:15Z",
          "hours": 4.69,
          "last_seen_ts": "2026-06-06T21:46:47Z",
          "sessions": 1
        }
      },
      "first_seen_ts": "2026-06-06T17:05:15Z",
      "last_seen_ts": "2026-06-06T21:44:47Z"
    },
    "16:a3:ee:79:d7:8f": {
      "alias": "juan cassani",
      "days": {
        "2026-05-28": {
          "first_seen_ts": "2026-05-28T15:04:22Z",
          "hours": 0.5,
          "last_seen_ts": "2026-05-28T15:34:38Z",
          "sessions": 1
        },
        "2026-06-01": {
          "first_seen_ts": "2026-06-01T13:12:23Z",
          "hours": 0.58,
          "last_seen_ts": "2026-06-01T13:47:24Z",
          "sessions": 1
        },
        "2026-06-05": {
          "first_seen_ts": "2026-06-05T14:28:15Z",
          "hours": 0.84,
          "last_seen_ts": "2026-06-05T15:18:35Z",
          "sessions": 1
        }
      },
      "first_seen_ts": "2026-05-28T15:04:22Z",
      "last_seen_ts": "2026-06-05T15:16:35Z"
    },
    "1c:3c:d4:06:a0:9b": {
      "alias": "Router Casa",
      "days": {
        "2026-05-24": {
          "first_seen_ts": "2026-05-24T16:47:37Z",
          "hours": 7.21,
          "last_seen_ts": "2026-05-25T00:00:00Z",
          "sessions": 1
        },
        "2026-05-25": {
          "first_seen_ts": "2026-05-25T00:00:00Z",
          "hours": 24.0,
          "last_seen_ts": "2026-05-26T00:00:00Z",
          "sessions": 0
        },
        "2026-05-26": {
          "first_seen_ts": "2026-05-26T00:00:00Z",
          "hours": 24.0,
          "last_seen_ts": "2026-05-27T00:00:00Z",
          "sessions": 0
        },
        "2026-05-27": {
          "first_seen_ts": "2026-05-27T00:00:00Z",
          "hours": 24.0,
          "last_seen_ts": "2026-05-28T00:00:00Z",
          "sessions": 0
        },
        "2026-05-28": {
          "first_seen_ts": "2026-05-28T00:00:00Z",
          "hours": 24.0,
          "last_seen_ts": "2026-05-29T00:00:00Z",
          "sessions": 0
        },
        "2026-05-29": {
          "first_seen_ts": "2026-05-29T00:00:00Z",
          "hours": 24.0,
          "last_seen_ts": "2026-05-30T00:00:00Z",
          "sessions": 0
        },
        "2026-05-30": {

[620 more lines in file. Use offset=81 to continue.]


{'rotate_keep': 30, 'rotate_max_bytes': 4194304, 'rotated_compression': 'gzip'}
daily devices 16
sample [('12:6e:0c:42:eb:c9', {'alias': 'Macbook sebas', 'days': {'2026-06-06': {'first_seen_ts': '2026-06-06T17:05:15Z', 'hours': 4.69, 'last_seen_ts': '2026-06-06T21:46:47Z', 'sessions': 1}}, 'first_seen_ts': '2026-06-06T17:05:15Z', 'last_seen_ts': '2026-06-06T21:44:47Z'})]


**Migrating old files**

I realize I need to compress existing old files and possibly convert legacy rotates. Currently, my implementation only handles future rotates. The user asked for conservation, so I should migrate the existing old rotated jsonl files to gz format. It seems like a good plan, and I’m ready to take the next steps to get this done efficiently. Let’s make it happen!

python - <<'PY'
import gzip
from pathlib import Path
log_dir=Path('/home/sebas/runtime/agent-logs/wifi-presence')
for path in sorted(log_dir.glob('activity-*.jsonl')):
    gz=path.with_suffix(path.suffix + '.gz')
    if gz.exists():
        continue
    with path.open('rb') as src, gzip.open(gz,'wb',compresslevel=6) as dst:
        dst.writelines(src)
    path.unlink()
    print('compressed', path.name, '->', gz.name)
PY


compressed activity-20260526-160119.jsonl -> activity-20260526-160119.jsonl.gz
compressed activity-20260528-213316.jsonl -> activity-20260528-213316.jsonl.gz
compressed activity-20260531-101721.jsonl -> activity-20260531-101721.jsonl.gz
compressed activity-20260602-185215.jsonl -> activity-20260602-185215.jsonl.gz
compressed activity-20260605-041624.jsonl -> activity-20260605-041624.jsonl.gz


**Considering data collection**

I’m wondering if I need to rerun the collector. It doesn’t seem necessary right now, but the event_files include gz files, which could be relevant. It’s possible that a summary has already been built from an older jsonl file, and that daily summary persists. It might be helpful to inspect the sizes of these files to understand what's going on. Let’s see what I find!

stat -c '%n | size=%s' /home/sebas/runtime/agent-logs/wifi-presence/activity-* /home/sebas/runtime/agent-logs/wifi-presence/activity.jsonl | sort


/home/sebas/runtime/agent-logs/wifi-presence/activity-20260526-160119.jsonl.gz | size=73655
/home/sebas/runtime/agent-logs/wifi-presence/activity-20260528-213316.jsonl.gz | size=71806
/home/sebas/runtime/agent-logs/wifi-presence/activity-20260531-101721.jsonl.gz | size=73103
/home/sebas/runtime/agent-logs/wifi-presence/activity-20260602-185215.jsonl.gz | size=72138
/home/sebas/runtime/agent-logs/wifi-presence/activity-20260605-041624.jsonl.gz | size=72521
/home/sebas/runtime/agent-logs/wifi-presence/activity.jsonl | size=3169961


**Deciding on memory use**

I’m considering whether I should save memory, but I’m thinking it might not be necessary. It could be a durable decision for the end of the session. However, it feels okay to skip it this time. What I need is a final concise mention that I've completed everything, noted the files that have changed, and summarized the result. Making sure I cover all bases!


Updated #4 (pending → completed)

Revisar actividad últimas 4 semanas desde wifi tracker

Encontrar datos del wifi tracker, calcular actividad de las últimas 4 semanas y responder al usuario.

Revisar metadata y móviles en wifi tracker

Ver si hay más historia por metadata/rotación y contar móviles con perfil de conexión.

Renombrar aliases y explicar retención wifi tracker

Actualizar aliases de móviles indicados por el usuario y revisar por qué rota el log, si se puede recuperar y comprimir.

actualizando aliases y revisando retención

Mejorar retención wifi tracker

Implementar compresión de logs rotados y resumen diario persistente para conservar más historial del wifi tracker con bajo uso de espacio. Actualizar código y docs, validar funcionamiento.


Listo.

Hecho:
- compresión gzip de rotados
- `ROTATE_KEEP` default:
  `5 -> 30`
- resumen diario persistente:
  `daily-summary.json`
- summary web ahora incluye:
  - `daily_summary`
  - `retention`
- docs actualizadas
- migré los rotados viejos a `.gz`

Impacto real:
- cada rotado pasó de ~4.2 MB
  a ~72 KB aprox
- con 30 rotados ahora entra
  muchísimo más historial
- además queda rollup diario
  aunque más adelante venza
  el crudo fino

Archivos:
- `/home/sebas/pi-config/bin/wifi-presence-log`
- `/home/sebas/pi-config/docs/wifi-tracker.md`
- nuevo dato:
  `/home/sebas/runtime/agent-logs/wifi-presence/daily-summary.json`

Validé:
- script compila
- corrí un ciclo real
- summary regenerada OK
- daily summary poblada OK
- rotados comprimidos OK

Estado ahora:
- rotados:
  `activity-*.jsonl.gz`
- actual:
  `activity.jsonl`
- diario persistente:
  `daily-summary.json`

Si querés, siguiente paso útil:
1. exponer historial diario en UI/API
2. subir aún más retención
3. hacer pruning por antigüedad
   además de cantidad


[telegram] te respondi vagamente "arreglalo". en realidad me interesa ver si es mejor sacar la rotacion y guardar todo por ahora. cuanto pesa una semana de datos aprox?


**Calculating file sizes**

I'm working on calculating sizes of files based on observed data. I notice that uncompressed files are around 4.2MB every 2.2 days, while compressed, they are about 72KB for the same timeframe. If I calculate, for a week, that gives about 12.6MB uncompressed and 11MB compressed per year for rotated files. However, keeping everything without rotation leads to significant storage needs, around 650MB a year, which might be manageable but could affect file operations.

**Deciding on file management**

I'm considering how to manage my files efficiently. It seems better to keep the rotation but in a permissive and compressed manner, rather than keeping all raw files in date-partitioned gz format. I should explain that this approach balances storage efficiency with accessibility while maintaining organization. Compressed files save space, making rotation more manageable. I want to ensure I keep data accessible without overwhelming the storage resources.

Sí. Mirándolo frío:

Una semana pesa aprox:

- sin comprimir:
  `~12 a 13 MB / semana`
- comprimido:
  `~200 a 250 KB / semana`

Sale de tus datos reales:
- cada rotado viejo:
  `~4.2 MB`
- cubría aprox:
  `~2.3 días`
- eso da:
  `~1.8 MB/día`
  => `~12.6 MB/semana`

Y ya comprimidos:
- cada rotado quedó en
  `~72 KB`
- eso da:
  `~31 KB/día`
  => `~220 KB/semana`

Entonces:

- guardar todo sin rotación:
  totalmente viable por ahora
- costo anual sin comprimir:
  `~650 MB/año`
- costo anual comprimido:
  `~11 a 12 MB/año`

Pero ojo:
sacar rotación del todo no es lo mejor.

No por espacio.
Por operación.

Problemas de un solo archivo infinito:
- peor manejo si se corrompe
- más incómodo para inspección
- rotación futura más áspera
- algunas lecturas/backs ups son peores

Mejor opción:
- dejar rotación
- pero casi “infinita” para vos
- rotar por tamaño o fecha
- comprimir siempre
- conservar todo

O sea:
- no perder nada
- no crecer mal
- no depender de un archivo gigante

Recomendación concreta:
- mantener rotación
- subir `ROTATE_KEEP` mucho
  o directamente no borrar
- seguir comprimiendo `.gz`

Si querés, hago esto ahora:
1. no borrar nunca rotados
2. seguir rotando y comprimiendo
3. opcional: rotación diaria
   en vez de por tamaño

Esa sería mi elección.


[telegram] 1


todo

Desactivar borrado de rotados wifi tracker

Cambiar wifi tracker para que no elimine logs rotados y conserve todo, manteniendo compresión y rotación.


Created #5: Desactivar borrado de rotados wifi tracker (pending)

Mejorar retención wifi tracker


edit

ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "30"))

ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "0"))

    if ROTATE_KEEP > 0:
        old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
        for path in old[:-ROTATE_KEEP]:
            try:
                path.unlink()
            except FileNotFoundError:
                pass


Successfully replaced 2 block(s) in /home/sebas/pi-config/bin/wifi-presence-log.

     ...
  28 PING_WORKERS = int(os.environ.get("WIFI_PRESENCE_PING_WORKERS", "32"))
  29 HOSTNAME_TIMEOUT = float(os.environ.get("WIFI_PRESENCE_HOSTNAME_TIMEOUT", "0.35"))
  30 SUMMARY_TAIL_LINES = int(os.environ.get("WIFI_PRESENCE_SUMMARY_TAIL_LINES", "20000"))
  31 ROTATE_MAX_BYTES = int(os.environ.get("WIFI_PRESENCE_ROTATE_MAX_BYTES", str(4 * 1024 * 1024)))
- 32 ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "30"))
+ 32 ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "0"))
  33 OFFLINE_GAP_MINUTES = int(os.environ.get("WIFI_PRESENCE_OFFLINE_GAP_MINUTES", "5"))
  34
  35
  36 def sh(cmd: str) -> str:
     ...
  71     LOG_PATH.replace(rotated)
  72     with rotated.open("rb") as src, gzip.open(compressed, "wb", compresslevel=6) as dst:
  73         dst.writelines(src)
  74     rotated.unlink(missing_ok=True)
- 75     old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
- 76     for path in old[:-ROTATE_KEEP]:
- 77         try:
- 78             path.unlink()
- 79         except FileNotFoundError:
- 80             pass
+ 75     if ROTATE_KEEP > 0:
+ 76         old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
+ 77         for path in old[:-ROTATE_KEEP]:
+ 78             try:
+ 79                 path.unlink()
+ 80             except FileNotFoundError:
+ 81                 pass
  81
  82
  83 def load_json(path: Path, default):
  84     try:
     ...

--- /home/sebas/pi-config/bin/wifi-presence-log
+++ /home/sebas/pi-config/bin/wifi-presence-log
@@ -28,9 +28,9 @@
 PING_WORKERS = int(os.environ.get("WIFI_PRESENCE_PING_WORKERS", "32"))
 HOSTNAME_TIMEOUT = float(os.environ.get("WIFI_PRESENCE_HOSTNAME_TIMEOUT", "0.35"))
 SUMMARY_TAIL_LINES = int(os.environ.get("WIFI_PRESENCE_SUMMARY_TAIL_LINES", "20000"))
 ROTATE_MAX_BYTES = int(os.environ.get("WIFI_PRESENCE_ROTATE_MAX_BYTES", str(4 * 1024 * 1024)))
-ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "30"))
+ROTATE_KEEP = int(os.environ.get("WIFI_PRESENCE_ROTATE_KEEP", "0"))
 OFFLINE_GAP_MINUTES = int(os.environ.get("WIFI_PRESENCE_OFFLINE_GAP_MINUTES", "5"))

 def sh(cmd: str) -> str:
@@ -71,14 +71,15 @@
     LOG_PATH.replace(rotated)
     with rotated.open("rb") as src, gzip.open(compressed, "wb", compresslevel=6) as dst:
         dst.writelines(src)
     rotated.unlink(missing_ok=True)
-    old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
-    for path in old[:-ROTATE_KEEP]:
-        try:
-            path.unlink()
-        except FileNotFoundError:
-            pass
+    if ROTATE_KEEP > 0:
+        old = sorted(list(LOG_DIR.glob("activity-*.jsonl")) + list(LOG_DIR.glob("activity-*.jsonl.gz")))
+        for path in old[:-ROTATE_KEEP]:
+            try:
+                path.unlink()
+            except FileNotFoundError:
+                pass

 def load_json(path: Path, default):
     try:


Rotated logs are gzip-compressed and the default keep window is larger now (`ROTATE_KEEP=30`).

Rotated logs are gzip-compressed. Default now is keep forever (`ROTATE_KEEP=0`). Set a positive value only if you want capped retention.


Successfully replaced 1 block(s) in /home/sebas/pi-config/docs/wifi-tracker.md.

    ...
 77
 78 ## Retention
 79
 80 The raw JSONL log rotates automatically when it grows past the configured size threshold.
-81 Rotated logs are gzip-compressed and the default keep window is larger now (`ROTATE_KEEP=30`).
+81 Rotated logs are gzip-compressed. Default now is keep forever (`ROTATE_KEEP=0`). Set a positive value only if you want capped retention.
 82
 83 The collector also writes `daily-summary.json`, a durable per-device per-day rollup with hours, sessions, first seen, and last seen. That file is rebuilt from retained raw logs and survives even when old fine-grained snapshots age out.
 84
 85 Recent UI/API activity still reads from the live tail for speed; long-range history should use `daily-summary.json`.
    ...

--- /home/sebas/pi-config/docs/wifi-tracker.md
+++ /home/sebas/pi-config/docs/wifi-tracker.md
@@ -77,9 +77,9 @@

 ## Retention

 The raw JSONL log rotates automatically when it grows past the configured size threshold.
-Rotated logs are gzip-compressed and the default keep window is larger now (`ROTATE_KEEP=30`).
+Rotated logs are gzip-compressed. Default now is keep forever (`ROTATE_KEEP=0`). Set a positive value only if you want capped retention.

 The collector also writes `daily-summary.json`, a durable per-device per-day rollup with hours, sessions, first seen, and last seen. That file is rebuilt from retained raw logs and survives even when old fine-grained snapshots age out.

 Recent UI/API activity still reads from the live tail for speed; long-range history should use `daily-summary.json`.


bash

python - <<'PY'
import runpy
ns=runpy.run_path('/home/sebas/pi-config/bin/wifi-presence-log')
print('ROTATE_KEEP', ns['ROTATE_KEEP'])
PY


(no output)


ROTATE_KEEP 0


Updated #5 (pending → completed)

Mejorar retención wifi tracker


Hecho.

Cambié:
- `ROTATE_KEEP=0`
- significado:
  no borra rotados nunca
- sigue:
  rotando + comprimiendo `.gz`

Archivo tocado:
- `/home/sebas/pi-config/bin/wifi-presence-log`
- docs:
  `/home/sebas/pi-config/docs/wifi-tracker.md`

Validé:
- compila OK
- config efectiva:
  `ROTATE_KEEP 0`

Resultado:
- no perdés historial
- no te queda un archivo gigante
- espacio sigue siendo mínimo
  porque los rotados van gzip

Si querés, próximo paso útil:
- mostrar en la UI
  “historial total retenido:
  desde X hasta hoy”


/new


queued new Telegram Pi session: pi-telegram


Hecho.