Skip to content

lablink_client_service.heartbeat

Liveness heartbeat for the LabLink client VM.

Sends a small POST to the allocator every HEARTBEAT_INTERVAL_SECONDS so the allocator can detect silent failures (dead container, broken network, hung host, OOM, out-of-band EC2 termination). The body also carries a handful of cheap health signals for early warning.

HEARTBEAT_POST_TIMEOUT_SECONDS = 5 module-attribute

Build the heartbeat JSON payload.

Source code in packages/client/src/lablink_client_service/heartbeat.py 
55
56
57
58
59
60
61
62
def build_payload(vm_id: str | None, boot_id: str | None) -> dict:
    """Build the heartbeat JSON payload."""
    return {
        "vm_id": vm_id,
        "boot_id": boot_id,
        "timestamp": datetime.now(timezone.utc).isoformat(),
        "disk_free_pct": sample_disk_free_pct(),
    }
Source code in packages/client/src/lablink_client_service/heartbeat.py 
124
125
126
127
128
129
130
131
132
133
134
@hydra.main(version_base=None, config_name="config")
def main(cfg: Config) -> None:
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )
    logging.getLogger("lablink_client_service").setLevel(logging.DEBUG)

    base_url, client_secret, _ = get_client_env(cfg)
    run_heartbeat_loop(allocator_url=base_url, client_secret=client_secret)

Read the kernel-assigned per-boot UUID. Cached by the caller.

Source code in packages/client/src/lablink_client_service/heartbeat.py 
33
34
35
36
37
38
39
40
def read_boot_id() -> str | None:
    """Read the kernel-assigned per-boot UUID. Cached by the caller."""
    try:
        with open(BOOT_ID_PATH, "r") as f:
            return f.read().strip()
    except OSError as e:
        logger.warning(f"Could not read boot_id: {e}")
        return None

Long-running loop that sends a heartbeat every interval seconds.

Mirrors the AutoRebootService loop shape in reboot.py: an Event-driven sleep so callers can stop the loop, and an outer exception guard so a rogue exception in a sampler does not silently kill the thread and leave the container running with no liveness signal.

Source code in packages/client/src/lablink_client_service/heartbeat.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def run_heartbeat_loop(
    allocator_url: str,
    client_secret: str = "",
    interval: int = HEARTBEAT_INTERVAL_SECONDS,
    stop_event: threading.Event | None = None,
) -> None:
    """Long-running loop that sends a heartbeat every `interval` seconds.

    Mirrors the AutoRebootService loop shape in reboot.py: an Event-driven
    sleep so callers can stop the loop, and an outer exception guard so a
    rogue exception in a sampler does not silently kill the thread and
    leave the container running with no liveness signal.
    """
    logger.info("Starting heartbeat loop")
    base_url = sanitize_url(allocator_url)
    headers = {"Content-Type": "application/json"}
    headers.update(get_auth_headers(client_secret))

    vm_id = os.getenv("VM_NAME")
    boot_id = read_boot_id()

    if stop_event is None:
        stop_event = threading.Event()

    while not stop_event.is_set():
        try:
            payload = build_payload(vm_id=vm_id, boot_id=boot_id)
            send_heartbeat(base_url=base_url, headers=headers, payload=payload)
        except Exception:
            logger.exception("Heartbeat iteration failed; continuing")
        stop_event.wait(interval)

Return integer percent of free space on the filesystem at path.

Source code in packages/client/src/lablink_client_service/heartbeat.py 
43
44
45
46
47
48
49
50
51
52
def sample_disk_free_pct(path: str = "/") -> int:
    """Return integer percent of free space on the filesystem at `path`."""
    try:
        usage = shutil.disk_usage(path)
        if usage.total == 0:
            return 0
        return int((usage.free / usage.total) * 100)
    except OSError as e:
        logger.debug(f"disk_free_pct probe failed: {e}")
        return 0

POST one heartbeat. Swallows network errors — never raises.

Heartbeat integrity is in the allocator noticing when we stop sending. A failed individual POST is not worth crashing the client.

Source code in packages/client/src/lablink_client_service/heartbeat.py 
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def send_heartbeat(
    base_url: str,
    headers: dict,
    payload: dict,
) -> None:
    """POST one heartbeat. Swallows network errors — never raises.

    Heartbeat integrity is in the allocator noticing when we *stop*
    sending. A failed individual POST is not worth crashing the client.
    """
    try:
        response = requests.post(
            f"{base_url}/api/heartbeat",
            json=payload,
            headers=headers,
            timeout=HEARTBEAT_POST_TIMEOUT_SECONDS,
        )
        if response.status_code >= 400:
            logger.debug(
                f"Heartbeat POST returned {response.status_code}: "
                f"{response.text[:200]}"
            )
    except requests.exceptions.RequestException as e:
        logger.debug(f"Heartbeat POST failed: {e}")