Diagnostic of the dask task failures caused by dead dask workers.
When the dask workers running a task die (e.g. the worker pod is killed by Kubernetes when it exceeds its
memory limit), the task fails with a KilledWorker exception that doesn't tell why. Run from the
rs-dpr-service side, this module reads the dask scheduler event log to explain the worker deaths.
The real cause is only known by Kubernetes (pod status OOMKilled, Evicted, ...), which the rs-dpr-service
cannot read: the explanations below are the probable causes deduced from the dask events.
explain_task_failure(dask_client, exception)
Explain why the dask workers running a failed task died.
Parameters:
| Name |
Type |
Description |
Default |
dask_client
|
Client | None
|
dask client connected to the cluster that ran the task
|
required
|
exception
|
BaseException
|
exception raised by the dask future
|
required
|
Returns:
| Type |
Description |
str
|
The explanation, to append to the error message, or an empty string if the task failure was not caused
|
str
|
by dead workers or if the explanation cannot be retrieved.
|
Source code in docs/rs-dpr-service/rs_dpr_service/dask/worker_diagnostics.py
| def explain_task_failure(dask_client: DaskClient | None, exception: BaseException) -> str:
"""
Explain why the dask workers running a failed task died.
Args:
dask_client: dask client connected to the cluster that ran the task
exception: exception raised by the dask future
Returns:
The explanation, to append to the error message, or an empty string if the task failure was not caused
by dead workers or if the explanation cannot be retrieved.
"""
if not dask_client or not isinstance(exception, KilledWorker):
return ""
lines = [f"Diagnostic of the dask workers that died while running the task {exception.task!r}:"]
last_worker = exception.last_worker
memory_limit = getattr(last_worker, "memory_limit", None)
if memory_limit:
lines.append(f"Dask worker memory limit: {format_bytes(memory_limit)}")
try:
removals = [
(timestamp, msg)
for timestamp, msg in _get_events(dask_client, "all")
if msg.get("action") == "remove-worker" and exception.task in msg.get("processing-tasks", ())
]
dead_workers = {msg.get("worker") for _, msg in removals}
# Workers restarted by their nanny because they exceeded the dask memory budget
memory_restarts = {
msg.get("worker"): msg
for _, msg in _get_events(dask_client, "worker-restart-memory")
if msg.get("worker") in dead_workers
}
except Exception as error: # pylint: disable=broad-exception-caught
logger.warning(f"Cannot read the dask scheduler events: {error}")
lines.append(f"Cannot read the dask scheduler events: {error}")
lines.append(_HINT)
return "\n".join(lines)
if not removals:
lines.append("No worker removal found in the dask scheduler events.")
# Group the dead workers by cause, keep the chronological order
causes: dict[str, list[str]] = {}
for timestamp, msg in removals:
worker = msg.get("worker")
detail = f"{worker} at {_format_time(timestamp)}"
if worker in memory_restarts:
cause = _MEMORY_RESTART_CAUSE
detail += f" (memory: {format_bytes(memory_restarts[worker].get('rss', 0))})"
else:
cause = _removal_cause(str(msg.get("stimulus_id", "")))
causes.setdefault(cause, []).append(detail)
for cause, workers in causes.items():
lines.append(f"- {len(workers)} worker(s): {cause}. Workers: {', '.join(workers)}")
lines.append(_HINT)
return "\n".join(lines)
|