Failed to fetch: status={{ resp.status_code }}
{{ resp.text }}
{% else %}
{{ resp.text }}
{% endif %}
{% endfor %}
{% endblock %}
a
{% extends "base.html" %}
{% block header %}
{{ fetch_summary }}{% endif %}
Failed to fetch: status={{ resp.status_code }}
{{ resp.text }}
{% else %}
{% endif %}
{% endfor %}
{% endblock %}
a
{% extends "base.html" %}
{% block header %}
{% for k, v in zip(keys, values) -%}
{{ k }}: {{ v | truncate(100) }}
{% endfor %}
{% endblock %}
c F ] tR t^tR R ltR R ltR R ltR R ltR tR
# ) IndexHandlerc V ^8 d QhRR/# returnz list[Route] )formats "h/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/distributed/debug/_debug_handlers.py __annotate__IndexHandler.__annotate__ s * * * c 0 \ R V P 4 .# )/r _handleselfs &r routesIndexHandler.routes s c4<<())r" c V ^8 d QhRR/# r r z
list[NavLink]r )r s "r r r! s & &= &r" c \ R R4 .# )r$ Homer
r' s &r nav_linksIndexHandler.nav_links s V$%%r" c V ^8 d QhRR/# r r zdict[str, str]r )r s "r r r! s . .> .r" c R \ /# z
index.html)INDEX_TEMPLATEr' s &r templatesIndexHandler.templates s
n--r" c V ^8 d QhRRRR/# r reqr r bytesr )r s "r r r! s : :- :% :r" c 8 VP P R 4 # r5 )frontendrender_template)r( r; s &&r r& IndexHandler._handle s ||++L99r" r N __name__
__module__ __qualname____firstlineno__r) r0 r7 r& __static_attributes__r r" r r r s *&.: :r" r c R ] tR t^tR R ltR R ltR R ltR R ltR R
ltR t R # )
StacksHandlerc V ^8 d QhRR/# r r )r s "r r StacksHandler.__annotate__ s 0 0 0r" c 0 \ R V P 4 .# )/stacksr% r' s &r r) StacksHandler.routes s i .//r" c V ^8 d QhRR/# r, r )r s "r r rJ s ; ;= ;r" c \ R R4 .# )rL zPython Stack Tracesr/ r' s &r r0 StacksHandler.nav_links s #89::r" c V ^8 d QhRRRR/# r: r )r s "r r rJ
-
%
r" c p \ R V P R7 w r#VP P RRW#R7 # )dump_tracebacktimeoutz
raw_resp.htmlStackstitleaddrsrespsr
fetch_timeoutr> r? r( r; rZ r[ s && r r& StacksHandler._handle s: !14;M;MN ||++ 85 ,
r" c V ^8 d QhRR/# r r z
str | Noner )r s "r r rJ s j r" c \ R V P R7 w r. p\ W4 pV' d# VP V4 VP R4 \ \ W4 4 F[ w pw rgVP RV RV R24 TP VP ^8X d
VP MRVP 24 K] RP V4 # )rT rU === Rank : ===Error:
r r] r append enumeratezip status_codetextjoinr( rZ r[ partssummaryiaddrresps & r dumpStacksHandler.dump s !14;M;MN &u4 LL ! LL (U): ;OA| LL9QCr$t4 5 LL ! - - 4 GDDTDTCU:V
<
yy r" c V ^8 d QhRR/# r r strr )r s "r r rJ s s r" c R # )stacksr r' s &r
dump_filenameStacksHandler.dump_filename s r" r N
rB rC rD rE r) r0 r& rv r} rF r r" r rH rH s 0;
r" rH c ^ ] tR t^tR R ltR R ltR R ltR R ltR R
ltR R lt R
t
R# ) PySpyHandlerc V ^8 d QhRR/# r r )r s "r r PySpyHandler.__annotate__ s 4 4 4r" c 0 \ R V P 4 .# ) /pyspy_dumpr% r' s &r r) PySpyHandler.routes s mT\\233r" c V ^8 d QhRR/# r, r )r s "r r r s 9 9= 9r" c \ R R4 .# )r z
py-spy Stacksr/ r' s &r r0 PySpyHandler.nav_links s
788r" c V ^8 d QhRR/# r3 r )r s "r r r s 8 8> 8r" c R \ /# )pyspy_dump.html)PYSPY_DUMP_TEMPLATEr' s &r r7 PySpyHandler.templates s !#677r" c V ^8 d QhRRRR/# r: r )r s "r r r s
-
%
r" c VP 4 pR V9 d V' d RV 2MRp\ RW P R7 w r4VP P RVVR7 # ) nonblockingznonblocking=1&
nonblocking=1
pyspy_dumprU r rZ r[ )
get_raw_queryr r] r> r? )r( r; queryrZ r[ s && r r& PySpyHandler._handle sa !!# %05nUG,?E u>P>PQ ||++ ,
r" c V ^8 d QhRR/# ra r )r s "r r r s j r" c \ R RV P R7 w r. p\ W4 pV' d# VP V4 VP R4 \ \ W4 4 F[ w pw rgVP RV RV R24 TP VP ^8X d
VP MRVP 24 K] RP V4 # ) r r rU rc rd re rf rg rh ri rp s & r rv PySpyHandler.dump s /43E3E
&u4 LL ! LL (U): ;OA| LL9QCr$t4 5 LL ! - - 4 GDDTDTCU:V
<
yy r" c V ^8 d QhRR/# ry r )r s "r r r s s r" c R # )r r r' s &r r} PySpyHandler.dump_filename s r" r N rB rC rD rE r) r0 r7 r& rv r} rF r r" r r r s% 498
r" r c ] tR tRtR R ltR R ltR R lt]R R l4 tR
R lt R R
lt
R R lt R R lt R R lt
R R ltR R ltRtR# )FlightRecorderHandleri c V ^8 d QhRR/# r r )r s "r r "FlightRecorderHandler.__annotate__
s
r" c \ R V P 4 \ RV P 4 \ RV P 4 \ RV P 4 .# ) /fr_trace/fr_trace_json/fr_trace_nccl/fr_trace_nccl_json)r _handle_fr_trace_handle_fr_trace_json_handle_fr_trace_nccl_handle_fr_trace_nccl_jsonr' s &r r) FlightRecorderHandler.routes
sO +t 4 4 5 "D$>$> ? "D$>$> ? ')H)H I
r" c V ^8 d QhRR/# r, r )r s "r r r s
=
r" c ^ \ R R4 \ RR4 \ RR4 \ RR4 .# )r zFlightRecorder CPUr (JSON)r FlightRecorder NCCLr r/ r' s &r r0 FlightRecorderHandler.nav_links s9 K!5 6 $h / $&; < )8 4
r" c V ^8 d QhRR/# r3 r )r s "r r r 4 4> 4r" c R \ /#
fr_trace.htmlFR_TRACE_TEMPLATEr' s &r r7 FlightRecorderHandler.templates !233r" c $ V ^8 d QhRRRRRR/# )r rZ list[str]r[ list[Response]r r r )r s "r r r s! 0 0 0> 0h 0r" c \ 4 pVP . R 7 pRVn RVn / p\ V4 FH w rVVP
^8w d K RVRW, /VP
4 CpRV9 d . VR& WtRV R2& KJ V' g \ R4 h\ \ VP 4 4 4 R, p\ WCV4 # ))argsTrank host_nameentriesz.jsonz All workers failed to respondversion) r
parse_argsallow_incomplete_ranksverboserk rm json RuntimeErrornextitervaluesr ) rZ r[ configr detailsr ru rv r s && r _build_db FlightRecorderHandler._build_db s b )&*# #E*JD 3&U[ ))+D
$"$Y*.d4&& ' + >? ?tGNN ,-.y9w//r" c ( V ^8 d QhRRRRRRRR/# r serverr rZ r r[ r r r< r )r s "r r r 8 ,
$
-6
?M
r" c V P W#4 pVP R R\ W#4 \ VP \
P RR7 \ VP \ P RR7 \ VP \ P RR7 \ VP \ P RR7 R7 # )r FlightRecorderhtmlheaderstablefmtrY
fetch_summarygroups memberships collectives ncclcalls)
r r? r r r r _fieldsr r r r r r r( r rZ r[ dbs &&&& r _render_tables$FlightRecorderHandler._render_tables8 s ^^E
)%% ".u<BIIu}}vN
(:(:V !
(:(:V r||X5E5EPVW &
r" c V ^8 d QhRRRR/# r: r )r s "r r r J s E E$6 E5 Er" c \ R V P R7 w r#V P VP V\ V4 4 # )
fr_trace_jsonrU r r] r r> listr^ s && r r &FlightRecorderHandler._handle_fr_traceJ s3 $:L:LM ""3<<U DDr" c V ^8 d QhRRRR/# r: r )r s "r r r N s
);
r" c r \ R V P R7 w r#VP P RRVVR7 # )r rU json_resp.htmlr rX r\ r^ s && r r +FlightRecorderHandler._handle_fr_trace_jsonN s> $:L:LM ||++ " ,
r" c V ^8 d QhRRRR/# r: r )r s "r r r W s E E); E Er" c \ R RV P R7 w r#V P VP V\ V4 4 # )dump_nccl_trace_jsononlyactive=truerU r r^ s && r r +FlightRecorderHandler._handle_fr_trace_ncclW s; "$5t?Q?Q
""3<<U DDr" c V ^8 d QhRRRR/# r: r )r s "r r r ] s
.@
U
r" c t \ R RV P R7 w r#VP P RRVVR7 # )r r rU r r rX r\ r^ s && r r 0FlightRecorderHandler._handle_fr_trace_nccl_json] sF "$5t?Q?Q
||++ ' ,
r" c V ^8 d QhRR/# ra r )r s "r r r h s 9 9 j 9 r" c , . p\ R V P R7 w r#\ W#4 pV' d# VP V4 VP R4 V P W#4 pVP RR\
VP \ P RR7 R\
VP \ P RR7 R\
VP \ P RR7 R \
VP \ P RR7 . 4 \ R
R V P R7 w rgV P Wg4 pVP RR R\
VP \ P RR7 R\
VP \ P RR7 R\
VP \ P RR7 R \
VP \ P RR7 .
4 RP# V4 # \ d TP R
4 L/i ; i)r rU rc z=== FR Trace ===--- Groups ---plainr --- Memberships ------ Collectives ------ NCCL Calls ---r r z=== FR Trace NCCL ===z1
=== FR Trace NCCL ===
Failed to fetch NCCL tracerh )r r] r rj r extendr r r r r r r r r r Exceptionro ) r( rq rZ r[ rr r
nccl_addrs
nccl_respsnccl_dbs & r rv FlightRecorderHandler.dumph s $:L:LM &u4 LL ! LL
^^E
)
" EMMGL% 1C1CgV% 1C1CgV$ x/?/?'R
P%.&(94CUCU& "J nnZ{{ fetch_summary }}{% endif %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}
Failed to fetch: status={{ resp.status_code }}
{{ resp.text }}
{% else %}
{{ resp.text }}
{% endif %}
{% endfor %}
{% if dump_triggered %}
Failed to dump: status={{ resp.status_code }}
{{ resp.text }}
{% else %}
{{ resp.text }}
{% endif %}
{% endfor %}
{% endif %}
{% endblock %}
c x ] tR tRtRtR R ltR R ltR R lt]R R
l4 t R R lt
R
R lt R R lt Rt
R# ) TorchCommsHealthCheckHandlerig zBHealth check that detects watchdog timeouts and triggers FR dumps.c V ^8 d QhRR/# r r )r s "r r )TorchCommsHealthCheckHandler.__annotate__j s A A Ar" c 0 \ R V P 4 .# )/torchcomms_health_checkr% r' s &r r) #TorchCommsHealthCheckHandler.routesj s 0$,,?@@r" c V ^8 d QhRR/# r, r )r s "r r r m s J J= Jr" c \ R R4 .# )r zTorchComms Healthr/ r' s &r r0 &TorchCommsHealthCheckHandler.nav_linksm s 24GHIIr" c V ^8 d QhRR/# r3 r )r s "r r r p s G G> Gr" c R \ /# ) torchcomms_health_check.html)HEALTH_CHECK_TEMPLATEr' s &r r7 &TorchCommsHealthCheckHandler.templatesp s .0EFFr" c V ^8 d QhRRRR/# )r r[ r r boolr )r s "r r r t s n r" c
V FB pVP ^8X g K VP 4 pTP RR4 ' d KA R# R# \ d \ P R4 Kk i ; i)a Return True if any rank explicitly reports itself as unhealthy.
Only responses with a 200 status code are inspected. Non-200
responses (connection errors, timeouts, etc.) are intentionally
ignored so that a single unreachable rank does not trigger an
expensive Flight Recorder dump across the entire job. A dump is
only triggered when a rank is reachable *and* positively reports
``{"healthy": false}``.
zGfailed to parse health check response as JSON; treating rank as healthyhealthyTF)rm r r logger exceptionget)r[ ru datas & r _any_unhealthy+TorchCommsHealthCheckHandler._any_unhealthys sp D 3& 99;D xx 400 ! $$3 s A A/.A/c V ^8 d QhRRRR/# r: r )r s "r r r s
-
%
r" c
\ R V P R7 w r#Rp. p. pV P V4 ' d \ RV P R7 w rVRpVP P R\ W#4 VVVVVR7 # )torchcomms_health_checkrU Ftorchcomms_fr_dump_fileTr )r rZ r[ dump_triggered
dump_addrs
dump_resps)r r] r r> r? r )r( r; rZ r[ r r r s && r r& $TorchCommsHealthCheckHandler._handle s !:DDVDVW "
%'
u % %%.)43E3E& "J "N||++ *.u<)!! ,
r" c V ^8 d QhRR/# ra r )r s "r r r s j r" c 4 \ R V P R7 w r. p\ W4 pV' d# VP V4 VP R4 \ \ W4 4 F[ w pw rgVP RV RV R24 TP VP ^8X d
VP MRVP 24 K] V P V4 ' d VP R4 VP R4 \ RV P R7 w r\ \ W4 4 F[ w pw rgVP R V RV R
24 TP VP ^8X d
VP MRVP 24 K] R P V4 # ) r rU rc rd re rf rg z3=== Unhealthy rank detected, triggering FR dump ===r z --- Rank z ---rh )
r r] r rj rk rl rm rn r ro )
r( rZ r[ rq rr rs rt ru r r s
& r rv !TorchCommsHealthCheckHandler.dump sg !:DDVDVW &u4 LL ! LL (U): ;OA| LL9QCr$t4 5 LL ! - - 4 GDDTDTCU:V
<
u % % LL LLN O%.)43E3E& "J $-S-H#I