+
    –°³ja ã                  óÌ  € R t ^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	t	^ RI
t
^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RIHtHt ^ RIHtHtHtHt ^ RIt^ RIHt ^ RIHt ^ R	IH t  ^ R
I!H"t"H#t# ^ RI$H%t% ^ RI&H't'H(t(H)t) ^ RI*H+t+H,t, ^ RI-H.t.H/t/H0t0H1t1H2t2H3t3H4t4H5t5 ^ RI6H7t7H8t8H9t9H:t:H;t;H<t<H=t=H>t>H?t?H@t@HAtA ^ RIBHCtC ^ RIDHEtE ^ RIFHGtG ^ RIHHItJ ^ RIKHLtL ^ RIMHNtN ]'       d'   ^ RIOHPtPHQtQHRtRHStSHTtT ^ RIUHVtV ^ RIWHXtX ^ RIYHZtZ ^ RI[H\t\ ]]t^]]t_]]t`]! RRR7      ta]PÄ                  PÆ                  PÉ                  4       '       d
   ^ RIeHftgHhth M ! R R 4      tgR! R" lth]PÒ                  PÕ                  ]kR#4      tlR$ R% ltm^R&InHoto ]Pà                  ! R'R(7       ! R) R*4      4       tqR+ R, ltr]Pæ                  R- R. l4       tt]Pæ                  R/ R0 l4       tu]Pæ                  R1 R2 l4       tvR3 R4 ltw ! R5 R64      txR7 R8 lty]Pô                  ! 4       tz]y! ]z4       ]Pö                  ! 4       t| ! R9 R:4      t}R; R< lt~R= R> ltR? R@ lt€RA RB lt�RC RD lt‚R�RE RF lltƒRG RH lt„RI RJ lt…]Pæ                  RK RL l4       t†RƒRMRRNR‚ROR‚RPR‚RQRRRSR‚RTR/RU RV lllt‡ ! RW R4      tˆRX RY lt‰RZ R[ ltŠ]Pæ                  R\ R] l4       t‹]Pæ                  R^ R_ l4       tŒR` Ra lt�]Ž]]]]3,          t�]Ž]�]ˆR,          ,          ]�]NR,          ,          ]�]R,          ,          R,          ]]3,          t‘]Ž]‘Rb3,          t’]�]�]“,          ,          t”]�]•R,          ,          t– ! Rc Rd4      t—]�t˜]�t™]�tš ! Re Rf4      t› ! Rg Rh]›4      tœ]œ! 4       t� ! Ri Rj]›4      tž ! Rk Rl]›4      tŸ ! Rm Rn4      t Ro Rp lt¡Rq Rr lt¢Rs Rt lt£R„Ru Rv llt¤Rw Rx lt¥Ry Rz lt¦ ! R{ R|]4      t§ ! R} R~]4      t¨ ! R R€4      t©R# )…aµ  
CUDA graph trees are a safety abstraction over CUDAGraphs, similar to make_graph_callables,
which share the same memory pool.  Sharing a memory pool is an extremely
important optimization when chaining multiple CUDA graphs together, as it
prevents you from needing to copy intermediate tensors from one graph to the
next, and reduces overall memory usage by allowing dead memory from the first
pool to be reused in the second.

The standard graph/make_graph_callables support sharing memory pool, but
with a lot of caveats.  CUDA graph trees remove these restrictions:

* Previously, if you recorded graphs A, B, you had to replay A, B in that
  order.  With CUDA graph trees, after replaying A, you can change your
  mind and record/replay a different graph B'; we will support efficient
  execution of both A, B and A, B', using only max(mem(A, B), mem(A, B')).  In
  other words: we support arbitrary trees of CUDA graph operations, not just
  sequences (this is why this feature is called CUDA graph trees.)

* Previously, if you executed graph A, some non-CUDA graph code, and then
  graph B, after executing graph B, it was not safe to retain any references
  to intermediates produced by A.  With CUDA graph trees, we track if any
outputs of graph A are still live by the time graph B is run, and make
  sure graph B doesn't clobber there memory when reusing the CUDA graphs
  pool.  You'll get a separate recording of B depending on what tensors
  stay live or dead.

CUDA graph trees are flexible enough to be used in Dynamo across graph breaks,
which is their primary use case.

The ability to switch from replay to record is fairly nontrivial: remember that
when you replay a CUDA graph, you only replay CUDA operations; no CPU side state
is updated.  In particular, the CPU-side book-keeping for the allocator is not
reconstructed.  However, to record a new child CUDA graph, we must restore this
book-keeping.  This is what checkpoint pool state is used for.
)ÚannotationsN©Údefaultdict)ÚAbstractContextManager)ÚautoÚEnum)ÚAnyÚcastÚTYPE_CHECKINGÚTypeVar)ÚTensor)ÚCustomClassBase)ÚCallbackTrigger)ÚCURRENT_STREAM_INDEXÚset_external_object_by_index)ÚGenerationTracker)ÚcountersÚdynamo_timedÚpreserve_rng_state)ÚControlFlowOpWarmupDispatchModeÚ)CUDAGraphCaptureControlFlowOpDispatchMode)Úalign_inputs_from_check_idxsÚcopy_misaligned_inputsÚcopy_strided_storage_Úget_expanded_dimsÚget_input_idxs_to_checkÚindex_expanded_dimsÚremove_unaligned_input_idxsÚstatic_input)Úcheck_for_mutationÚCheckInvariantStatusÚcollect_cuda_data_ptrsÚ
FunctionIDÚ#log_cudagraph_skip_and_bump_counterÚlog_data_ptr_mismatchÚ"maybe_warning_due_to_dynamic_shapeÚ	ModelTypeÚ
OutputTypeÚPlaceholderInfoÚWrappedFunction©Úis_custom_class_obj)ÚStorageWeakRef)ÚUntypedStorage)Ú_pytree)Ú
OrderedSet)ÚTensorWeakRef)ÚCallableÚ	GeneratorÚIterableÚIteratorÚSequence)Ú	CompileId)Ú	InputType)Ú_POOL_HANDLE)Ú_boolÚSÚStorageWeakRefWrapper)Úbound)Ú"_cuda_CUDAAllocator_AllocatorStateÚ_set_cached_tensors_enabledc                  ó   € ] tR t^ztRtR# )ÚAllocatorState© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__rA   ó    Ú`/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/_inductor/cudagraph_trees.pyr@   r@   z   s   † ÛrG   r@   c               ó    € V ^8„  d   QhRRRR/# )é   Úenabledr9   ÚreturnÚNonerA   )Úformats   "rH   Ú__annotate__rO   }   s   € ÷ ñ ¨Uð °tñ rG   c                ó   € R # ©NrA   ©rK   s   &rH   r>   r>   }   s   € ÙrG   Ú
cudagraphsc               ó    € V ^8„  d   QhRRRR/# )rJ   Úinputsú	list[Any]rL   ÚstrrA   )rN   s   "rH   rO   rO   „   s   € ÷ ñ ˜ið ¨Cñ rG   c                óø   € R  R lp\        V 4      pV^ 8X  d   R# ^
p\        V RV 4       UUu. uF  w  rEV! WE4      NK  	  pppW#8”  d   VP                  RW#,
           R24       RP                  V4      # u uppi )c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   ÚiÚintÚinpr   rL   rW   rA   )rN   s   "rH   rO   Ú'format_inputs_log.<locals>.__annotate__…   s!   € ÷ 	9ñ 	9�sð 	9 ð 	9¨ñ 	9rG   c                ó4  € \        V\        P                  4      '       dQ   R V  R\        VP	                  4       4       RVP                  4        RVP                   RVP                  4       R R2# Vf   R V  R2# R V  R\        V4      P                   R	V R2# )
Ú[z]: Tensor(size=z	, stride=z, dtype=z, data_ptr=0xÚXÚ)z]: Nonez]: Ú()
Ú
isinstanceÚtorchr   ÚlistÚsizeÚstrideÚdtypeÚdata_ptrÚtyperB   )rZ   r\   s   &&rH   Úformat_itemÚ&format_inputs_log.<locals>.format_item…   sž   € Ü�cœ5Ÿ<™<×(Ò(à�A�3�o¤d¨3¯8©8«:Ó&6Ð%7°yÀÇÁÃÀð OØŸ™˜ =°·±³ÀÐ0BÀ!ðEðð Š[Ø�q�c˜�>Ð!à�q�c˜œT #›Y×/Ñ/Ð0°°#°°aÐ8Ð8rG   z[]Nz... (z more items)ú, )ÚlenÚ	enumerateÚappendÚjoin)rU   rk   ÚnÚ	max_itemsrZ   r\   Úpartss   &      rH   Úformat_inputs_logru   „   s}   € õ	9ô 	ˆF‹€AØˆA„vÙð €IÜ/8¸À
ÀÐ9KÔ/LÔMÑ/L¡V Q‰[˜Ö Ñ/L€EÑMØ„}Ø�‰�u˜Q�]˜O¨<Ð8Ô9à�9‰9�UÓÐùó	 Ns   ®A6)ÚconfigT)Úfrozenc                  ó&   € ] tR t^ t$ RtR]R&   RtR# )ÚGraphIDz(Unique counter of a cuda graph recordingr[   ÚidrA   N)rB   rC   rD   rE   Ú__doc__Ú__annotations__rF   rA   rG   rH   ry   ry       s
   ‡ á.à‡GrG   ry   c               ó   € V ^8„  d   QhRR/# ©rJ   rL   rM   rA   )rN   s   "rH   rO   rO   §   s   € ÷ +ñ +˜Tñ +rG   c                 óB   € \         P                  P                  4        R# )aU  
Cublas keeps a persistent workspace allocation for running matmuls. This poses a problem for
doing warmup within a CUDAGraph private pool because we do not want persistent allocations from
one one run to the next. When we begin a new run of a cudagraphs path (generation), all tensors
from the previous generation are freed. This frees them the memory pool, but not elsewhere.
A tensor in the cublas workspace would continue to be in use the workspace but would also get allocated
in the next run. The memory would be in use in two places.

To solve this, we clear cublas caches before and after warming up or recording. If a workspace is required
it will be allocated to the cudagraph private pool and accounted for in the allocator for the duration of the
program. There is no overhead to this on replay since cudagraphs removes allocation overhead.
N)rd   Ú_CÚ_cuda_clearCublasWorkspacesrA   rG   rH   Úclear_cublass_cacher‚   §   s   € ô 
‡H�H×(Ñ(Ö*rG   c               ó   € V ^8„  d   QhRR/# ©rJ   rL   úGenerator[None, None, None]rA   )rN   s   "rH   rO   rO   ¸   s   € ÷ ñ Ð9ñ rG   c               #  ó^   "  € \        4         Rx € \        4        R#   \        4        i ; i5i)zOContext manager around clearing cublas caches that will clear on enter and exitN)r‚   rA   rG   rH   Úclear_cublas_managerr‡   ·   s"   é € ô ÔðÛäÖøÔÕüs   ‚-Ž ’-ž*ª-c               ó   € V ^8„  d   QhRR/# r„   rA   )rN   s   "rH   rO   rO   Â   s   € ÷ =ñ =Ð%@ñ =rG   c               #  ó  "  € \         P                  P                  4       p \         P                  P                  R 4        Rx € \         P                  P                  V 4       R#   \         P                  P                  T 4       i ; i5i©FN)rd   r€   Ú$_cuda_get_conv_benchmark_empty_cacheÚ%_cudnn_set_conv_benchmark_empty_cache)Úprevs    rH   Údisable_conv_cache_emptyingrŽ   Á   sU   é € ä�8‰8×8Ñ8Ó:€DÜ	‡H�H×2Ñ2°5Ô9ð=Ûä�‰×6Ñ6°tÖ<øŒ�‰×6Ñ6°tÕ<üs   ‚>B
ÁA& Á!B
Á&!BÂB
c               ó   € V ^8„  d   QhRR/# r„   rA   )rN   s   "rH   rO   rO   Ì   s   € ÷ 	;ñ 	;Ð"=ñ 	;rG   c               #  óˆ  "  € \         P                  P                  4       p  V '       g)   \         P                  P                  P                  4        Rx € V '       g,   \         P                  P                  P                  R4       R# R#   T '       g+   \         P                  P                  P                  R4       i i ; i5i)z8Turns on history recording in the CUDA Caching AllocatorN)rd   r€   Ú_cuda_isHistoryEnabledÚcudaÚmemoryÚ_record_memory_historyrR   s    rH   Úenable_history_recordingr•   Ë   sz   é € ô �h‰h×-Ñ-Ó/€Gð;ßÜ�J‰J×Ñ×4Ñ4Ô6ÛçÜ�J‰J×Ñ×4Ñ4°TÖ:ñ ø�wÜ�J‰J×Ñ×4Ñ4°TÕ:ð üs   ‚C¢4B Á5CÂ4B?Â?Cc               ó   € V ^8„  d   QhRR/# )rJ   rL   zAbstractContextManager[None]rA   )rN   s   "rH   rO   rO   Ø   s   € ÷ &ñ &Ð;ñ &rG   c                 ó€   € \         P                  P                  '       g   \        P                  ! 4       # \        4       # rQ   )rv   ÚtritonÚ!cudagraph_trees_history_recordingÚ
contextlibÚnullcontextr•   rA   rG   rH   Úget_history_recordingrœ   Ø   s*   € ä�=‰=×:×:Ð:Ü×%Ò%Ó'Ð'Ü#Ó%Ð%rG   c                  ób   € ] tR t^ßtRtR R ltR R ltR R ltR R	 ltR
 R lt	R R lt
RtR# )ÚTreeManagerContainera^  
Manages the lifetime of the tree manager. Like `PrivatePool` in cuda caching allocator,
the tree and its corresponding memory pool should be kept alive as long as any outstanding
graph or tensor which is an output of a graph remains alive.

There is a single tree manager container per device.

The lifecycle of a tree_manager is:
-  Is constructed, no graph, no fns, no tensors
-  Tree manager is fetched, resulting in tree manager being allocated
-  We generate a bunch of functions, calling add_strong_reference
-  These functions die, calling finalize_reference
-  When all the functions die, we finalize_tree_manager.

TODO: in the future, we would like to do the following once storage weak refs land
-  We look for all the live storages and add references to THOSE
-  We count as storages die
-  All the storages are dead, we deallocate the tree manager
c               ó    € V ^8„  d   QhRRRR/# ©rJ   Údevice_indexr[   rL   rM   rA   )rN   s   "rH   rO   Ú!TreeManagerContainer.__annotate__ô   s   € ÷ %ñ % Sð %¨Tñ %rG   c                	ó~   € R V n         ^ V n        Wn        ^ V n        R V n        \
        P                  ! 4       V n        R # rQ   )Útree_managerÚlive_cudagraphify_fnsr¡   Úlive_storages_countÚgraphÚ	threadingÚLockÚlock©Úselfr¡   s   &&rH   Ú__init__ÚTreeManagerContainer.__init__ô   s;   € ð :>ˆÔð &'ˆÔ"à(Ôð
 $%ˆÔ Ø26ˆŒ
ä—N’NÓ$ˆŽ	rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¢     s   € ÷ 	-ñ 	- $ñ 	-rG   c                	óú   € V P                   ;_uu_ 4        V ;P                  ^,          un        V P                  ^ 8X  d    RV n        V P                  ^ 8X  d   RV n        RRR4       R#   + '       g   i     R# ; i©é   N)rª   r¦   r§   r¥   r¤   ©r¬   s   &rH   Ú_finalize_tensorÚ%TreeManagerContainer._finalize_tensor  sU   € Ø�Y�Y‹YØ×$Ò$¨Õ)Õ$Ø×'Ñ'¨1Ô,Ø!�”
ð ×-Ñ-°Ô2Ø(,�DÔ%÷ �Y�YŠYús   •A
A)Á)A:	c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¢     s   € ÷ .ñ .¨$ñ .rG   c                	óÜ   € V P                   ;_uu_ 4        V ;P                  ^,          un        V P                  ^ 8X  d   V P                  4        RRR4       R#   + '       g   i     R# ; ir±   )rª   r¥   Ú_finalize_tree_managerr³   s   &rH   Úfinalize_cudagraphify_fnÚ-TreeManagerContainer.finalize_cudagraphify_fn  sB   € Ø�Y�Y‹YØ×&Ò&¨!Õ+Õ&Ø×)Ñ)¨QÔ.Ø×+Ñ+Ô-÷ �Y�YŠYús   •;AÁA+	c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¢     s   € ÷ !ñ !¨ñ !rG   c                	ój   € V P                   P                  4       '       g   \        R 4      hRV n        R# )zexpected lock to be heldN)rª   ÚlockedÚAssertionErrorr¤   r³   s   &rH   r¸   Ú+TreeManagerContainer._finalize_tree_manager  s)   € Ø�y‰y×Ñ×!Ò!Ü Ð!;Ó<Ð<Ø ˆÖrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   ÚfnzCallable[..., Any]rL   rM   rA   )rN   s   "rH   rO   r¢   1  s   € ÷ <ñ <Ð'9ð <¸dñ <rG   c                	óØ   € V P                   ;_uu_ 4        V ;P                  ^,          un        RRR4       \        P                  ! WP                  4       R#   + '       g   i     L2; ir±   )rª   r¥   ÚweakrefÚfinalizer¹   )r¬   rÁ   s   &&rH   Úadd_strong_referenceÚ)TreeManagerContainer.add_strong_reference1  s?   € Ø�Y�Y‹YØ×&Ò&¨!Õ+Õ&÷ ô 	×Ò˜×:Ñ:Ö;÷ �Yús   •AÁA)	c               ó   € V ^8„  d   QhRR/# )rJ   rL   ÚCUDAGraphTreeManagerrA   )rN   s   "rH   rO   r¢   7  s   € ÷ %ñ %Ð"6ñ %rG   c                	óÐ   € V P                   ;_uu_ 4        V P                  f   \        V P                  4      V n        V P                  uuR R R 4       #   + '       g   i     R # ; irQ   )rª   r¤   rÈ   r¡   r³   s   &rH   Úget_tree_managerÚ%TreeManagerContainer.get_tree_manager7  s?   € Ø�Y�Y‹YØ× Ñ Ò(Ü$8¸×9JÑ9JÓ$K�Ô!Ø×$Ñ$÷ �Y�Y‹Yús   •4AÁA%	)r¡   r§   r¥   r¦   rª   r¤   N)rB   rC   rD   rE   r{   r­   r´   r¹   r¸   rÅ   rÊ   rF   rA   rG   rH   rž   rž   ß   s*   † ñõ(%õ&	-õ.õ!õ2<÷%ñ %rG   rž   c               ó    € V ^8„  d   QhRRRR/# )rJ   Úlocalzthreading.localrL   rM   rA   )rN   s   "rH   rO   rO   >  s   € ÷ Oñ O˜?ð O¨tñ OrG   c                óø   € / V n         \        \        P                  4      V n        \
        P                  P                  R V P                   4       \
        P                  P                  RV P                  4       R# ©Útree_manager_containersÚtree_manager_locksN)rÐ   r   r¨   r©   rÑ   rd   r€   Ú_stash_obj_in_tls)rÍ   s   &rH   Ú_initialize_tlsrÓ   >  sS   € à$&€EÔ!Ü*¬9¯>©>Ó:€EÔô 
‡H�H×ÑÐ8¸%×:WÑ:WÔXÜ	‡H�H×ÑÐ3°U×5MÑ5MÖNrG   c                  ó   € ] tR tRt^ tRtR# )ÚMarkStepBoxiT  rA   N)rB   rC   rD   rE   Úmark_step_counterrF   rA   rG   rH   rÕ   rÕ   T  s   † ØÔrG   rÕ   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   rO   X  s   € ÷ 'ñ '˜ñ 'rG   c                 ó@   € \         ;P                  ^,          un        R# )zJIndicates that a new iteration of inference or training is about to begin.N©rÕ   rÖ   rA   rG   rH   Úmark_step_beginrÚ   X  s   € ô ×!Ò! QÕ&×!rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   rO   _  s   € ÷ )ñ ) ñ )rG   c                 óJ   € \        RR7      p V e   V P                  4        R# R# )z¬Request another warmup for the active CUDA Graph Trees function.

This is a no-op unless called synchronously from a function currently running
in CUDA Graph Trees warmup.
F)Úcreate_if_none_existsN)Úget_managerÚmark_warmup_incomplete)Úmanagers    rH   rß   rß   _  s%   € ô °Ô6€GØÒØ×&Ñ&Ö(ñ rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   rO   j  s   € ÷ &ñ &˜tñ &rG   c                 ó   € \         P                  P                  R4      '       g   R# \        \        R4      p \        \        R4      pVP                  4        Fh  w  r#T;_uu_ 4        V P                  V4      pV'       d   VP                  '       g    RRR4       KF  VP                  P                  4        RRR4       Kj  	  \        R4       V P                  4        ^ \        n        R#   + '       g   i     K¥  ; i)zClear all cudagraph treesrÐ   NrÑ   F)rd   r€   Ú_is_key_in_tlsÚget_objrÍ   ÚitemsÚgetr¤   Úshutdownr>   ÚclearrÕ   rÖ   )Úcontainer_dictÚ
locks_dictÚdevicerª   Ú	containers        rH   Úreset_cudagraph_treesrí   j  s¼   € ô
 �8‰8×"Ñ"Ð#<×=Ò=áÜœUÐ$=Ó>€NÜœÐ 4Ó5€JØ"×(Ñ(Ö*‰ˆßŠTØ&×*Ñ*¨6Ó2ˆIß I×$:×$:Ð$:Ø÷ ‰Tð
 ×"Ñ"×+Ñ+Ô-÷ ‰Tñ +ô   Ô&Ø×ÑÔà$%„KÖ!÷ �TˆTús   Á'+C,ÂC,ÂC,Ã,C=c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   rÍ   r   Ú	attr_namerW   rL   rA   )rN   s   "rH   rO   rO   ‚  s!   € ÷ /ñ /�3ð / 3ð /¨3ñ /rG   c                óî   € \        W4      '       d   \        W4      # \        P                  P	                  V4      '       g   \        V 4       \        W4      # \        P                  P                  V4      # rQ   )ÚhasattrÚgetattrrd   r€   rã   rÓ   Ú_get_obj_in_tls)rÍ   rï   s   &&rH   rä   rä   ‚  sU   € Üˆu× Ò Ü�uÓ(Ð(Ü�8‰8×"Ñ" 9×-Ò-Ü˜ÔÜ�uÓ(Ð(Ü�8‰8×#Ñ# IÓ.Ð.rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   r¡   r[   rL   rž   rA   )rN   s   "rH   rO   rO   ‹  s   € ÷ ,ñ , ð ,Ð(<ñ ,rG   c                óØ   € \        \        R 4      p\        \        R4      V ,          pT;_uu_ 4        W9  d   \        V 4      W&   W,          uuRRR4       #   + '       g   i     R# ; irÏ   )rä   rÍ   rž   )r¡   ré   rª   s   &  rH   Úget_containerrö   ‹  sK   € ÜœUÐ$=Ó>€NÜ”5Ð.Ó/°Õ=€Dç	ŠØÔ-Ü+?ÀÓ+MˆNÑ(àÕ+÷	 
��‹ús   ²AÁA)	c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   r¡   ú
int | NonerÝ   ÚboolrL   zCUDAGraphTreeManager | NonerA   )rN   s   "rH   rO   rO   –  s)   € ÷ Añ AØðAØ<@ðAà ñArG   c                ó6  € V fE   \         P                  P                  4       '       g   R # \         P                  P                  4       p V'       d   \	        V 4      P                  4       # \        \        R4      P                  V 4      pVf   R # VP                  # )NrÐ   )
rd   r’   Úis_initializedÚcurrent_devicerö   rÊ   rä   rÍ   ræ   r¤   )r¡   rÝ   rì   s   && rH   rÞ   rÞ   –  sw   € ð ÒÜ�z‰z×(Ñ(×*Ò*ÙÜ—z‘z×0Ñ0Ó2ˆçÜ˜\Ó*×;Ñ;Ó=Ð=äœÐ8Ó9×=Ñ=¸lÓK€IØÒ$ˆ4Ð@¨)×*@Ñ*@Ð@rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   Úint_keyzint | tuple[int, ...]rL   rù   rA   )rN   s   "rH   rO   rO   ¥  s   € ÷ ñ Ð(=ð À$ñ rG   c                ó„   € \         P                  P                  RJ ;'       g    V \         P                  P                  9   # )zh
Returns true if all dynamic shapes should be captured or the dynamic shape
int_key should be captured.
N)rv   r˜   Úcudagraph_capture_sizes)rþ   s   &rH   Úis_cudagraph_capture_sizesr  ¥  s7   € ô 	�‰×-Ñ-°Ð5÷ 	<ð 	<Ø”f—m‘m×;Ñ;Ñ;ðrG   c               ó0   € V ^8„  d   QhRRRRRRRRR	RR
R/# )rJ   Úmodelr&   rU   úlist[InputType]Ústatic_input_idxsúSequence[int]Úargsr   ÚkwargsrL   rA   )rN   s   "rH   rO   rO   °  sD   € ÷ :!ñ :!Øð:!àð:!ð %ð:!ð ð	:!ð
 ð:!ð ñ:!rG   c                óò   a aaaa	a
a€ / o	\        V4       UUu. uF  w  rV\        V\        4      '       g   K  VNK!  	  pppV'       d   \        P                  ! V!  MR  o
Ro?R VV	V
VVV V3R llpV# u uppi )c                ó   € R # rQ   rA   ©Ú_s   &rH   Ú<lambda>Ú#cudagraphify_impl.<locals>.<lambda>»  s   € ÉDrG   Fc               ó    € V ^8„  d   QhRRRR/# )rJ   rU   r  rL   r'   rA   )rN   s   "rH   rO   Ú'cudagraphify_impl.<locals>.__annotate__Á  s   € ÷ 'ñ ' oð '¸*ñ 'rG   c                óÔ  <€ S
! V 4      p\        V4      '       g	   S! V 4      # S	P                  V4      pVe	   V! V 4      # SP                  RR4      pVf   \        P                  RV4       M\        P                  RW14       S'       g   \	        S	V4      o\        V S4      p\        V S4      p\        W4       \        SW.SO5/ SB w  r&\        4       p\        W$VR7      pVS	V&   V# )NÚ
compile_idÚ z7[%s] Recording cudagraph tree for graph without symintsz/[%s] Recording cudagraph tree for symint key %s)Úinputs_to_checkÚmutated_input_idxs)r  ræ   ÚlogÚinfor%   r   r   r   Úcudagraphifyr/   r   )rU   rþ   rÁ   r  Úcheck_input_idxsÚnew_static_input_idxsÚoutr  r  Úfn_cacheÚget_intsÚhas_warnr  r  r  s   &       €€€€€€€rH   Údeferred_cudagraphifyÚ0cudagraphify_impl.<locals>.deferred_cudagraphifyÁ  sò   ø€ ñ ˜6Ó"ˆä)¨'×2Ò2Ù˜“=Ð à�\‰\˜'Ó"ˆØŠ>Ù�f“:ÐØ—Z‘Z ¨bÓ1ˆ
ØŠ?Ü�H‰HØIÈ:õô �H‰HØAÀ:ô÷ Ü9¸(ÀGÓLˆHô 3°6Ð;LÓMÐÜ ;¸FÐDUÓ VÐÜ˜vÔ8ä˜u fÐUÀdÒUÈfÑU‰ˆä.8«lÐÜ)ØÐEWô
ˆð ˆ�Ñàˆ
rG   )ro   rc   r[   ÚoperatorÚ
itemgetter)r  rU   r  r  r  rZ   Úvrþ   r  r  r  r  s   f&fjl    @@@rH   Úcudagraphify_implr$  °  sj   þ€ ð ;=€Hô ' vÔ.ÔEÑ.‘T�Q´*¸QÄ×2D�qˆqÑ.€GÑEß5<”H×'Ò'¨Ò1Á.€Hà€Hà÷'ö 'ðR !Ð ùóa Fs
   ˜A3·A3c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rJ   ÚnamerW   r  úCompileId | NoneÚmodeúCompilationMode | NonerL   zGenerator[Any, None, None]rA   )rN   s   "rH   rO   rO   î  s0   € ÷ ñ Ø
ðà ðð !ðð  ñ	rG   c              #  ó¢   "  € \        V RVV\        P                  8H  RR7      ;_uu_ 4        Rx € RRR4       R#   + '       g   i     R# ; i5i)z÷
Makes usages of dynamo_timed in this file less verbose. NOTE: This CM sums
all durations into a single column in the dynamo_compile table. Use only if
you consider the timed region to be part of the runtime overhead associated
with the compiler.
TÚruntime_cudagraphify_time_us)Úlog_pt2_compile_eventr  Úis_backwardÚdynamo_compile_column_usN)r   ÚCompilationModeÚBACKWARD)r&  r  r(  s   &&&rH   Údynamo_timed_cudagraphr1  í  sG   é € ô 
ØØ"ØØœO×4Ñ4Ñ4Ø!?÷
ö 
ó 	÷
÷ 
÷ 
ò 
üs   ‚*A¬;±
A»A	Á	AÚstack_tracesÚ	constantsÚplaceholdersr  Úkernel_free_cudagraphFÚuser_visible_output_idxsr  c               óP   € V ^8„  d   QhRRRRRRRRR	R
RR
RRRRRRRRRR
RRRRRR/# )rJ   r  r&   rU   r  r  r  r¡   r[   r-  rù   Úis_inferencer2  úStackTraces | Noner3  útuple[torch.Tensor, ...]r4  útuple[PlaceholderInfo, ...]r  útuple[int, ...]r5  r6  r  r'  rL   útuple[ModelType, OutputType]rA   )rN   s   "rH   rO   rO     s”   € ÷ 'ñ 'Øð'àð'ð %ð'ð
 ð'ð ð'ð ð'ð %ð'ð (ð'ð .ð'ð (ð'ð  ð'ð .ð'ð !ð'ð "ñ'rG   c       
        ó€  € V'       d   V'       d   \        R 4      hV'       d   \        P                  M'V'       d   \        P                  M\        P                  p\        RWÍ4      ;_uu_ 4        \        V4      P                  4       pRRR4       XP                  V VVVVVVV	V
VV4      #   + '       g   i     L+; i)z+expected not (is_backward and is_inference)zcudagraphify.get_containerN)	r¾   r/  r0  Ú	INFERENCEÚFORWARDr1  rö   rÊ   Úadd_function)r  rU   r  r¡   r-  r8  r2  r3  r4  r  r5  r6  r  r(  rà   s   &&&$$$$$$$$$$  rH   r  r    s¦   € ÷  —|ÜÐJÓKÐK÷ ô 	× Ò ç+7Œo×'Ò'¼_×=TÑ=Tð 	ô 
 Ð <¸j×	OÕ	OÜ Ó-×>Ñ>Ó@ˆ÷ 
Pð ×ÑØØØØØØØØØØ Øóð ÷ 
P×	Oús   Á0B-Â-B=	c                  ó    € ] tR tRt$ Rt. ROtR]R&   RR R llt]RR R	 ll4       t	R
 R lt
R R ltR R ltR R ltR R ltR R ltRtR# )r;   i-  zS
Wrapper around a storage weak ref. Will deallocate it upon expiration if invoked.
zStorageWeakRef | NoneÚstorage_refNc               ó$   € V ^8„  d   QhRRRRRR/# )rJ   r\   zTensor | UntypedStorageÚextra_ref_checkúCallable[[], bool] | NonerL   rM   rA   )rN   s   "rH   rO   Ú"StorageWeakRefWrapper.__annotate__6  s(   € ÷ /ñ /à$ð/ð 3ð/ð 
ñ	/rG   c                ó  € \        V\        4      '       d   VP                  4       pM/\        V\        4      '       g   \	        R\        V4       24      hTp\        V4      V n        VP                  4       V n	        W n
        R# )zÇ
extra_ref_check is an additional check we need to run to check if the
weak ref has expired. in checking storage use count we assume extra_ref_check
will hold an additional reference to the storage.
zexpected UntypedStorage, got N)rc   r   Úuntyped_storager-   r¾   rj   r,   Úrefri   Ú	_data_ptrrE  )r¬   r\   rE  Ústors   &&& rH   r­   ÚStorageWeakRefWrapper.__init__6  sg   € ô �cœ6×"Ò"Ø×&Ñ&Ó(‰Dä˜c¤>×2Ò2Ü$Ð'DÄTÈ#ÃYÀKÐ%PÓQÐQØˆDÜ! $Ó'ˆŒØŸ™›ˆŒØ.ÖrG   c          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )rJ   Úclsztype[StorageWeakRefWrapper]Úcdatar   ri   r[   rE  rF  rL   r;   rA   )rN   s   "rH   rO   rG  K  s:   € ÷ 
ñ 
Ø(ð
àð
ð ð
ð 3ð	
ð
 
ñ
rG   c                	óv   € V P                  V 4      pW$n        \        P                  ! V4      Vn        W4n        V# rQ   )Ú__new__rK  r,   Úfrom_weakrefrJ  rE  )rO  rP  ri   rE  Úinstances   &&&& rH   Úfrom_weakref_and_data_ptrÚ/StorageWeakRefWrapper.from_weakref_and_data_ptrJ  s5   € ð —;‘;˜sÓ#ˆØ%ÔÜ%×2Ò2°5Ó9ˆŒØ#2Ô ØˆrG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   zStorageWeakRefPointer | NonerA   )rN   s   "rH   rO   rG  W  s   € ÷ ñ Ð6ñ rG   c                	ó^   € V P                  4       '       d   R # V P                  P                  # rQ   )ÚexpiredrJ  rP  r³   s   &rH   Ú__call__ÚStorageWeakRefWrapper.__call__W  s   € Ø�<‰<�>Š>Ùà�x‰x�~‰~ÐrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   rP  r   rL   rM   rA   )rN   s   "rH   rO   rG  ]  s   € ÷ ñ  #ð ¨$ñ rG   c                	óZ   € V P                   P                  4        WP                   n        R # rQ   )rJ  Ú__del__rP  )r¬   rP  s   &&rH   Úswap_weakrefÚ"StorageWeakRefWrapper.swap_weakref]  s   € Ø�‰×ÑÔØ�‰ŽrG   c               ó   € V ^8„  d   QhRR/# ©rJ   rL   r[   rA   )rN   s   "rH   rO   rG  a  s   € ÷ ñ ˜#ñ rG   c                ó   € V P                   # )z8NB: returns the data ptr even if the storage has expired)rK  r³   s   &rH   ri   ÚStorageWeakRefWrapper.data_ptra  s   € à�~‰~ÐrG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   rG  e  s   € ÷ $ñ $¨ñ $rG   c                	ó   € R V n         R # rQ   ©rE  r³   s   &rH   Úremove_extra_referenceÚ,StorageWeakRefWrapper.remove_extra_referencee  s
   € Ø#ˆÖrG   c               ó   € V ^8„  d   QhRR/# ©rJ   rL   rù   rA   )rN   s   "rH   rO   rG  h  s   € ÷ ñ ˜ñ rG   c                	ó  € V P                   e   V P                  4       '       g   R# \        P                  P                  V P                  P
                  4      pV P                   e
   V^,          pV^ 8  d   \        RV 24      hV^ 8H  # )NFzexpected stor_count >= 0, got )rE  rd   r€   Ú_storage_Use_CountrJ  rP  r¾   )r¬   Ú
stor_counts   & rH   rY  ÚStorageWeakRefWrapper.expiredh  sv   € Ø×ÑÒ+°D×4HÑ4H×4JÒ4JÙä—X‘X×0Ñ0°·±·±Ó@ˆ
Ø×ÑÒ+ð ˜!�OˆJØ˜Œ>Ü Ð#AÀ*ÀÐ!NÓOÐOØ˜Q‰ÐrG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   rW   rA   )rN   s   "rH   rO   rG  v  s   € ÷ Hñ H˜#ñ HrG   c                	ó®   € V P                   e!   V P                   P                  4       '       d   RV P                  4        R2# RV P                  4        R2# )NzStorageWeakRefWrapper to z; deadz; alive)rJ  rY  ri   r³   s   &rH   Ú__repr__ÚStorageWeakRefWrapper.__repr__v  sJ   € Ø�8‰8Ò˜tŸx™x×/Ñ/×1Ò1Ø.¨t¯}©}«Ð.?¸vÐFÐFà.¨t¯}©}«Ð.?¸wÐGÐGrG   )rK  rE  rJ  )rJ  rK  rE  rQ   )rB   rC   rD   rE   r{   Ú	__slots__r|   r­   ÚclassmethodrU  rZ  r_  ri   rh  rY  rr  rF   rA   rG   rH   r;   r;   -  sR   ‡ ñò 8€Ià&Ó&÷/ð( ö
ó ð
õõõõ$õ÷Hñ HrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   Úweak_refúStorageWeakRefWrapper | NonerL   rù   rA   )rN   s   "rH   rO   rO   }  s   € ÷ -ñ -Ð2ð -°tñ -rG   c                ó   € \        V 4      R J# rQ   )Úmaybe_deref)rw  s   &rH   Úis_liver{  }  s   € Ü�xÓ ¨Ð,Ð,rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   rw  rx  rL   z(tuple[StorageWeakRefPointer, int] | NonerA   )rN   s   "rH   rO   rO   �  s   € ÷ 	"ñ 	"Ø*ð	"à-ñ	"rG   c                óJ   € V f   R # V ! 4       pVf   R # WP                  4       3# rQ   )ri   )rw  Úrs   & rH   rz  rz  �  s.   € ð ÒÙÙ‹
€AØ‚yÙà×ÑÓ!Ð!Ð!rG   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rJ   rë   r[   Úmem_poolútuple[int, int]Ústreamútorch.cuda.StreamrL   r…   rA   )rN   s   "rH   rO   rO   Ž  s,   € ÷ 4ñ 4Øð4Ø*ð4Ø4Eð4à ñ4rG   c           
   #  óD  "  € \         P                  P                  4        VP                  \         P                  P	                  4       4       \         P                  P                  V4      ;_uu_ 4        \         P                  ! V 4      ;_uu_ 4        \         P                  P                  W4        Rx € \         P                  P                  W4       \         P                  P                  W4        RRR4       RRR4       \         P                  P	                  4       P                  V4       R#   \         P                  P                  Y4       \         P                  P                  Y4       i ; i  + '       g   i     LŠ; i  + '       g   i     L•; i5i)ai  
Context manager to use cuda graph pool for new allocations. If you use this manager
all cudagraph tensors in use should be reflected in the allocator or they will be overwritten.
existing_graph should already have been used in a capture, and the mem_pool must already exist,
because this manager will not preserve a reference to the pool which keeps it alive.
N)rd   r’   ÚsynchronizeÚwait_streamÚcurrent_streamr‚  rë   r€   Ú&_cuda_beginAllocateCurrentThreadToPoolÚ_cuda_endAllocateToPoolÚ_cuda_releasePool)rë   r€  r‚  s   &&&rH   Ú_use_cuda_memory_pool_managerr‹  �  sñ   é € ô 
‡J�J×ÑÔØ
×Ñ”u—z‘z×0Ñ0Ó2Ô3ä	�‰×	Ñ	˜6×	"Õ	"¤E§L¢L°×$8Õ$8ô 	�‰×7Ñ7¸ÔIð	9Ûä�H‰H×,Ñ,¨VÔ>Ü�H‰H×&Ñ& vÕ8÷ %9×	"ô 
‡J�J×ÑÓ×+Ñ+¨FÖ3øô �H‰H×,Ñ,¨VÔ>Ü�H‰H×&Ñ& vÕ8ú÷ %9×$8ú×	"×	"üsO   ‚A3F Á5FÂ E:	Â5D7Â9>E:	Ã8FÄ 7F Ä7A E7Å7E:	Å:F
ÆFÆF	ÆF c               ó   € V ^8„  d   QhRR/# r„   rA   )rN   s   "rH   rO   rO   ©  s   € ÷ 	
ñ 	
Ð0Kñ 	
rG   c               #  ón   "  € \        \        \        P                  P	                  4       4       Rx € R# 5i)aZ  Update the external object registry so custom ops see the capture stream.

During cudagraph recording/warmup the current stream differs from the
trace-time default stream.  The external object at CURRENT_STREAM_INDEX
must reflect the actual current stream so that custom ops (e.g. event
record/wait) executed during capture use the right stream.
N)r   r   rd   r’   r‡  rA   rG   rH   Ú&_update_current_stream_external_objectrŽ  ¨  s"   é € ô !Ô!5´u·z±z×7PÑ7PÓ7RÔSÝ	ùs   ‚35c               ó    € V ^8„  d   QhRRRR/# )rJ   ÚtzTensor | NonerL   rx  rA   )rN   s   "rH   rO   rO   µ  s   € ÷ $ñ $�-ð $Ð$@ñ $rG   c                ó€   € \        V \        P                  4      '       g   V e   \        RV  24      hR # \	        V 4      # )Nzexpected t to be None, got )rc   rd   r   r¾   r;   )r�  s   &rH   Ú
map_to_refr’  µ  s9   € Ü�aœŸ™×&Ò&ØŠ=Ü Ð#>¸q¸cÐ!BÓCÐCÙÜ  Ó#Ð#rG   .c                  ó–   € ] tR tRtRtR R lt]P                  R R l4       tR R lt	]
R	 R
 l4       tR R ltR R ltR R ltRtR# )ÚCUDAWarmupNodeiÏ  a  
Simplified Wrapper around A CUDA Model that wraps outputs in storage refs and exposes
apis to get the live storages in the current chain of warmup.

A CUDAWarmupNode may have either CUDAGraphNode or CUDAWarmupNode as a parent, but may only have
CUDAWarmupNode as children, because we cannot record or execute with tensors which do not have stable
memory addresses.

CUDAWarmupNode and CUDAGraphNode have a number of differences that make it easier to use separate classes.
- Much of the CUDAGraphNode logic & initialization is based on the tensor properties of first recording. In the
first instance of warmup, these are not finalized yet.
- All Inputs to the RecordedFunction must be copied over to the cuda graph memory pool, this is unnecessary in warmup.
- CUDAWarmup is only used once and so does not need to optimize as much bookkeeping. It is much simpler.

NB: this class and CUDAGraphNode need to expose `path_live_weakrefs`, `all_outputs_are_dead`, and
`self.outputs_weakrefs`, `stack_traces`, and `tensor_weakrefs` for compatibility.
c               ó@   € V ^8„  d   QhRRRRRRRRR	R
RRRRRRRRRR/
# )rJ   Úwrapped_functionr)   Úparentú%CUDAGraphNode | CUDAWarmupNode | NoneÚcuda_graphs_poolr�  Úexisting_cuda_graphztorch.cuda.CUDAGraph | Noner¡   r[   r2  r9  r‚  rƒ  Úalready_warmrù   rz   ry   rL   rM   rA   )rN   s   "rH   rO   ÚCUDAWarmupNode.__annotate__â  sn   € ÷ ñ à)ðð 6ðð *ð	ð
 9ðð ðð )ðð "ðð ðð ðð 
ñrG   c
                	ó¾   € Wn         VP                  V n        W n        W0n        . V n        . V n        W@n        R V n        WPn        W`n	        Wpn
        W€n        W�n        R# rŠ   )r–  r6  r—  r™  Úoutputs_weakrefsÚtensor_weakrefsrš  Úhas_runr¡   r2  r‚  r›  rz   )
r¬   r–  r—  r™  rš  r¡   r2  r‚  r›  rz   s
   &&&&&&&&&&rH   r­   ÚCUDAWarmupNode.__init__â  s\   € ð !1ÔØ(8×(QÑ(QˆÔ%Ø=CŒØ 0ÔØDFˆÔØ;=ˆÔØ#6Ô ØˆŒØ(ÔØ(ÔØŒØ(ÔØŽrG   c               ó   € V ^8„  d   QhRR/# ©rJ   rL   ú'tuple[PathUserVisibleStorageGroup, ...]rA   )rN   s   "rH   rO   rœ  ý  ó   € ÷ Uñ Uà	0ñUrG   c                	ó>   € \        \        V P                  4      4      # rQ   ©Ú(collect_path_user_visible_storage_groupsÚtupleÚ_path_from_rootr³   s   &rH   Ú path_user_visible_storage_groupsÚ/CUDAWarmupNode.path_user_visible_storage_groupsü  ó   € ô 8¼¸d×>RÑ>RÓ8SÓTÐTrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   Ú
new_inputsr   rL   r'   rA   )rN   s   "rH   rO   rœ    s   € ÷ Lñ L˜cð L jñ LrG   c                	ó  a aaa€ S P                   '       d   \        R 4      h\        S P                  4        Uu. uF!  q"! 4       '       g   K  VP	                  4       NK#  	  up4      oR VVV 3R llpV! 4       p\
        P                  P                  '       dM   S P                  '       g;   \        S P                  4       4      p\        S P                  S P                  V4       \        P                  P                  S P                  4      ;_uu_ 4        \!        4       ;_uu_ 4        \#        4       ;_uu_ 4        \%        S P                  S P                  S P&                  4      ;_uu_ 4        \)        4       ;_uu_ 4        \+        4       ;_uu_ 4        \-        4       ;_uu_ 4        S P.                  P1                  S4      pRRR4       RRR4       RRR4       RRR4       RRR4       RRR4       RRR4       \        \2        ,          ! 4       oV F+  pV! 4       pVf   K  SP5                  VP6                  4       K-  	  \9        S4      ^ 8w  d   \        R\9        S4       24      hR V3R llp	S P:                  P=                  X U
u. uF  q©! V
4      '       d   \?        V
4      MRNK  	  up
4       S P@                  P=                  V U
u. uF  q©! V
4      '       d   \C        V
4      MRNK  	  up
4       \
        P                  P                  '       dM   S P                  '       g;   \        S P                  4       4      p\        S P                  S P                  V4       V# u upi   + '       g   i     EL±; i  + '       g   i     EL½; i  + '       g   i     ELÉ; i  + '       g   i     ELÕ; i  + '       g   i     ELá; i  + '       g   i     ELí; i  + '       g   i     ELù; iu up
i u up
i )z*Wrapped function should never be run twicec               ó   € V ^8„  d   QhRR/# )rJ   rL   z+list[weakref.ReferenceType[UntypedStorage]]rA   )rN   s   "rH   rO   Ú(CUDAWarmupNode.run.<locals>.__annotate__  s   € ÷ 	&ñ 	&Ð(Sñ 	&rG   c                 óX  <€ \         P                  ! SSP                  P                  4       U u. uFn  p \	        V \
        P                  4      '       g   K%  V P                  4       P                  4       S9  g   KJ  \        P                  ! V P                  4       4      NKp  	  pp V# u up i rQ   )Ú	itertoolsÚchainr–  r3  rc   rd   r   rI  ri   rÃ   rJ  )r�  Únon_cudagraph_inpsÚexisting_path_data_ptrsr¯  r¬   s     €€€rH   Úget_non_cudagraph_inpsÚ2CUDAWarmupNode.run.<locals>.get_non_cudagraph_inps  s�   ø€ ô #Ÿš¨°T×5JÑ5J×5TÑ5TÔUó"áU�AÜ˜a¤§¡×.ô 1ð ×%Ñ%Ó'×0Ñ0Ó2Ð:QÑQô 1”—’˜A×-Ñ-Ó/Ö0ÙUð ð "ð &Ð%ùò"s   ¯"B'Á B'Á;(B'Nú%expected new_inputs to be empty, got c               ó    € V ^8„  d   QhRRRR/# )rJ   Úor   rL   rù   rA   )rN   s   "rH   rO   r²  ;  s   € ÷ 	ñ 	�sð 	˜tñ 	rG   c                óü   <€ \        V \        P                  4      ;'       d[    V P                  ;'       dG    V P	                  4       P
                  S9  ;'       d"    V P	                  4       P                  4       ^ 8g  # ©é    )rc   rd   r   Úis_cudarI  Ú_cdatari   )r¼  Únon_cudagraph_inps_storage_ptrss   &€rH   Úadd_refÚ#CUDAWarmupNode.run.<locals>.add_ref;  so   ø€ ä˜1œeŸl™lÓ+÷ 8ð 8Ø—I‘I÷8ð 8à×%Ñ%Ó'×.Ñ.Ð6UÑU÷8ð 8ð ×%Ñ%Ó'×0Ñ0Ó2°aÑ7ð	rG   )"r   r¾   r/   Úpath_live_weakrefsri   rv   r˜   Úslow_path_cudagraph_assertsr›  re   Úcheck_memory_poolr¡   r™  rd   r’   rë   rŽ   r‡   r‹  r‚  rŽ  r   rœ   r–  r  r   ÚaddrÁ  rn   rž  Úextendr’  rŸ  r0   )r¬   r¯  r�  r¸  Únon_cudagraph_inps_storagesÚrefsr  ÚstorageÚsrÃ  r¼  Úout_refsr·  rÂ  s   ff          @@rH   ÚrunÚCUDAWarmupNode.run  sè  û€ Ø�<�<ˆ<Ü Ð!MÓNÐNô #-Ø#'×#:Ñ#:Ô#<ÓDÑ#<˜aÀÇŒ\ˆQ�Z‰ZŽ\Ñ#<ÑDó#
Ð÷	&ò 	&ñ '=Ó&>Ð#ä�=‰=×4×4Ð4¸T×=N×=NÐ=NÜ˜×/Ñ/Ó1Ó2ˆDÜ˜d×/Ñ/°×1FÑ1FÈÔMô �J‰J×Ñ˜d×/Ñ/×0Õ0Ü'×)Õ)Ü ×"Õ"Ü)Ø×!Ñ! 4×#8Ñ#8¸$¿+¹+÷õ ô 3×4Õ4Ü+×-Õ-Ü!×#Õ#à×'Ñ'×-Ñ-¨jÓ9ˆC÷ $÷ .÷ 5÷	÷ #÷ *÷ 1ô& +5´S®/Ó*;Ð'Û2ˆGÙ“	ˆAØŒ}Ø/×3Ñ3°A·H±HÖ=ñ 3ô
 ˆz‹?˜aÔÜ Ø7¼¸J»Ð7HÐIóð ÷
	ð 	ð 	×Ñ×$Ñ$Ù<?Ó@¹C°q˜g aŸjšjŒZ˜Œ]¨dÒ2¹CÑ@ô	
ð 	×Ñ×#Ñ#Ù?BÓC¹s¸! ¨§¢Œ]˜1Ô°Ò5¹sÑCô	
ô �=‰=×4×4Ð4¸T×=N×=NÐ=NÜ˜D×3Ñ3Ó5Ó6ˆHÜ˜d×/Ñ/°×1FÑ1FÈÔQàˆ
ùòK E÷8 $×#Ð#ú÷ .×-Ð-ú÷ 5×4Ð4ú÷	÷ ð ú÷ #×"Ð"ú÷ *×)Ð)ú÷ 1×0Ð0üòP Aùò Ds³   ¹M(ÁM(ÄO%Ä)O	Ä<4N=Å0N)ÆNÆNÆ)M-ÇNÇNÇN)ÇN=Ç%O	Ç-O%Ê#O9Ë#O>Í-M>!Í8	NÎNÎ	NÎN&Î 	N)Î)N:Î4	N=Î=OÏ	O	ÏO"Ï	O%Ï%O6	c               ó   € V ^8„  d   QhRR/# )rJ   rL   z5Generator[CUDAGraphNode | CUDAWarmupNode, None, None]rA   )rN   s   "rH   rO   rœ  Q  s   € ÷ 	#ñ 	#à	>ñ	#rG   c              #  	óŽ   "  € . pT pV'       d    VP                  V4       VP                  pK'  \        V4       R j  x€L
  R #  L5irQ   )rp   r—  Úreversed)r¬   ÚnodesÚnodes   &  rH   rª  ÚCUDAWarmupNode._path_from_rootP  s6   é € ð ˆØ/3ˆßØ�L‰L˜ÔØ—;‘;ŠDä˜E“?×"Ô"ùs   ‚A�-A¼A½Ac               ó   € V ^8„  d   QhRR/# ©rJ   rL   zIterator[StorageWeakRefWrapper]rA   )rN   s   "rH   rO   rœ  \  s   € ÷ !ñ !Ð$Cñ !rG   c              #  óˆ   "  € V P                    F-  pVP                   F  p\        V4      '       g   K  Vx € K  	  K/  	  R# 5i)zEReturns all live storages weakrefs that created by nodes in this pathN)rª  rž  r{  )r¬   rÕ  Úoutputs   &  rH   rÅ  Ú!CUDAWarmupNode.path_live_weakrefs\  s3   é € à×(Ô(ˆDØ×/Ô/�Ü˜6—?”?Ø ”Ló 0ó )ùs
   ‚-A´Ac               ó   € V ^8„  d   QhRR/# rk  rA   )rN   s   "rH   rO   rœ  c  s   € ÷ 3ñ 3 dñ 3rG   c                	ó>   € \        V P                  4       4      '       * # rQ   )re   rÅ  r³   s   &rH   Úall_outputs_are_deadÚ#CUDAWarmupNode.all_outputs_are_deadc  s   € Ü˜×/Ñ/Ó1Ó2Ô2Ð2rG   c               ó    € V ^8„  d   QhRRRR/# ©rJ   r�  útorch.TensorrL   rù   rA   )rN   s   "rH   rO   rœ  f  s   € ÷ ñ °ð Àñ rG   c                	óœ   € V P                  4        F7  pVP                  4       P                  4       VP                  4       8X  g   K6   R # 	  R# )TF)rÅ  rI  ri   )r¬   r�  Ústorage_weak_refs   && rH   Ú_is_cuda_graph_recorded_tensorÚ-CUDAWarmupNode._is_cuda_graph_recorded_tensorf  sB   € Ø $× 7Ñ 7Ö 9ÐØ× Ñ Ó"×+Ñ+Ó-Ð1A×1JÑ1JÓ1LÖLÚñ !:ñ rG   )r›  r™  r¡   rš  r   rz   rž  r—  r2  r‚  rŸ  r6  r–  N)rB   rC   rD   rE   r{   r­   Ú	functoolsÚcached_propertyr«  rÏ  Úpropertyrª  rÅ  rÞ  rå  rF   rA   rG   rH   r”  r”  Ï  sW   † ñõ$ð4 ×ÑôUó ðUõ
Lð\ ô	#ó ð	#õ!õ3÷ñ rG   r”  c                  ó   € ] tR tRt. tRtR# )ÚOutputAliasInfois  rA   N)rB   rC   rD   rE   rt  rF   rA   rG   rH   rë  rë  s  s   † Ø„IrG   rë  c                  ó   € ] tR tRtRtRtR# )Ú_UnaliasedStorageiw  zISingleton to mark that the graph output constructs a new alias or is NonerA   N)rB   rC   rD   rE   r{   rF   rA   rG   rH   rí  rí  w  s   † ÝOrG   rí  c                  ó8   € ] tR tRt$ RtR.tR]R&   R R ltRtR# )	ÚAliasesPriorGraphOutputi~  z>Marks that the graph output aliases an output of a prior graphÚindexÚPathOutputIndexc               ó    € V ^8„  d   QhRRRR/# ©rJ   rð  rñ  rL   rM   rA   )rN   s   "rH   rO   Ú$AliasesPriorGraphOutput.__annotate__…  s   € ÷ ñ ˜oð °$ñ rG   c                	ól   € \        V\        4      '       g   \        R \        V4       24      hWn        R# )z"expected index to be a tuple, got N)rc   r©  r¾   rj   rð  ©r¬   rð  s   &&rH   r­   Ú AliasesPriorGraphOutput.__init__…  s+   € Ü˜%¤×'Ò'Ü Ð#EÄdÈ5ÃkÀ]Ð!SÓTÐTØŽ
rG   ©rð  N©	rB   rC   rD   rE   r{   rt  r|   r­   rF   rA   rG   rH   rï  rï  ~  s   ‡ ÙDà�	€IàÓ÷ñ rG   rï  c                  ó8   € ] tR tRt$ RtR.tR]R&   R R ltRtR# )	ÚAliasesNewOutputi‹  zIMarks that the graph output aliases an index in the new, returned outputsrð  r[   c               ó    € V ^8„  d   QhRRRR/# )rJ   rð  r[   rL   rM   rA   )rN   s   "rH   rO   ÚAliasesNewOutput.__annotate__’  s   € ÷ ñ ˜cð  dñ rG   c                	ól   € \        V\        4      '       g   \        R \        V4       24      hWn        R# )z!expected index to be an int, got N)rc   r[   r¾   rj   rð  rö  s   &&rH   r­   ÚAliasesNewOutput.__init__’  s+   € Ü˜%¤×%Ò%Ü Ð#DÄTÈ%Ã[ÀMÐ!RÓSÐSØŽ
rG   rø  Nrù  rA   rG   rH   rû  rû  ‹  s   ‡ ÙOà�	€IàƒJ÷ñ rG   rû  c                  óZ  € ] tR tRtRtR R lt]P                  R R l4       tR R lt	R	 R
 lt
R R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R  ltR! R" lt]R# R$ l4       t]R% R& l4       t]R' R( l4       tR) R* ltR+ R, lt]R- R. l4       tR/ R0 lt]R1 R2 l4       t ]R3 R4 l4       t!R5 R6 lt"R7 R8 lt#R9 R: lt$R; R< lt%R= R> lt&R? R@ lt'RA RB lt(RC RD lt)]RSRE RF ll4       t*RTRH RI llt+RJ RK lt,RL RM lt-RN RO lt.RP RQ lt/RRt0RG# )UÚCUDAGraphNodei˜  a:  
A single recording of a function into a CUDA Graph. Recordings of CUDA Graphs share a single memory pool
and are structured into a tree, where there is a single recording that can precede it (parent) and multiple
subsequent recordings that may follow (children). A node will have no parent if it is the first recording
in a tree; i.e., when it is first recorded, there are no live tensors from a previous recording which
would force a dependency.

On first recording, all of the live tensors in the current CUDA Graph Node path will be
reflected in the corresponding private pool. On subsequent executions, the caching allocator
is unaffected when the graph is replayed.

In order to support recording a subsequent cuda graph recording after execution of this graph,
we checkpoint the state of the memory pool so that it may later be resumed.

WrappedFunction should have already been warmed up prior to invocation.

See [setCheckpointPoolState] for further explanation, as well as
https://user-images.githubusercontent.com/13564/222815509-374f3400-f83d-4f7d-8fa6-4a092b3250bb.png
c               óD   € V ^8„  d   QhRRRRRRRRR	R
RRRRRRRRRRRR/# )rJ   r–  r)   rz   ry   r—  úCUDAGraphNode | NonerU   r  r™  r8   r¡   r[   r2  r9  r‚  rƒ  r(  r)  r  r'  rL   rM   rA   )rN   s   "rH   rO   ÚCUDAGraphNode.__annotate__­  s…   € ÷ A ñ A à)ðA ð ðA ð %ð	A ð
  ðA ð 'ðA ð ðA ð )ðA ð "ðA ð %ðA ð %ðA ð 
ñA rG   c                	óv  € \        V\        \        34      '       g   \        R \	        V4       24      hWn        VP                  V n        W n        W`n        Wpn	        W€n
        RV n        Ve   \        P                  ! V4      MRV n        WPn        \!        \        4      V n        . V n        V P&                   Uu. uF  q»P$                  NK  	  upV n        V P&                   Uu. uF  q»P                  NK  	  upV n        . V n        \/        V4       UUu. uFB  w  rÍ\        V\0        P2                  4      '       g   K'  V P5                  V4      '       g   K@  VNKD  	  uppV n        V Uu. uF6  p\        V\0        P2                  4      '       d   V P9                  V4      MRNK8  	  upV n        R.\=        V4      ,          V n        \A        R \/        V4       4       4      p\A        VPB                  4      p\A        V P6                  4      p\        VV,          V,          4      V n!        \E        \=        V4      4       Uu. uF  pVV PB                  9  g   K  VNK  	  upV n#        \H        R,          R;;,          \=        V PF                  4      ,          uu&   \K        VV,
          V,
          4      V n&        \        VV,          4      V n'        R R lp\E        \=        V4      4       Uu. uF  pV! VW@PB                  4      NK  	  upV n(        \/        V4       UUu. uFC  w  pp\        V\0        P2                  4      '       d   WÀPB                  9  d   \S        V4      M. NKE  	  uppV n*        . V n+        . V n,        . V n-        . V n.        . V n/        V P`                  eR   V P`                  PX                  pV Pc                  V P(                  4      pV Pe                  VV4      pVV n+        VV n-        V Pg                  V4      pVPi                  4        ?VPj                  '       d   RM\0        Pl                  Po                  4       V n8        V Uu. uFE  p\        V\0        P2                  4      '       d!   V Ps                  V Pu                  V4      4      MTNKG  	  upV n;        RV n<        . V n=        . V n>        . V n?        . V n@        \ƒ        R	W©4      ;_uu_ 4        V P…                  VP†                  V4      V nD        RRR4       . V nE        V Pˆ                  f   \        R
4      hV Pˆ                   F¡  p\        V\0        P2                  4      '       d/   V PŠ                  P�                  V Pu                  VRR7      4       KQ  \        V\Ž        \	        R4      34      '       g   \        \	        V4      4      hV PŠ                  P�                  V4       K£  	  V Pp                  e   V Pp                  P‘                  4        R# R# u upi u upi u uppi u upi u upi u upi u uppi u upi   + '       g   i     EL8; i)z)expected inputs to be list or tuple, got TNFc              3  óP   "  € T F  w  r\        V4      '       g   K  Vx € K  	  R # 5irQ   r*   )Ú.0rZ   r\   s   &  rH   Ú	<genexpr>Ú)CUDAGraphNode.__init__.<locals>.<genexpr>
  s    é € ð '
Ù-‘&�!Ô1DÀS×1I�AŠAÓ-ùs   ‚&œ
&ÚinductorÚ$cudagraph_recorded_non_static_inputsc               ó(   € V ^8„  d   QhRRRRRRRR/# )	rJ   Úidxr[   rU   r  r  ú	list[int]rL   rø   rA   )rN   s   "rH   rO   Ú,CUDAGraphNode.__init__.<locals>.__annotate__$  s0   € ÷ 	ñ 	Øð	à#ð	ð  )ð	ð ñ		rG   c                ó‚   € W,          p\        V\        P                  4      '       d   W9   d   VP                  4       # R # rQ   )rc   rd   r   ri   )r  rU   r  r\   s   &&& rH   Úmaybe_get_static_data_ptrÚ9CUDAGraphNode.__init__.<locals>.maybe_get_static_data_ptr$  s0   € ð
 •+ˆCÜ˜#œuŸ|™|×,Ò,°Ô1IØ—|‘|“~Ð%ÙrG   zCUDAGraphNode.recordz)expected recording_outputs to not be None)Úignore_storage_offset)Irc   re   r©  r¾   rj   r–  r6  rz   rë   r2  r‚  Ú rerecord_if_static_inputs_changerÃ   rJ  Ú_parentr™  r   Úchildrenrž  rª  Úpath_weakrefsÚpath_stacktracesrŸ  ro   rd   r   rå  Úcudagraph_managed_idxsÚ!_is_alias_of_live_recorded_tensorÚ live_cudagraph_managed_path_refsrn   Úpreserved_aliased_inputsr/   r  ÚrangeÚnon_static_input_idxr   Ú	LevelListÚnon_managed_static_input_idxsÚtensor_static_input_idxsÚstatic_input_data_ptrsr   Úexpanded_dimsÚrecorded_liveness_before_graphÚrecorded_liveness_after_graphÚ"expected_dead_indices_before_graphÚ!expected_dead_indices_after_graphÚlive_indices_after_graphr—  Ú_get_livenessÚ_get_different_indicesÚ#_allocate_and_copy_recording_inputsrè   r5  r’   Ú	CUDAGraphr§   Ú!_reconstruct_from_tensor_metadataÚ_tensor_metadataÚreconstructed_inputsÚcheckpointed_caching_stateÚoutput_storage_aliasÚunaliased_in_all_pathsÚcached_tensor_outputsÚstatic_output_tensorsr1  Ú_recordr  Úrecording_outputsÚoutputs_metadatarp   r[   Úreplay)r¬   r–  rz   r—  rU   r™  r¡   r2  r‚  r(  r  rÕ  r  r�  Úopaque_input_idxsr  r  rZ   r  ÚxÚprevious_livenessÚcurr_livenessÚdifferent_indicesÚrecording_inputsr  s   &&&&&&&&&&&              rH   r­   ÚCUDAGraphNode.__init__­  s×  € ô ˜&¤4¬ -×0Ò0Ü Ø;¼DÀ»L¸>ÐJóð ð !1ÔØ(8×(QÑ(QˆÔ%ØŒØ"ŒØ(ÔØŒð 15ˆÔ-ð /5Ò.@”w—{’{ 6Ô*ÀdˆŒà 0Ôô @KÌ4Ó?PˆŒð KMˆÔà.2×.BÒ.BóS
Ù.B d×!Ô!Ñ.BñS
ˆÔð +/×*>Ò*>ó@
Ù*> $×ÔÑ*>ñ@
ˆÔð BDˆÔô
 $ FÔ+ô2
á+‘�Ü˜!œUŸ\™\×*ô à/3×/RÑ/RÐST×/U÷ ˆCÙ+ò2
ˆÔ#ñ óT
ñ �ô ˜a¤§¡×.Ò.ð ×6Ñ6°qÔ9àòñ ñT
ˆÔ-ð ;@¸Ä3ÀvÃ;Õ9NˆÔ%ô 'ñ '
Ü% fÔ-ó'
ó 
Ðô 'Ð'7×'IÑ'IÓJÐÜ!+¨D×,GÑ,GÓ!HÐä,0ØÐ 6Õ6Ð9JÕJó-
ˆÔô
 œS ›[Ô)ó5
Ù)�!¨Q°d×6LÑ6LÑ-L�AˆAÑ)ñ5
ˆÔ!ô 	�ÕÐC×DÌØ×%Ñ%óI
õ 	
ÓDô >GØÐ 6Õ6Ð9JÕJó>
ˆÔ*ô 48ØÐ 6Õ6ó4
ˆÔ%õ	ô œ3˜v›;Ô'ó>
á'�ñ & a¨×1GÑ1GÖHÙ'ñ>
ˆÔ#ô$ $ FÔ+ô	/
ñ ,‘��Qô ˜!œUŸ\™\×*Ò*¨s×:PÑ:PÔ/Pô ˜aÔ àòñ ,ò	/
ˆÔð LNˆÔ+ØJLˆÔ*ð
 JLˆÔ/ØHJˆÔ.ð @BˆÔ%à�;‰;Ò"Ø $§¡× IÑ IÐØ ×.Ñ.¨t×/AÑ/AÓBˆMà $× ;Ñ ;Ø! =ó!Ðð 3@ˆDÔ/Ø6GˆDÔ3à×CÑCÀFÓKÐð 	�‰ŒØð %×:×:Ð:‰DÄÇ
Á
×@TÑ@TÓ@Vð 	Œ
ñ &ó	6
ñ &�ô ˜!œUŸ\™\×*Ò*ð ×2Ñ2°4×3HÑ3HÈÓ3KÔLàòñ &ñ	6
ˆÔ!ð& BFˆÔ'ð IKˆÔ!ð 9;ˆÔ#Ø@BˆÔ"ð
 ACˆÔ"ô $Ð$:¸J×MÕMØ8<¿¹Ø ×&Ñ&Ð(8ó9ˆDÔ"÷ Nð JLˆÔð
 ×!Ñ!Ò)Ü Ð!LÓMÐMØ×)Ô)ˆCÜ˜#œuŸ|™|×,Ò,Ø×%Ñ%×,Ñ,Ø×)Ñ)¨#ÀUÐ)ÓKöô " #¬¬T°$«ZÐ'8×9Ò9Ü(¬¨c«Ó3Ð3Ø×%Ñ%×,Ñ,¨SÖ1ñ *ð �:‰:Ò!Ø�J‰J×ÑÖñ "ùòYS
ùò@
ùó2
ùòT
ùòF5
ùò4>
ùó/
ùò`6
÷X N×MÐMúsP   Â:W=Ã$XÄ$XÄ>XÅXÅ*<XÈ?XÉXË)XÌA	XÑAX"Ó"X'Ø'X8	c               ó   € V ^8„  d   QhRR/# r£  rA   )rN   s   "rH   rO   r  ±  r¥  rG   c                	ó>   € \        \        V P                  4      4      # rQ   r§  r³   s   &rH   r«  Ú.CUDAGraphNode.path_user_visible_storage_groups°  r­  rG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   Údstsr  ÚsrcsrL   rM   rA   )rN   s   "rH   rO   r  ¶  s$   € ÷ ;ñ ;Ø#ð;Ø+:ð;à	ñ;rG   c                	óì  € . p. pV P                    F¿  p\        W%,          \        P                  4      '       g   K+  V P                  V,          p\        W,          V4      p\        P                  ! V4      ^ 8w  d   \        W,          W%,          4       RW%&   K‰  VP                  V4       VP                  \        W%,          V4      4       RW%&   KÁ  	  V'       d   \        P                  ! W44       R# R# )r¿  N)
r  rc   rd   r   r#  r   Ú_debug_has_internal_overlapr   rp   Ú_foreach_copy_)r¬   rD  rE  Údst_tensorsÚsrc_tensorsr  r#  Úindexed_dsts   &&&     rH   Ú _copy_inputs_and_remove_from_srcÚ.CUDAGraphNode._copy_inputs_and_remove_from_src¶  sÂ   € ð ˆØˆØ×,Ô,ˆCÜ˜d�i¬¯©×6Ò6ÙØ ×.Ñ.¨sÕ3ˆMÜ-¨d­i¸ÓGˆKÜ×0Ò0°Ó=ÀÔBä% d¥i°µÔ;Ø �‘	ÙØ×Ñ˜{Ô+Ø×ÑÔ2°4µ9¸mÓLÔMØˆD‹Iñ -÷ Ü× Ò  Ö:ñ rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   r¯  r  rL   rM   rA   )rN   s   "rH   rO   r  Ì  s   € ÷ 3ñ 3¸ð 3ÈTñ 3rG   c                	ój  a€ V P                   '       g    \        P                  P                  VV P                  V P
                  4      '       gc   \        V P                  P                  VV P                  V P
                  \        P                  4      o\        P                  ! R V3R l4       R# R# R# )Fc                 ó   <€ S # rQ   rA   )Ú	error_msgs   €rH   r  Ú>CUDAGraphNode.check_static_inputs_are_stable.<locals>.<lambda>Þ  s   ø€ ©	rG   N)r  rd   r€   Ú#_tensors_data_ptrs_at_indices_equalr"  r   r$   r–  r4  r    ÚStaticInputIdxMismatchÚ_check)r¬   r¯  rQ  s   &&@rH   Úcheck_static_inputs_are_stableÚ,CUDAGraphNode.check_static_inputs_are_stableÌ  s‘   ø€ ð ×5×5Ð5Ü—H‘H×@Ñ@ØØ×+Ñ+Ø×2Ñ2÷ò ô .Ø×%Ñ%×2Ñ2ØØ×+Ñ+Ø×2Ñ2Ü$×;Ñ;óˆIô �LŠL˜Ô 1Ö2ññ 6rG   c               ó    € V ^8„  d   QhRRRR/# ©rJ   r¯  r  rL   r'   rA   )rN   s   "rH   rO   r  à  s   € ÷ ñ ¨?ð ¸zñ rG   c                	óø   € \         P                  P                  '       d   V P                  4        \	        V4      ^ 8w  d   \        R\	        V4       24      hV P                  pRV n        Vf   \        R4      hV# )r¿  rº  Nzexpected outputs to not be None)rv   r˜   Úfast_path_cudagraph_assertsÚ(debug_check_invariants_before_invocationrn   r¾   r6  ©r¬   r¯  Úoutputss   && rH   Úrun_first_inputsÚCUDAGraphNode.run_first_inputsà  ss   € Ü�=‰=×4×4Ð4Ø×9Ñ9Ô;ô ˆz‹?˜aÔÜ Ø7¼¸J»Ð7HÐIóð ð ×(Ñ(ˆØ!%ˆÔØŠ?Ü Ð!BÓCÐCØˆrG   c               ó    € V ^8„  d   QhRRRR/# rY  rA   )rN   s   "rH   rO   r  ð  s   € ÷ ñ ˜oð °*ñ rG   c                	óª  € V P                  V4       V P                  V P                  V4       V P                  4        V P	                  4       pVP                  4        \        P                  P                  '       d   V P                  4        \        P                  P                  '       d   \        P                  P                  4        R V n        V# ©F)rV  rL  r/  Ú	run_graphÚreconstruct_outputsrè   rv   r˜   r[  Ú'debug_check_invariants_after_invocationÚforce_cudagraph_syncrd   r’   r…  Ústatic_inputs_stabler]  s   && rH   rÏ  ÚCUDAGraphNode.runð  s•   € Ø×+Ñ+¨JÔ7à×-Ñ-¨d×.GÑ.GÈÔTà�‰Ôà×*Ñ*Ó,ˆØ×ÑÔä�=‰=×4×4Ð4Ø×8Ñ8Ô:ä�=‰=×-×-Ð-Ü�J‰J×"Ñ"Ô$ð %*ˆÔ!àˆrG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   r'   rA   )rN   s   "rH   rO   r    s   € ÷ Añ A Zñ ArG   c           	     ó  € V P                   '       g   V P                  4        . pV P                  p\        \	        V P
                  V P                  4      4       EF+  w  pw  rE\        V\        4      '       gL   \        V\        \        R4      34      '       g   \        R\        V4       24      hVP                  V4       Kj  V P                   V,          pVe)   VP                  e   RVn        VP                  V4       K©  V P                  V,          pVe8   V P                  V,          e   \        RV R24      hVP                  V4       K÷  V P!                  WE4      p\        V\"        4      '       g   Vf   V P%                  WX4      p	Mk\        V\        4      '       g   \        R\        V4       24      hV P%                  V\'        \(        P*                  W,          4      P-                  4       4      p	VP                  V	4       V P                  V,          p
V
f   \        R4      hV
P/                  V	P-                  4       P1                  4       4       V'       g   EK
  W29   g   EK  \3        V	4      V P4                  V&   EK.  	  V# )zRReconstruct output tensors according to their saved metadata and alias informationNzexpected int or None, got zexpected outputs_weakrefs[z] to be Nonez expected storage to be int, got úexpected w to not be None)r3  Ú_initialize_cached_tensorsr6  ro   Úzipr1  r7  rc   Údictr[   rj   r¾   rp   Ú_backward_hooksr4  rž  Ú*prepare_alias_info_for_tensor_constructionr-   r-  r	   rd   r   rI  r_  Ú	_weak_refr0   rŸ  )r¬   r^  r6  rZ   Ústorage_infoÚmetadataÚcached_tÚstatic_trÌ  r  Úws   &          rH   re  Ú!CUDAGraphNode.reconstruct_outputs  s0  € ð ×)×)Ð)Ø×+Ñ+Ô-à ˆØ#'×#@Ñ#@Ð ä+4Ü�×)Ñ)¨4×+@Ñ+@ÓA÷,
Ñ'ˆAÑ'�ô ˜h¬×-Ò-Ü! (¬S´$°t³*Ð,=×>Ò>Ü(Ð+EÄdÈ8ÃnÐEUÐ)VÓWÐWØ—‘˜xÔ(Ùà×1Ñ1°!Õ4ˆHØÒ#ð ×+Ñ+Ò7Ø/3�HÔ,ð —‘˜xÔ(Ùà×1Ñ1°!Õ4ˆHØÒ#Ø×(Ñ(¨Õ+Ò7Ü(Ð+EÀaÀSÈÐ)UÓVÐVØ—‘˜xÔ(Ùà×EÑEØóˆGô ˜'¤>×2Ò2°g²oØ×<Ñ<¸XÓO‘ä! '¬3×/Ò/Ü(Ø:¼4À»=¸/ÐJóð ð ×<Ñ<Øœd¤5§<¡<°Õ1AÓB×RÑRÓTó�ð �N‰N˜3ÔØ×%Ñ% aÕ(ˆAØŠyÜ$Ð%@ÓAÐAØ�N‰N˜3×.Ñ.Ó0×:Ñ:Ó<Ô=ß'Ò'¨A×,IÜ*7¸Ó*<�×$Ñ$ QÔ'ñg,
ðj ˆrG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   Úout_alias_infozOutputAliasInfo | Nonert  zdict[str, Any] | int | NonerL   zUntypedStorage | None | intrA   )rN   s   "rH   rO   r  H  s(   € ÷ $ñ $à.ð$ð .ð$ð 
%ñ	$rG   c                	ó°  € \        V\        \        R 4      34      '       g   V\        J d   R # \        V\        4      '       d\   VP
                  w  r4V P                  V,          V,          pVf   \        R4      h\        P                  P                  V! 4       4      # \        V\        4      '       g   \        R\        V4       24      hVP
                  # )Nzexpected ref to not be Nonezexpected AliasesNewOutput, got )rc   r[   rj   ÚUnaliasedStoragerï  rð  r  r¾   rd   r-   Ú_new_with_weak_ptrrû  )r¬   rz  rt  ÚdepthÚexisting_output_indexrJ  s   &&&   rH   rq  Ú8CUDAGraphNode.prepare_alias_info_for_tensor_constructionH  s»   € ô �x¤#¤t¨D£zÐ!2×3Ò3ØÔ!1Ó1áä�nÔ&=×>Ò>Ø+9×+?Ñ+?Ñ(ˆEØ×$Ñ$ UÕ+Ð,AÕBˆCØŠ{Ü$Ð%BÓCÐCÜ×'Ñ'×:Ñ:¹3»5ÓAÐAä˜.Ô*:×;Ò;Ü Ø1´$°~Ó2FÐ1GÐHóð ð ×#Ñ#Ð#rG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   z!list[UntypedStorage | None | int]rA   )rN   s   "rH   rO   r  `  s   € ÷ ñ à	*ñrG   c                	óœ   € . p\        V P                  V P                  4       F%  w  r#VP                  V P	                  W#4      4       K'  	  V# rQ   )rn  r1  r7  rp   rq  )r¬   Úoutput_storagesr1  rt  s   &   rH   Ú!prepare_storages_for_constructionÚ/CUDAGraphNode.prepare_storages_for_construction`  sV   € ð ˆÜ.1Ø×%Ñ% t×'<Ñ'<ö/
Ñ*Ð ð ×"Ñ"Ø×?Ñ?Ø(óöñ/
ð ÐrG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r  o  s   € ÷ ñ ˜4ñ rG   c                	ó¨   € V P                   P                  '       d   R # V P                  f   \        R4      hV P                  P	                  4        R # )Núexpected graph to not be None)r–  r5  r§   r¾   r8  r³   s   &rH   rd  ÚCUDAGraphNode.run_grapho  s=   € Ø× Ñ ×6×6Ð6ÙØ�:‰:ÒÜ Ð!@ÓAÐAØ�
‰
×ÑÖrG   c               ó   € V ^8„  d   QhRR/# rk  rA   )rN   s   "rH   rO   r  v  s   € ÷ ñ  dñ rG   c                óˆ   € V P                    F1  w  r\        V P                  V,          V,          4      '       g   K0   R# 	  R# )z;All outputs of the path from this node to its root are deadFT)r(  r{  r  )r¬   r~  Úoutput_indexs   &  rH   rÞ  Ú"CUDAGraphNode.all_outputs_are_deadv  s9   € à#'×#@Ô#@ÑˆEÜ�t×)Ñ)¨%Õ0°Õ>×?Ô?Úñ $Añ rG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   r  r&   rU   r  rL   r'   rA   )rN   s   "rH   rO   r  }  s&   € ÷ [ñ [˜Yð [°ð [ÀJñ [rG   c                ó¬  a a€ R VV 3R llp\         P                  ! V! 4       S P                  P                  4       Uu/ uF+  pVP	                  4       P                  4       \        V4      bK-  	  ppS P                  P                  '       Ed‘   \        4       ;_uu_ 4        \        P                  P                  S P                  4      ;_uu_ 4        \        4       ;_uu_ 4        \        S P                  S P                  S P                  4      ;_uu_ 4        \!        4       ;_uu_ 4        \#        4       ;_uu_ 4        \%        4       ;_uu_ 4        V! S4      pRRR4       RRR4       RRR4       RRR4       RRR4       RRR4       RRR4       \'        S4      ^ 8w  d   \)        R\'        S4       24      h\+        X\,        \.        34      '       g   V.pM!\+        V\.        4      '       d   \-        V4      p\1        \2        V4      pS P5                  We4       V# S P6                  f   \)        R4      h\8        P:                  P<                  '       dè   S P>                  f   . M"\-        S P>                  PA                  4       4      pT\C        S4       UU	u. uFt  w  r‰\+        V	\        PD                  4      '       g   K'  VS P                  PF                  9  g   KD  V	P	                  4       P                  4       ^ 8w  g   Ki  \        V	4      NKv  	  up	p,          p\I        S P                  S P                  V4       \        4       ;_uu_ 4        \        P                  P                  S P                  4      ;_uu_ 4        \        4       ;_uu_ 4        \        P                  P7                  S P6                  S P                  S P                  RR7      ;_uu_ 4        \!        4       ;_uu_ 4        \K        4       ;_uu_ 4        \%        4       ;_uu_ 4        V! S4      pRRR4       RRR4       RRR4       RRR4       RRR4       RRR4       RRR4       \'        S4      ^ 8w  d   \)        R\'        S4       24      h\+        X\,        \.        34      '       g   V3pS P5                  We4       V# u upi   + '       g   i     ELe; i  + '       g   i     ELq; i  + '       g   i     EL}; i  + '       g   i     EL‰; i  + '       g   i     EL•; i  + '       g   i     EL¡; i  + '       g   i     EL­; iu up	pi   + '       g   i     EL1; i  + '       g   i     EL=; i  + '       g   i     ELI; i  + '       g   i     ELU; i  + '       g   i     ELa; i  + '       g   i     ELm; i  + '       g   i     ELy; i)zRecord the modelc               ó   € V ^8„  d   QhRR/# )rJ   rL   z#Generator[torch.Tensor, None, None]rA   )rN   s   "rH   rO   Ú+CUDAGraphNode._record.<locals>.__annotate__€  s   € ÷ 	ñ 	Ð#Fñ 	rG   c               3  óÚ   <"  € SP                   P                   FK  p SV ,          p\        V\        P                  4      '       g   K.  SP                  V4      '       d   KG  Vx € KM  	  R # 5irQ   )r–  r  rc   rd   r   rå  )rZ   Ú_inprU   r¬   s     €€rH   Ústatic_input_iterÚ0CUDAGraphNode._record.<locals>.static_input_iter€  sQ   øé € Ø×*Ñ*×<Ô<�Ø˜a•y�ÜØœ%Ÿ,™,÷ô à×=Ñ=¸d×CÔCØ”Jó =ùs   ƒA A+ÁA+Á!
A+Nz!expected inputs to be empty, got rˆ  Úthread_local)r‚  ÚpoolÚcapture_error_mode)&r´  rµ  r–  r3  rI  ri   r;   r5  r   rd   r’   rë   r‡   r‹  r™  r‚  rŽ  r   rœ   rn   r¾   rc   re   r©  r	   r'   Ú_add_first_outputsr§   rv   r˜   rÆ  r—  rÅ  ro   r   r  rÇ  r   )
r¬   r  rU   r”  r\   Ú$static_input_persistent_storage_ptrsÚstatic_outputsr“   rZ   Úelems
   f&f       rH   r5  ÚCUDAGraphNode._record}  s6  ù€ ÷	ñ 	ô !—’Ù!Ó# T×%:Ñ%:×%DÑ%DôóR
ñ�ð ×ÑÓ!×*Ñ*Ó,Ô.CÀCÓ.HÒHñð 	-ð R
ð × Ñ ×6×6Ñ6ä"×$Õ$Ü—
‘
×!Ñ! $§+¡+×.Õ.Ü$×&Õ&Ü-Ø—K‘K ×!6Ñ!6¸¿¹÷õ ô 7×8Õ8Ü/×1Õ1Ü%×'Õ'á!& v£�÷ (÷ 2÷ 9÷	÷ '÷ /÷ %ô �6‹{˜aÔÜ$Ð'HÌÈVËÈÐ%VÓWÐWä˜n¬t´U¨m×<Ò<Ø"0Ð!1‘Ü˜N¬E×2Ò2Ü!% nÓ!5�Ü!¤*¨nÓ=ˆNØ×#Ñ#Øôð "Ð!à�:‰:ÒÜ Ð!@ÓAÐAä�=‰=×4×4Ð4ð —k‘kÒ)‘¬t°D·K±K×4RÑ4RÓ4TÓ/Uð ð ä(¨Ô0ôá0‘G�AÜ˜d¤E§L¡L×1ô ,ð ˜T×2Ñ2×DÑDÑDô ,ð ×(Ñ(Ó*×3Ñ3Ó5¸Ñ:ô	 ,Ô% dÖ+Ù0òõ ˆFô ˜dŸk™k¨4×+@Ñ+@À&ÔIô × Õ Ü�J‰J×Ñ˜dŸk™k×*Õ*Ü ×"Õ"Ü�J‰J×ÑØ—
‘
Ø—{‘{Ø×*Ñ*Ø#1ð	 ÷ õ ô 3×4Õ4Ü5×7Õ7Ü!×#Õ#á" 6›]ˆN÷ $÷ 8÷ 5÷÷ #÷ +÷ !ô" ˆv‹;˜!ÔÜ Ð#DÄSÈÃ[ÀMÐ!RÓSÐSä˜.¬4´¨-×8Ò8Ø,Ð.ˆNð 	×Ñ ÔUð Ðùò_R
÷$ (×'Ð'ú÷ 2×1Ð1ú÷ 9×8Ð8ú÷	÷ ð ú÷ '×&Ð&ú÷ /×.Ð.ú÷ %×$Ð$üóB÷, $×#Ð#ú÷ 8×7Ð7ú÷ 5×4Ð4ú÷÷ ð ú÷ #×"Ð"ú÷ +×*Ð*ú÷ !× Ð úsV  ¿1R3Â!2T0ÃT	Ã&4TÄS4Ä-S Å SÅ	R8ÅSÅ$S Å,S4Å4TÅ<T	ÆT0Ê$UËUË  UÌUÍ2WÎV.	ÎA
VÏ VÏ3U2ÐUÐ	U
Ð"UÐ*U2Ð2VÐ:VÑV.	Ñ
WÒ8S	!Ó	SÓSÓ	S Ó S1Ó+	S4Ó4TÓ?	TÔTÔ	T	ÔT-Ô'	T0Ô0U	Õ
U!Õ	UÕU/Õ)	U2Õ2VÕ=	VÖVÖ	VÖV+Ö%	V.	Ö.V?Ö9	W×W	c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   r^  r'   rš  z dict[int, StorageWeakRefWrapper]rL   rM   rA   )rN   s   "rH   rO   r  Ú  s.   € ÷ fñ fàðfð /Oðfð 
ñ	frG   c                ó"  a a€ S P                   pS P                  S P                  4      pS P                  W44      pVS n        \        S P                  4      ^ 8w  d"   \        R\        S P                  4       24      h/ p\        \        V4      4       Uu. uF  pRNK  	  upS n	        \        \        V4      4       Uu. uF  pRNK  	  upS n
        \        V4       EF|  w  opVe!   \        V\        P                  4      '       g"   S P                  P!                  \"        4       KL  \        P$                  ! VP&                  ;'       g"    VP)                  4       P+                  4       ^ 8H  VV 3R l4       VP-                  VP)                  4       P+                  4       4      p	VP)                  4       P+                  4       ^ 8H  p
V	'       d
   V	! 4       f	   V
'       d.   S P                  P!                  R4       VS P                  S&   EK/  S P/                  V4      pVen   S P1                  V4       \        S P2                  4       F  w  rÍW½8X  g   K  RS P4                  V&   K  	  S P                  P!                  \7        V4      4       EK±  VP)                  4       P+                  4       V9   d[   WhP)                  4       P+                  4       ,          pRS P                  V&   S P                  P!                  \9        V4      4       EK.  SWhP)                  4       P+                  4       &   S P                  P!                  \"        4       RS P                  S&   EK  	  S P:                  f+   \        \        V4      4       Uu. uF  pRNK  	  upS n        M.\        S P:                  4      \        V4      8w  d   \        R4      hS P                  '       d   \        R4      h\=        VS P                  4       Fª  w  pp\        V\        P                  4      '       d   Ve9   S P                  P!                  R4       S P>                  P!                  R4       Kb  S P                  P!                  \A        V4      4       S P>                  P!                  \C        V4      4       K¬  	  S P                  S P                  4      S n"        \        PF                  PI                  S PJ                  S PL                  4      S n'        \        \        S P                  4      4       Fw  p\        \        S P                  V,          4      4       FK  p\Q        S P                  V,          V,          4      '       g   K.  S PR                  P!                  VV34       KM  	  Ky  	  S PU                  4        \V        PX                  PZ                  '       d;   \]        S PJ                  S PL                  \_        S Pa                  4       4      4       R# R# u upi u upi u upi )zIAdd the outputs from the first invocation of the node and set up metadataz+expected outputs_weakrefs to be empty, got FNc                 ó\   <€ R SP                   '       d   SP                   S ,           2# R 2# )zHExpected all cuda outputs in cuda graph recording. Non cuda output from z	(unknown))r2  )rZ   r¬   s   €€rH   r  Ú2CUDAGraphNode._add_first_outputs.<locals>.<lambda>ú  s6   ø€ ðØ48×4E×4EÐ4E˜D×-Ñ-¨aÕ0ÐWñYØKVÐWñYrG   Tz&Wrong number of stack traces passed inz%expected outputs_weakrefs to be empty)1r$  r)  r  r*  r'  rn   rž  r¾   r  r2  r4  ro   rc   rd   r   r1  rp   r|  rU  rÀ  rI  ri   ræ   r  Ú#_mark_prior_graph_output_as_aliasedr  r  rï  rû  r2  rn  rŸ  r;   r0   r%  r€   Ú_cuda_getCheckpointStaterë   r™  r0  r{  r(  rf  rv   r˜   rÆ  rÇ  re   rÅ  )r¬   r^  rš  Úprev_livenessr<  ÚdeltaÚoutput_new_storages_indexr  r¼  rJ  Úis_empty_storageÚpath_refr  Úinp_path_refrð  r  Ústatic_output_tensorr~  rŒ  rZ   s   f&&                @rH   r™  Ú CUDAGraphNode._add_first_outputsÚ  sÔ  ù€ ð ×;Ñ;ˆØ×*Ñ*¨4×+=Ñ+=Ó>ˆà×+Ñ+¨MÓIˆØ16ˆÔ.äˆt×$Ñ$Ó%¨Ô*Ü Ø=¼cÀ$×BWÑBWÓ>XÐ=YÐZóð ð @BÐ!ä6;¼CÀ»LÔ6IÓ&JÑ6I°£uÑ6IÑ&JˆÔ#Ü49¼#¸g»,Ô4GÓ%HÑ4G¨q£dÑ4GÑ%HˆÔ"ä˜g×&‰DˆAˆqØŠy¤
¨1¬e¯l©l× ;Ò ;Ø×)Ñ)×0Ñ0Ô1AÔBÙä�LŠLØ—	‘	×@Ð@˜Q×.Ñ.Ó0×9Ñ9Ó;¸qÑ@õôð 7×:Ñ:Ø×!Ñ!Ó#×,Ñ,Ó.óˆCð
  !×0Ñ0Ó2×;Ñ;Ó=ÀÑBÐß™›Ò)×.>Ø×)Ñ)×0Ñ0°Ô6Ø01�×*Ñ*¨1Ñ-Úà×=Ñ=¸aÓ@ˆHØÒ#Ø×8Ñ8¸ÔBä)2Ø×9Ñ9ö*Ñ%�Cð  Ö/Ø=A˜×5Ñ5°cÓ:ñ	*ð
 ×)Ñ)×0Ñ0Ô1HÈÓ1RÔSÚà× Ñ Ó"×+Ñ+Ó-Ð1JÔJØ1×2CÑ2CÓ2E×2NÑ2NÓ2PÕQ�Ø5:�×+Ñ+¨EÑ2Ø×)Ñ)×0Ñ0Ô1AÀ%Ó1HÔIÚàHIÐ%×&7Ñ&7Ó&9×&BÑ&BÓ&DÑEØ×%Ñ%×,Ñ,Ô-=Ô>Ø-1ˆD×'Ñ'¨Ô*ñY 'ð\ ×ÑÒ$Ü/4´S¸³\Ô/BÓ CÑ/B¨!£Ñ/BÑ CˆDÕä�4×$Ñ$Ó%¬¨W«Ô5Ü$Ð%MÓNÐNà× × Ð Ü Ð!HÓIÐIÜ),¨W°d×6PÑ6PÖ)QÑ%ˆCÐ%Ü˜c¤5§<¡<×0Ò0Ð4HÒ4TØ×%Ñ%×,Ñ,¨TÔ2Ø×$Ñ$×+Ñ+¨DÖ1à×%Ñ%×,Ñ,Ô-BÀ3Ó-GÔHØ×$Ñ$×+Ñ+¬M¸#Ó,>Ö?ñ *Rð .2×-?Ñ-?À×@RÑ@RÓ-SˆÔ*Ü*/¯(©(×*KÑ*KØ�K‰K˜×.Ñ.ó+
ˆÔ'ô
 œ3˜t×1Ñ1Ó2Ö3ˆEÜ %¤c¨$×*<Ñ*<¸UÕ*CÓ&DÖ E�Ü˜4×-Ñ-¨eÕ4°\ÕB×CÔCØ×1Ñ1×8Ñ8¸%ÀÐ9NÖOó !Fñ 4ð
 	×4Ñ4Ô6Ü�=‰=×4×4Ð4ÜØ—‘˜T×2Ñ2´D¸×9PÑ9PÓ9RÓ4Söñ 5ùò[ 'KùÚ%Hùòb !Ds   ÂXÂ>XÎXc               ó    € V ^8„  d   QhRRRR/# ró  rA   )rN   s   "rH   rO   r  B  s   € ÷ #ñ #¸ð #ÈTñ #rG   c                óÖ   € Vw  r#\        V P                  4      V,          pRVP                  V&   V P                  V,          V,          pVf   \	        R4      hVP                  4        R# )zLRemove a graph output from the unaliased, cached tensors in an ancestor nodeFNzexpected x to not be None)re   rª  r2  r  r¾   rh  )r¬   rð  r~  rŒ  rÕ  r:  s   &&    rH   r¢  Ú1CUDAGraphNode._mark_prior_graph_output_as_aliasedB  sa   € à#ÑˆÜ�D×(Ñ(Ó)¨%Õ0ˆØ49ˆ×#Ñ# LÑ1Ø×Ñ˜uÕ% lÕ3ˆØŠ9Ü Ð!<Ó=Ð=Ø	× Ñ Ö"rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r  L  s   € ÷ =3ñ =3¨Dñ =3rG   c                	ó*  a	€ \        V P                  4      \        V P                  4      8w  d   \        R 4      h\	        \        V P                  V P                  V P                  4      4       EF   w  pw  r#pV'       g   V P                  P                  R4       K/  V\        Jd   \        RV 24      h\        V\        4      '       g   \        R\        V4       24      hV P                  V4      pV P                  W5R7      p\         P"                  P%                  V4       \&        P(                  ! V 4      o	R V	3R llp\*        P,                  ! WqR7      p\/        WhR7      V P                  V&   V P                  P                  V4       EK#  	  R# )	z7expected len(outputs_weakrefs) == len(outputs_metadata)Nz2expected storage_info to be UnaliasedStorage, got z"expected metadata to be dict, got )rÌ  c               ó    € V ^8„  d   QhRRRR/# )rJ   rZ   r[   rL   rù   rA   )rN   s   "rH   rO   Ú>CUDAGraphNode._initialize_cached_tensors.<locals>.__annotate__v  s   € ÷ )ñ ) #ð )¨$ñ )rG   c                óô   <€ S! 4       pVf   R# VP                  V 4      pVP                  V ,          P                  4       ^8”  d   V^8  d   \        RV 24      hV^8H  # V^8  d   \        RV 24      hV^8H  # )NFzexpected refcount >= 3, got zexpected refcount >= 2, got )Úget_output_refcountr3  Ú
_use_countr¾   )rZ   Úself_locÚrefcountÚself_refs   &  €rH   Úcheck_refcountÚ@CUDAGraphNode._initialize_cached_tensors.<locals>.check_refcountv  s‡   ø€ Ù#›:�ØÒ#Ù Ø#×7Ñ7¸Ó:�à×1Ñ1°!Õ4×?Ñ?ÓAÀAÔEà !”|Ü,Ð/KÈHÈ:Ð-VÓWÐWØ# q™=Ð(à !”|Ü,Ð/KÈHÈ:Ð-VÓWÐWØ# q™=Ð(rG   )rZ   rg  )rn   rž  r7  r¾   ro   rn  r1  r2  r3  rp   r|  rc   ro  rj   Úcreate_storager-  rd   r€   Ú_add_cached_tensorrÃ   rJ  rç  Úpartialr;   )
r¬   rZ   rs  rt  Úmake_cachedrÍ  r  r¹  Úcheckr¸  s
   &        @rH   rm  Ú(CUDAGraphNode._initialize_cached_tensorsL  sf  ø€ ô ˆt×$Ñ$Ó%¬¨T×-BÑ-BÓ)CÔCÜ ØIóð ô 9BÜØ×)Ñ)Ø×%Ñ%Ø×+Ñ+ó÷9
Ñ4ˆAÑ4�¨÷ Ø×*Ñ*×1Ñ1°$Ô7ÙàÔ#3Ó3Ü$ØHÈÈÐWóð ô ˜h¬×-Ò-Ü$Ø8¼¸h»Ð8HÐIóð ð ×#Ñ# HÓ-ˆAØ×8Ñ8¸Ð8ÓMˆCô �H‰H×'Ñ'¨Ô,ä—{’{ 4Ó(ˆH÷)ð )ô  ×%Ò% nÔ:ˆEä'<¸SÔ'XˆD×!Ñ! !Ñ$Ø×&Ñ&×-Ñ-¨c×2óm9
rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   rð  r[   rL   rA   )rN   s   "rH   rO   r  ‹  s   € ÷ Bñ B¨ð B°ñ BrG   c                	óP   € \         P                  ! V P                  V,          4      # rQ   )ÚsysÚgetrefcountr3  rö  s   &&rH   r´  Ú!CUDAGraphNode.get_output_refcount‹  s   € Ü�Š˜t×9Ñ9¸%Õ@ÓAÐArG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   r  rA   )rN   s   "rH   rO   r  �  s   € ÷ Dñ DÐ,ñ DrG   c                óB   € V P                   e   V P                  4       # R# )zunwraps the weakref to _parentN)r  r³   s   &rH   r—  ÚCUDAGraphNode.parentŽ  s   € ð "&§¡Ò!9ˆt�|‰|‹~ÐC¸tÐCrG   c               ó   € V ^8„  d   QhRR/# ©rJ   rL   z$Generator[CUDAGraphNode, None, None]rA   )rN   s   "rH   rO   r  ”  s   € ÷ ñ ÐCñ rG   c              #  óF   "  € T pV'       d   Vx € VP                   pK  R# 5i)zAReturns all nodes in the path starting at self and ending at rootN)r—  ©r¬   rÕ  s   & rH   Ú_path_to_rootÚCUDAGraphNode._path_to_root“  s"   é € ð ˆçØŠJØ—;‘;ŠDñ ùs   ‚
!�!c               ó   € V ^8„  d   QhRR/# rÊ  rA   )rN   s   "rH   rO   r  �  s   € ÷ ñ Ð!Eñ rG   c              #  ób   "  € \        \        V P                  4      4      pT Rj  x€L
  R#  L5i)zEReturns all nodes in the path starting at the root and ending at selfN)rÓ  re   rÍ  ©r¬   rÔ  s   & rH   rª  ÚCUDAGraphNode._path_from_rootœ  s%   é € ô œ˜d×0Ñ0Ó1Ó2ˆØ×Ôùs   ‚$/¦-§/c               ó    € V ^8„  d   QhRRRR/# rá  rA   )rN   s   "rH   rO   r  ¢  s   € ÷ ñ °ð Àñ rG   c                ó¸   € V P                    FI  pV F@  pVf   K	  VP                  4       pVP                  4       P                  4       V8X  g   K>    R# 	  KK  	  R# )z/Is this tensor an output of a node in this pathTF)r  ri   rI  )r¬   r�  Úoutput_refsrä  ri   s   &&   rH   rå  Ú,CUDAGraphNode._is_cuda_graph_recorded_tensor¢  sX   € à×-Ô-ˆKÛ$/Ð Ø#Ò+Ùð ,×4Ñ4Ó6�Ø×$Ñ$Ó&×/Ñ/Ó1°XÖ=Ûó %0ñ .ñ rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   r�  râ  rL   zPathOutputIndex | NonerA   )rN   s   "rH   rO   r  °  s   € ÷ 
ñ 
Øð
à	ñ
rG   c                	óæ   € \        V P                  4       FW  w  r#\        V4       FC  w  rE\        V4      ;pf   K  Vw  rxW�P                  4       P	                  4       8X  g   K>  W$3u u # 	  KY  	  R # rQ   )ro   r  rz  rI  ri   )	r¬   r�  r~  rÕ  rŒ  rC  Ústorage_and_ptrÚ_storageÚptrs	   &&       rH   r  Ú/CUDAGraphNode._is_alias_of_live_recorded_tensor°  sm   € ô #,¨D×,>Ñ,>Ö"?ÑˆEÜ-6°{Ö-CÑ)�Ü'2°;Ó'?Ð?�OÔLØ$3‘M�HØ×/Ñ/Ó1×:Ñ:Ó<Ö<Ø %Ð4Ô4ó	 .Dñ #@ñ rG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   Úindicesúlist[PathOutputIndex]rÕ  ú(list[list[StorageWeakRefWrapper | None]]rL   rù   rA   )rN   s   "rH   rO   r  ½  s&   € ÷ ñ Ø&ðà=ðð 
ñrG   c                óp   € V  F/  w  r#W,          V,          pVf   \        R4      hV! 4       f   K.   R# 	  R# )z;Check that all of the indices specified are dead referencesrl  FT)r¾   )rÞ  rÕ  r~  rŒ  rw  s   &&   rH   Ú_check_livenessÚCUDAGraphNode._check_liveness¼  s>   € ó $+ÑˆEØÕ" <Õ0ˆAØŠyÜ$Ð%@ÓAÐAÙ‹sŒÚñ $+ñ rG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   Úfunction_idr"   rÕ  r  rL   rM   rA   )rN   s   "rH   rO   r  Ê  s!   € ÷ 0ñ 0 Zð 0°}ð 0Èñ 0rG   c                óJ   € V P                   V,          P                  V4       R# )zAdds node as a child of selfN)r  rp   )r¬   rå  rÕ  s   &&&rH   Ú	add_childÚCUDAGraphNode.add_childÊ  s   € à�‰�kÕ"×)Ñ)¨$Ö/rG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   r�   úlist[list[bool]]ÚcurrrL   rß  rA   )rN   s   "rH   rO   r  Ï  s$   € ÷ ñ ØðØ&6ðà	ñrG   c           	     ó®  € . p\        V 4      \        V4      8”  d$   \        R\        V 4       R\        V4       24      h\        \        W4      4       F~  w  pw  rE\        V4      \        V4      8w  d$   \        R\        V4       R\        V4       24      h\        \        WE4      4       F"  w  pw  rxWx8w  g   K  VP	                  W634       K$  	  K€  	  V# )z(Find indices where the two lists differ.z%expected len(prev) <= len(curr), got z > z-expected len(outputs1) == len(outputs2), got z != )rn   r¾   ro   rn  rp   )	r�   rë  Údead_indicesrZ   Úoutputs1Úoutputs2ÚjÚoutput1Úoutput2s	   &&       rH   r*  Ú$CUDAGraphNode._get_different_indicesÎ  sÓ   € ð
 ˆÜˆt‹9”s˜4“yÔ Ü Ø7¼¸D»	°{À#ÄcÈ$ÃiÀ[ÐQóð ô (1´°T³Ö'AÑ#ˆAÑ#�Ü�8‹}¤ H£Ô-Ü$ØCÄCÈÃMÀ?ÐRVÔWZÐ[cÓWdÐVeÐfóð ô *3´3°xÓ3JÖ)KÑ%�Ñ%�GØÖ%Ø ×'Ñ'¨¨Ö/ó *Lñ (Bð ÐrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   Úweakrefsrà  rL   rê  rA   )rN   s   "rH   rO   r  ä  s    € ÷ Kñ KØ:ðKà	ñKrG   c                ó†   € \        V 4      ^ 8X  d   . # V  Uu. uF  p\        P                  ! \        V4      NK   	  up# u upi )zCMaps weakrefs to true if the reference is alive and false otherwise)rn   ÚpytreeÚtree_mapr{  )rõ  r^  s   & rH   r)  ÚCUDAGraphNode._get_livenessã  s8   € ô
 ˆx‹=˜AÔØˆIáAIÓJÁ°g”—’¤¨Ö1ÁÑJÐJùÒJs   —$>c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   Úexpected_livenessrê  Ú
newly_deadrß  rL   rM   rA   )rN   s   "rH   rO   r  í  s$   € ÷ 3ñ 3Ø!1ð3Ø?Tð3à	ñ3rG   c                	óÀ  € \         P                  P                  '       g   R # \        V P                  4       F1  w  r4V P
                  V,          VP                  Jg   K(  \        R4      h	  \        V P                  4      p\        V P                  4      p\        \        ,          ! 4       p\        \        ,          ! 4       p\        V4       FÌ  w  rš\        V
4       F¸  w  r¼V P
                  V	,          V,          p\        V4      ;pf   K0  V'       g   \        R4      hVw  ppVV9   Wø9   8X  g   \        R4      hVP                  V4       VP                  V4       WY,          P                  V,          R JpV'       g   K¦  VV9   g   K¯  \        R4      h	  KÎ  	  V F:  w  p	p\!        V P
                  V	,          V,          4      '       g   K1  \        R4      h	  R # )Nz5expected path_weakrefs[i] to be node.outputs_weakrefsz%expected output_liveness to be truthyz0expected data_ptr and weak_ptr liveness to agreez)expected stor_data_ptr not in live_blocksz:expected path_weakrefs[depth][output_index] to not be live)rv   r˜   r[  ro   rª  r  rž  r¾   re   Úget_block_addrsr™  r/   r   rz  rÈ  r4  r{  )r¬   rû  rü  rZ   rÕ  rÔ  Úlive_blocksÚlive_storage_data_ptrsÚlive_storage_weak_ptrsr~  Úoutputs_livenessÚ
output_idxÚoutput_livenessrw  Ústor_weak_ptr_and_data_ptrÚstor_weak_ptrÚstor_data_ptrÚis_persistent_aliasrŒ  s   &&&                rH   Údebug_assert_invariantsÚ%CUDAGraphNode.debug_assert_invariantsí  s¶  € ô �}‰}×8×8Ð8Ùä  ×!5Ñ!5Ö6‰GˆAØ×!Ñ! !Õ$¨D×,AÑ,AÕAÜ$ØKóð ñ 7ô �T×)Ñ)Ó*ˆä% d×&;Ñ&;Ó<ˆä!+¬C¦Ó!2ÐÜ!+¬C¦Ó!2Ðä'0Ð1BÖ'CÑ#ˆEÜ/8Ð9IÖ/JÑ+�
à×&Ñ& uÕ-¨jÕ9�Ü2=¸a³.Ð@Ð.ÔMß*Ü,Ð-TÓUÐUØ3MÑ0�M =à&Ð*@Ñ@Ø)ÑCôEô -ØNóð ð +×.Ñ.¨}Ô=Ø*×.Ñ.¨}Ô=ð �×:Ñ:¸:ÕFÈdÐRð (÷ +Ñ*Ø(¨KÖ7Ü"0Ø Kó#ð ó/ 0Kñ (Dó8 $.ÑˆE�<Ü�t×)Ñ)¨%Õ0°Õ>×?Ô?Ü$ØPóð ó $.rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r  "  s   € ÷ 
ñ 
¸$ñ 
rG   c                	óR   € V P                  V P                  V P                  4       R # rQ   )r	  r$  r&  r³   s   &rH   r\  Ú6CUDAGraphNode.debug_check_invariants_before_invocation"  s!   € Ø×$Ñ$Ø×/Ñ/°×1XÑ1Xö	
rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r  '  s   € ÷ 
ñ 
¸ñ 
rG   c                	óR   € V P                  V P                  V P                  4       R # rQ   )r	  r$  r'  r³   s   &rH   rf  Ú5CUDAGraphNode.debug_check_invariants_after_invocation'  s!   € Ø×$Ñ$Ø×/Ñ/°×1WÑ1Wö	
rG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   r  rA   )rN   s   "rH   rO   r  ,  s   € ÷ "ñ "°ñ "rG   c                ó  € V P                  V P                  4      pV P                  V P                  V4      p\	        V P
                  4      p. pV F4  w  rVVP                  W5,          P                  V,          R,          4       K6  	  V# )zx
Since this node was invoked, return data ptrs of all tensor outputs that have died
in the current executing tree path.
ri   )r)  r  r*  r%  re   rª  rp   r7  )r¬   r<  r*  ÚpathÚptrs_to_deallocater~  rŒ  s   &      rH   Údata_ptrs_dead_since_invocationÚ-CUDAGraphNode.data_ptrs_dead_since_invocation,  s†   € ð
 ×*Ñ*¨4×+=Ñ+=Ó>ˆØ!%×!<Ñ!<Ø×.Ñ.°ó"
Ðô �D×(Ñ(Ó)ˆØÐÛ#9ÑˆEØ×%Ñ%Ø•×,Ñ,¨\Õ:¸:ÕFöñ $:ð
 "Ð!rG   c               ó   € V ^8„  d   QhRR/# rØ  rA   )rN   s   "rH   rO   r  ?  s   € ÷ ñ Ð$Cñ rG   c              #  	ó¦   "  € V P                    F<  w  rV P                  V,          V,          pVf   K%  \        V4      '       g   K8  Vx € K>  	  R # 5irQ   )r(  r  r{  )r¬   rZ   rð  r  s   &   rH   rÅ  Ú CUDAGraphNode.path_live_weakrefs?  s>   é € Ø×1Ô1‰DˆAØ×$Ñ$ QÕ'¨Õ*ˆCØŒ¤7¨3§<¤<Ø”	ó 2ùs   ‚-A´AÁ
Ac               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r  E  s   € ÷ +ñ +¨Dñ +rG   c                	ó^  € V P                    F(  pVf   K	  \        P                  P                  V4       K*  	  V P                   P	                  4        \        V P                  4       FA  w  r#V'       g   K  V P                  V,          pVf   \        R4      hVP                  4        KC  	  R # )Nzexpected n to not be None)
r3  rd   r€   Ú_remove_cached_tensorrè   ro   r2  rž  r¾   rh  )r¬   r�  rZ   Ú	unaliasedrr   s   &    rH   Úremove_node_cached_tensorsÚ(CUDAGraphNode.remove_node_cached_tensorsE  s‡   € Ø×+Ô+ˆAØŒ}Ü—‘×.Ñ.¨qÖ1ñ ,ð 	×"Ñ"×(Ñ(Ô*ä% d×&AÑ&AÖB‰LˆAß‰yØ×)Ñ)¨!Õ,�Ø’9Ü(Ð)DÓEÐEØ×(Ñ(Ö*ó CrG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r  R  s   € ÷ .ñ .¨Dñ .rG   c                	óL   € V P                    F  pVP                  4        K  	  R # rQ   )rª  r  rÌ  s   & rH   Úremove_path_cached_tensorsÚ(CUDAGraphNode.remove_path_cached_tensorsR  s   € Ø×(Ô(ˆDØ×+Ñ+Ö-ó )rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r  V  s   € ÷ >ñ > $ñ >rG   c                ó   € R# )z3Clear the path state in this current executing nodeNrA   r³   s   &rH   Úclear_path_stateÚCUDAGraphNode.clear_path_stateV  s   ‚ rG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   r:  râ  r  rù   rL   údict[str, Any]rA   )rN   s   "rH   rO   r  [  s$   € ÷ 
ñ 
Øð
Ø04ð
à	ñ
rG   c                	óŠ  € \        V \        P                  4      '       g   \        R \	        V 4       24      hRV P                  4       P                  4       RV P                  4       P                  4       RV P                  RV P                  4       RV P                  RV P                  RV'       g   V P                  4       /# ^ /# )z%expected x to be a torch.Tensor, got Únbytesri   rf   rg   rh   rë   Ústorage_offset)rc   rd   r   r¾   rj   rI  r+  ri   Úshaperg   rh   rë   r,  )r:  r  s   &&rH   r.  ÚCUDAGraphNode._tensor_metadataZ  s®   € ô ˜!œUŸ\™\×*Ò*Ü Ð#HÌÈaËÈ	Ð!RÓSÐSð �a×'Ñ'Ó)×0Ñ0Ó2Ø˜×)Ñ)Ó+×4Ñ4Ó6Ø�A—G‘GØ�a—h‘h“jØ�Q—W‘WØ�a—h‘hØ×8M˜a×.Ñ.Ó0ð
ð 	
ð TUð
ð 	
rG   Nc               ó$   € V ^8„  d   QhRRRRRR/# )rJ   rt  r)  rÌ  zUntypedStorage | NonerL   r   rA   )rN   s   "rH   rO   r  l  s)   € ÷ Vñ VØ&ðVØ1FðVà	ñVrG   c                	ón   € Vf   V P                  V4      MTp\        P                  P                  W4      # rQ   )r»  rd   r€   Ú0_construct_CUDA_Tensor_From_Storage_And_Metadata)r¬   rt  rÌ  rÍ  s   &&& rH   r-  Ú/CUDAGraphNode._reconstruct_from_tensor_metadatal  s/   € ð .5ª_ˆD×Ñ Ô)À'ˆÜ�x‰x×HÑHÈÓUÐUrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   rt  r)  rL   ztorch.types.StoragerA   )rN   s   "rH   rO   r  r  s   € ÷ 
ñ 
 ~ð 
Ð:Mñ 
rG   c                	ón   € \         P                  P                  VR ,          VR,          VR,          4      # )ri   rë   r+  )rd   r€   Ú$_construct_storage_from_data_pointer)r¬   rt  s   &&rH   r»  ÚCUDAGraphNode.create_storager  s/   € Ü�x‰x×<Ñ<Ø�ZÕ  (¨8Õ"4°h¸xÕ6Hó
ð 	
rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   rU   r  rL   rA   )rN   s   "rH   rO   r  w  s   € ÷ % ñ % Ø%ð% à	ñ% rG   c                ó  € \         P                  P                  4        V P                  P	                  \         P                  P                  4       4       . p\        P                  ! RR7      ;_uu_ 4        \         P                  P                  V P                  4      ;_uu_ 4        \        V P                  V P                  V P                  R7      ;_uu_ 4        \        V4       F·  w  r4\        V\         P                  4      '       gV   \        V\        \         P                  \         34      '       g   \#        R\%        V4       24      hVP'                  V4       Kz  W0P(                  9  d   VP'                  \+        V4      4       K¦  VP'                  V4       K¹  	  V P-                  W!4       RRR4       RRR4       RRR4       V#   + '       g   i     L"; i  + '       g   i     L-; i  + '       g   i     T# ; i)zs
Allocate inputs for non static, non cudagraph managed tensors in the memory pool
and copy over the tensor values.
T©Úrecord)r€  r‚  z1expected int, Generator, or CustomClassBase, got N)rd   r’   r…  r‚  r†  r‡  ÚwarningsÚcatch_warningsrë   r‹  r™  ro   rc   r   r[   r2   r   r¾   rj   rp   r  r   rL  )r¬   rU   r>  rZ   r\   s   &&   rH   r+  Ú1CUDAGraphNode._allocate_and_copy_recording_inputsw  sm  € ô 	�
‰
×ÑÔ Ø�‰×Ñ¤§
¡
× 9Ñ 9Ó ;Ô<Ø,.Ðô ×#Ò#¨4×0Ö0Ü�J‰J×Ñ˜dŸk™k×*Õ*Ü)Ø—‘Ø×.Ñ.Ø—{‘{÷ö ô $ FÖ+‘�Ü! #¤u§|¡|×4Ò4Ü% c¬C´·±Ä/Ð+R×SÒSÜ,ØOÔPTÐUXÓPYÈ{Ð[óð ð %×+Ñ+¨CÖ0Ø×4Ñ4Ô4à$×+Ñ+¬L¸Ó,=Ö>à$×+Ñ+¨CÖ0ñ ,ð ×1Ñ1Ð2BÔK÷)÷ +÷ 1ð0  Ð÷-÷ ú÷ +×*ú÷ 1Ö0ð0  Ðús=   Á72G6Â)5G#	ÃCGÆ6G#	Æ>G6ÇG ÇG#	Ç#G3Ç.G6Ç6H	c               ó    € V ^8„  d   QhRRRR/# )rJ   rU   r  rL   z/tuple[CheckInvariantStatus, Callable[..., str]]rA   )rN   s   "rH   rO   r  ž  s$   € ÷ GWñ GWØ%ðGWà	8ñGWrG   c                óì  a€ \         P                  ! \        V P                  P                  VV P
                  4      p\        P                  P                  VV P
                  V P                  4      '       g7   \        P                  o\         P                  ! VV P                  S4      pSV3# V P                  V P                  V P                  4      '       g   \        P                  oSV3R l3# V P                   '       dr   \        P                  P                  VV P
                  V P"                  4      '       g7   \        P$                  o\         P                  ! VV P"                  S4      pSV3# V P                   F"  pV P&                  V,          '       d   K  RW&   K$  	  \        P(                  ! V P                  V P*                  V P                  4      R 4       \        P,                  R 3# )z›
Checks if this node can be run. The same pattern of tensor liveness, static inputs,
and tensors managed in the cudagraph private pool must remain stable.
c                 ó
   <€ S  # rQ   rA   )Ústatuss   €rH   r  Ú0CUDAGraphNode.check_invariants.<locals>.<lambda>Á  s	   ø€  f X ;rG   Nc                 ó   € R # )z‰TODO: graph recording observed an input tensor deallocate during graph  recording that did not occur during replay. Please file an issue.rA   rA   rG   rH   r  rB  â  s   € ñ QrG   c                 ó$   € \         P                   # rQ   )r    ÚSUCCESSrA   rG   rH   r  rB  å  s   € Ô8L×8TÑ8TÐ7UÐ5VrG   )rç  r½  r$   r–  r4  r"  rd   r€   rS  r  r    ÚCudagraphManagedIdxMismatchrâ  r&  r  Ú&ExpectedDeadIndicesBeforeGraphMismatchr  r!  rT  r  rU  r'  rE  )r¬   rU   Ú_loggerr  rA  s   &&  @rH   Úcheck_invariantsÚCUDAGraphNode.check_invariantsž  s¹  ø€ ô ×#Ò#Ü!Ø×!Ñ!×.Ñ.ØØ×'Ñ'ó	
ˆô �x‰x×;Ñ;ØØ×'Ñ'Ø×'Ñ'÷
ò 
ô
 *×EÑEˆFÜ×'Ò'ØØ×+Ñ+ØóˆGð
 ˜7�?Ð"à×#Ñ#Ø×3Ñ3°T×5GÑ5G÷
ò 
ô *×PÑPˆFØÔ.Ð.Ð.ð ×1×1Ð1Ü—H‘H×@Ñ@ØØ×+Ñ+Ø×-Ñ-÷ò ô *×@Ñ@ˆFÜ×'Ò'ØØ×-Ñ-ØóˆGð
 ˜7�?Ð"ð
 ×.Ô.ˆCØ×0Ñ0°×5Ö5Ø"�“ñ /ô 	�ŠØ× Ñ Ø×6Ñ6¸×8JÑ8JóñQô		
ô $×+Ñ+Ñ-VÐVÐVrG   c               ó   € V ^8„  d   QhRR/# rb  rA   )rN   s   "rH   rO   r  ç  s   € ÷ ñ  ñ rG   c                óœ   € ^ pV P                   P                  4        F+  pV F"  pV^,          pWP                  4       ,          pK$  	  K-  	  V# )z(Total number of descendents of this node)r  ÚvaluesÚnum_descendants)r¬   Únum_descr  Úchilds   &   rH   rN  ÚCUDAGraphNode.num_descendantsç  sI   € àˆØŸ™×,Ñ,Ö.ˆHÛ!�Ø˜A•�Ø×1Ñ1Ó3Õ3’ó "ñ /ð ˆrG   )&r  r3  r0  r  r™  r  rë   r#  r'  r&  r§   rz   r  r(  r   r  r1  r7  rž  r  r  r  r/  r%  r$  r6  r  r2  r"  r  rh  r4  r‚  r!  rŸ  r2  r6  r–  ©TrQ   )1rB   rC   rD   rE   r{   r­   rç  rè  r«  rL  rV  r_  rÏ  re  rq  r„  rd  rÞ  r5  r™  r¢  rm  r´  ré  r—  rÍ  rª  rå  r  Ústaticmethodrâ  rç  r*  r)  r	  r\  rf  r  rÅ  r  r"  r&  r.  r-  r»  r+  rI  rN  rF   rA   rG   rH   r  r  ˜  sc  † ñõ(A ðF ×ÑôUó ðUõ
;õ,3õ(õ õ*AõF$õ0õõõ[õzfõP#õ=3õ~Bð ôDó ðDð ôó ðð ôó ðõ
õ
ð ôó ðõ0ð ôó ðð( ôKó ðKõ3õj
õ

õ
"õ&õ+õ.õ>ð ö
ó ð
÷"Võ
õ
% õNGW÷Rñ rG   r  c               ó(   € V ^8„  d   QhRRRRRRRR/# )rJ   rÕ  zCUDAGraphNode | CUDAWarmupNoder~  r[   r  rL   rñ  rA   )rN   s   "rH   rO   rO   ñ  s,   € ÷ 	ñ 	Ø
(ð	Ø14ð	Ø;>ð	àñ	rG   c                óä   € \        V \        4      '       dY   V P                  V,          p\        V\        4      '       d   VP                  # \        V\
        4      '       d   WP                  3# W3# rQ   )rc   r  r1  rï  rð  rû  )rÕ  r~  r  Ú
alias_infos   &&& rH   Úcanonical_path_output_indexrW  ñ  s`   € ô �$œ×&Ò&Ø×.Ñ.¨sÕ3ˆ
Ü�jÔ"9×:Ò:Ø×#Ñ#Ð#Ü�jÔ"2×3Ò3Ø×+Ñ+Ð,Ð,Øˆ<ÐrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   r  z*tuple[CUDAGraphNode | CUDAWarmupNode, ...]rL   r¤  rA   )rN   s   "rH   rO   rO   ý  s$   € ÷ @Mñ @MØ
4ð@Mà,ñ@MrG   c           	     óJ  € \        4       p\        4       p\        4       p\        V 4       F¢  w  rEVP                   F�  pV\        VP                  4      8¼  d   \        R4      hVP                  WF34       VP                  \        WTV4      4       VP                  V,          pVf   Kn  VP                  VP                  4       4       K�  	  K¤  	  \        V 4       Fl  w  rE\        VP                  4       FN  w  rgWF3p\        WTV4      p	W�9   g$   W’9   g   Vf   K&  VP                  4       V9   g   K=  VP                  V4       KP  	  Kn  	  / p
V FŠ  w  rFW,          pVP                  pW¶,          pVf   K'  \        V\        4      '       d   VP                  MRpVP                  4       pV
P                  V. 4      P                  VVP                  VV34       KŒ  	  \        ;QJ d"    . R V
P!                  4        4       F  NK  	  5# ! R V
P!                  4        4       4      # )aU  Return path storage-alias groups for user-visible output cloning.

Starts from user-visible outputs, then includes outputs on the same path
that may alias those outputs by recorded storage pointer or alias metadata.
This is cached per node path so aliases discovered on a different tree path
do not affect the current generation transition.
z;expected user_visible_output_idxs to index outputs_weakrefsNc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irQ   )r©  )r  Úentriess   & rH   r  Ú;collect_path_user_visible_storage_groups.<locals>.<genexpr>=  s   é € ÐLÑ/K G”�w—�Ó/Kùs   ‚)r/   ro   r6  rn   rž  r¾   rÈ  rW  ri   rc   r  r3  Ú
setdefaultrp   rŸ  r©  rM  )r  Úselected_outputsÚselected_alias_outputsÚselected_data_ptrsr~  rÕ  r  rC  rŒ  Úalias_indexÚentries_by_data_ptrÚoutput_weakrefsr3  ri   s   &             rH   r¨  r¨  ý  sø  € ô 5?³LÐÜ:D»,ÐÜ*4«,Ðô ! –‰ˆØ×0Ô0ˆCØ”c˜$×/Ñ/Ó0Ô0Ü$ØQóð ð × Ñ  % Ô.Ø"×&Ñ&Ô'BÀ4ÐPSÓ'TÔUà×/Ñ/°Õ4ˆKØÔ&Ø"×&Ñ& {×';Ñ';Ó'=Ö>ó 1ñ 'ô ! –‰ˆÜ )¨$×*?Ñ*?Ö @ÑˆCØ!˜<ˆLä5°dÀ3ÓGˆKàÔ0ØÔ8ØÔ*Ø×(Ñ(Ó*Ð.@Ö@à ×$Ñ$ \Ö2ó !Añ 'ð HJÐÛ&‰
ˆØ�{ˆØ×/Ñ/ˆØ%Õ*ˆØÒÙô +5°T¼=×*IÒ*IˆD×&Ò&Ètð 	ð ×'Ñ'Ó)ˆØ×&Ñ& x°Ó4×;Ñ;àØ×$Ñ$Ø%Øð	ö	
ñ '÷( Œ5ÑLÐ/B×/IÑ/IÔ/KÓL�5ÐLˆ5ÑLÐ/B×/IÑ/IÔ/KÓLÓLÐLrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   Úpool_idr�  rL   r   rA   )rN   s   "rH   rO   rO   @  s   € ÷ Vñ V Oð V¸ñ VrG   c                óŠ   € \         P                  P                  4       pV Uu. uF  q"R ,          V 8X  g   K  VNK  	  up# u upi )Úsegment_pool_id)rd   r’   Úmemory_snapshot)re  ÚsegmentsÚsegments   &  rH   Úget_cudagraph_segmentsrk  @  s9   € Ü�z‰z×)Ñ)Ó+€HÙ#+ÓU¡8˜Ð7HÕ/IÈWÑ/T�GˆG¡8ÑUÐUùÒUs
   £A ·A c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   re  r�  Ú	live_onlyrù   rL   r  rA   )rN   s   "rH   rO   rO   E  s!   € ÷ ñ ˜_ð ¸ð Èñ rG   c                óÌ   € . p\        V 4       FR  pVR ,          pVR,           F9  pVR,          R8X  g	   V'       g   VP                  V4       WER,          ,          pK;  	  KT  	  V# )ÚaddressÚblocksÚstateÚactive_allocatedrf   )rk  rp   )re  rm  rp  rj  ÚaddrÚblocks   &&    rH   rþ  rþ  E  s`   € Ø€Fä)¨'Ö2ˆØ�yÕ!ˆØ˜X×&Ð&ˆEØ�W�~Ð!3Ô3¿9Ø—‘˜dÔ#à˜&•MÕ!ŠDó	 'ñ 3ð €MrG   c               ó    € V ^8„  d   QhRRRR/# )rJ   ÚframesrV   rL   rW   rA   )rN   s   "rH   rO   rO   S  s   € ÷ ?ñ ?�ið ? Cñ ?rG   c           	     óÎ   € V  Uu. uF0  p\         P                  ! VR ,          VR,          VR,          4      NK2  	  ppRP                  \         P                  ! V4      4      # u upi )ÚfilenameÚliner&  r  )Ú	tracebackÚFrameSummaryrq   Úformat_list)rv  ÚentryÚformatted_tracebacks   &  rH   Ú	format_tbr  S  sb   € ñ óáˆEô 	×Ò˜u ZÕ0°%¸µ-ÀÀvÅÖOÙð ð ð
 �7‰7”9×(Ò(Ð)<Ó=Ó>Ð>ùòs   …6A"c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rJ   rë   r[   re  r�  Úlive_storages_ptrszlist[StorageWeakRefWrapper]rL   rM   rA   )rN   s   "rH   rO   rO   \  s6   € ÷ d ñ d Øðd àðd ð 4ðd ð 
ñ	d rG   c                ó(  aaa€ \         ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   \        R4      hV Uu0 uF!  q3! 4       '       g   K  VP                  4       kK#  	  upo\        P                  P                  V SS4      '       d   R# \        P                  ! 4        \        P                  P                  4        \        S4      p/ pV FV  pVR,          pVR,           F=  pVR,          R	8X  d   VS9  d   W…V&   MSP                  V4       WxR
,          ,          pK?  	  KX  	  \        P                  ! \        S4      ^ 8H  VV3R l4       \        V4      ^ 8w  Ed™   . p	VP                  4        F7  w  r¨\        VP!                  R. 4      4      pV	P#                  RV
 RV 24       K9  	  RP%                  V	4      pRp\&        P(                  P*                  '       g   Rp\&        P(                  P,                  '       g   RMRpRp\&        P(                  P,                  '       d‘   ^ RIp. pVP1                  R4      p\3        V4       FM  w  po\5        V3R lV 4       R4      pVf   K"  RV R2pVP7                  VV^R7       VP#                  V4       KO  	  V'       d   RRP%                  V4       R2p\9        VP;                  4       4      pR\        V4       RV V RV RV RV 2p\=        V4      hR# u upi )zTValidate cudagraph pool allocations against tracked live storages and surface leaks.c              3  óB   "  € T F  p\        V\        4      x € K  	  R # 5irQ   )rc   r;   )r  rœ  s   & rH   r  Ú$check_memory_pool.<locals>.<genexpr>b  s   é € ÐVÑCU¸4Œz˜$Ô 5×6Ð6ÓCUùs   ‚FTz;expected all live_storages_ptrs to be StorageWeakRefWrapperNro  rp  rq  rr  rf   c                 ó   <€ R S  RS R2# )z2These storage data ptrs are not allocated in pool z but should be: zl. This could be a bug in inductor aliasing tracking or in a custom op's meta function. Please file an issue.rA   )re  Úunique_storagess   €€rH   r  Ú#check_memory_pool.<locals>.<lambda>ƒ  s    ø€ Ø@ÀÀ	ÐIYÐZiÐYjð kyñ zrG   rv  zData Pointer: z, history: 
Ú
r  zd- Set torch._inductor.config.triton.cudagraph_trees_history_recording = True for allocation origins
z- Objgraph backrefs disabled; set torch._inductor.config.triton.cudagraph_trees_objgraph = True (refs_live_tensor_{index}.svg)
râ  c              3  óL   <"  € T F  pS\        V4      9   g   K  Vx € K  	  R # 5irQ   )r!   )r  r�  Úbad_dps   & €rH   r  r„  ¨  s    øé € ÐO¡˜1¨6Ô5KÈAÓ5NÑ+N—Q’Q£ùs   ƒ$š
$Úrefs_live_tensor_z.svg)rx  Ú	max_depthz- Objgraph backrefs written: rm   z	Detected zz tensor(s) in the cudagraph pool not tracked as outputs. All live allocations must be tracked for correctness.
Debugging:
z9- Search gc.get_objects() for tensors with data_ptr() in z3
- Use refcycle to find what is preventing cleanup
zAllocations:
)Úallr¾   ri   rd   r€   Ú_cuda_checkPoolLiveAllocationsÚgcÚcollectr’   r…  rk  ÚremoverU  rn   rå   r  ræ   rp   rq   rv   r˜   r™   Úcudagraph_trees_objgraphÚobjgraphÚby_typero   ÚnextÚshow_backrefsr/   ÚkeysÚRuntimeError)rë   re  r�  rL  ri  Úallocated_not_in_live_storagesrj  rs  rt  Ú	formattedÚdpÚtraceÚformatted_sÚhistory_hintÚobjgraph_hintÚobjgraph_files_hintr“  Úgenerated_filesÚtensorsrð  Ú
bad_tensorrx  Údangling_addrsÚmsgrŠ  r†  s   &f&                     @@rH   rÇ  rÇ  \  sø  ú€ ÷ ‹3ÑVÑCUÓV�3�3Š3ÑVÑCUÓV×VÒVÜØIó
ð 	
ñ 4FÓPÑ3E¨4ÈÏ”�t—}‘}–Ñ3EÑP€Oô ‡x�x×.Ñ.¨v°wÀ×PÒPÙô ‡J‚J„LÜ	‡J�J×ÑÔä% gÓ.€Hà%'Ð"ãˆØ�yÕ!ˆØ˜X×&Ð&ˆEØ�W�~Ð!3Ô3Ø˜Ô.Ø;@°4Ò8à#×*Ñ*¨4Ô0à˜&•MÕ!ŠDó 'ñ ô 
‡L‚LÜˆOÓ Ñ!õ	
ôô Ð)Ó*¨aÕ/Øˆ	Ø7×=Ñ=Ö?‰IˆBÜ˜eŸi™i¨°"Ó5Ó6ˆEà×Ñ˜~¨b¨T°¸u¸gÐFÖGñ @ð —i‘i 	Ó*ˆàˆÜ�}‰}×>×>Ð>ð CˆLô —=‘=×9×9Ð9ñ3ð ð 	ð !Ðä�=‰=×1×1Ð1ãà)+ˆOà×&Ñ& ~Ó6ˆGÜ!*Ð+IÖ!J‘��vÜ!ÜO¡ÓOÐQUó�
ð Ò%ÙØ.¨u¨g°TÐ:�Ø×&Ñ& z¸HÐPQÐ&ÔRØ×&Ñ& xÖ0ñ "K÷ à3°D·I±I¸oÓ4NÐ3OÈrÐRð $ô $Ð$B×$GÑ$GÓ$IÓJˆàœÐ:Ó;Ð<ð =ð ˆnØˆoØGÈÐGWð XBà"Ð#Ø˜[˜Mð+ð 	ô ˜3ÓÐño 0ùòG Qs   ÁLÁ Lc                  óR   € ] tR tRtRt]! 4       t]! 4       t]! 4       t]! 4       t	Rt
R# )ÚExecutionStateiÃ  zÄ
Represents the state of the CUDAGraph Tree. Will be None if there is no live current memory allocated
in the cuda graph pool. Otherwise will reflect the state of the most recently executed node.
rA   N)rB   rC   rD   rE   r{   r   ÚNONEÚWARMUPÚ	RECORDINGÚ	EXECUTIONrF   rA   rG   rH   r§  r§  Ã  s%   † ññ
 ‹6€DÙ‹V€FÙ“€IÙ“„IrG   r§  c                  ó@   € ] tR tRt]! 4       t]! 4       t]! 4       tRtR# )r/  iÏ  rA   N)	rB   rC   rD   rE   r   r@  r0  r?  rF   rA   rG   rH   r/  r/  Ï  s   † Ù‹f€GÙ‹v€HÙ“„IrG   r/  c                  óp  € ] tR tRtRtR R ltR R ltR R ltR	 R
 ltR R lt	R R lt
R R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R  ltR! R" ltR# R$ ltR% R& lt]R' R( l4       t]R) R* l4       tR+ R, lt]R- R. l4       t]P8                  R/ R0 l4       tR1 R2 lt]R3 R4 l4       t]R5 R6 l4       t RXR8 R9 llt!RXR: R; llt"R< R= lt#R> R? lt$R@ RA lt%RXRBRC/RD RE lllt&RF RG lt'RH RI lt(]RJRC/RK RL ll4       t)RM RN lt*RORCRPRC/RQ RR llt+RS RT lt,RU RV lt-RWt.R7# )YrÈ   iÕ  a¡  
Groups individual recordings or executions of cuda graphs into a tree of recordings,
and checks required invariants, and manages warmups of graphs.

When graphs are recorded in the same tree, it enforces subsequent execution
to follow the same order and have the same output tensor livespans. To remove
unnecessary coupling of cuda graphs (and additional imposed invariants),
the tree manager will end a currently recording tree whenever it is valid - when
the memory pool no longer has any live allocations.

We ignore outputs from a previous generation that correspond to prior model outputs.
Currently this is hardcoded `GenerationTracker.generation` tracked in torch dynamo.
# TODO: make generation increment configurable, warn on overwrite.

We run graph warmups in the cudagraph memory pool and return the result on the first invocation
of a function. For many models it is important to reclaim activations as you run the backward.
If we were to warm up the model and keep an extra copy of the inputs around to subsequently
use for recording, we would incur a memory penalty. Additionally, if we are part way through training
your model and need to recompile, memory will be allocated to the cuda graph pool, so we run this
warmup run in the cuda graph memory pool. As for recording, warm up needs the state of live tensors
to be accurately reflected so we checkpoint the allocator state if we need to warm up following graph
replay.
c               ó    € V ^8„  d   QhRRRR/# r    rA   )rN   s   "rH   rO   Ú!CUDAGraphTreeManager.__annotate__î  s   € ÷ q
ñ q
 Sð q
¨Tñ q
rG   c                	óÖ  € \        \        4      V n        / V n        / V n        \        4       V n        \        4       V n        \        P                  P                  R 4       \        4       V n        \        ;_uu_ 4        \        P                  P                  V4      ;_uu_ 4        \        P                  P                  4        \        P                  P!                  4       V n        V P"                  P%                  \        P                  P'                  4       4       \        P                  P)                  4       V n        \        P                  P-                  4       V n        \0        P2                  ! R R7      ;_uu_ 4        \        P                  P+                  V P*                  V P.                  V P"                  RR7      ;_uu_ 4         RRR4       RRR4       RRR4       RRR4       \4        P6                  ! ^ 4      V n        \4        P6                  ! ^ 4      V n        \        \<        4      V n        \4        P6                  ! R	R	R7      V n         \        R 4      V n!        \D        PF                  V n$        RV n%        Wn&        RV n'        R	V n(        ^ V n)        ^ V n*        / V n+        / V n,        RV n-        RV n.        RV n/        \        P`                  Pc                  4       '       g
   RV n4        R# \        Pd                  Pg                  R4      V n4        R#   + '       g   i     ELU; i  + '       g   i     ELa; i  + '       g   i     ELm; i  + '       g   i     ELy; i)
Tr9  r–  )r—  r‚  r˜  N)ÚstartÚstepc                 ó   € \        R  4      # )c                 ó   € ^ # r¾  rA   rA   rG   rH   r  ÚACUDAGraphTreeManager.__init__.<locals>.<lambda>.<locals>.<lambda>*	  s   € ©rG   r   rA   rG   rH   r  Ú/CUDAGraphTreeManager.__init__.<locals>.<lambda>*	  s	   € ”K¡	Ô*rG   Fz5pytorch/inductor:disable_cudagraph_alias_invalidationéÿÿÿÿ)5r   re   ÚrootsÚids_to_funcsÚids_to_stack_tracesr/   Úwarmed_up_functionsÚwarned_functionsrd   r€   r>   Úwarned_mutationÚgraph_capture_lockr’   rë   r…  ÚStreamr‚  r†  r‡  r,  r§   Úgraph_pool_handleÚcuda_graphs_thread_poolr;  r<  r´  ÚcountÚgraph_counterÚfunc_counterro  Ú#non_cudagraph_managed_mutation_hintÚwarmup_node_counterÚnum_rerecordr§  r¨  Ú
path_stateÚactive_warmup_functionr¡   Úcurrent_nodeÚcurrent_genÚdebug_fail_counterÚdebug_checkpointing_counterÚ
id_to_modeÚid_to_compile_idÚ$has_live_user_visible_output_cloningÚ'running_forwards_with_pending_backwardsr(  Ú_environmentÚ	is_fbcodeÚ_utils_internalÚjustknobs_checkÚdisable_invalidate_aliasesr«   s   &&rH   r­   ÚCUDAGraphTreeManager.__init__î  sŒ  € ô
 =HÌÓ<MˆŒ
ð @BˆÔàIKˆÔ ä;E»<ˆÔ ô 9C»ˆÔÜ�‰×,Ñ,¨TÔ2ô 8B³|ˆÔ÷  Ö¤§¡×!2Ñ!2°<×!@Õ!@Ü�J‰J×"Ñ"Ô$ÜŸ*™*×+Ñ+Ó-ˆDŒKØ�K‰K×#Ñ#¤E§J¡J×$=Ñ$=Ó$?Ô@ô 7<·j±j×6JÑ6JÓ6LˆDŒJÜ+0¯:©:×+GÑ+GÓ+IˆDÔ(ô ×'Ò'¨t×4Ö4Ü—
‘
× Ñ Ø—J‘JØ×5Ñ5ØŸ;™;Ø'5ð	 !÷ õ ð ÷÷ 5÷ "A×ô( 'Ÿ_š_¨QÓ/ˆÔÜ%ŸOšO¨AÓ.ˆÔô œÓð 	Ô0ô $-§?¢?¸À"Ô#EˆÔ ô JUÙ*óJ
ˆÔô )×-Ñ-ˆŒØ9=ˆÔ#Ø(Ôð DHˆÔð
 !#ˆÔð #$ˆÔà+,ˆÔ(à=?ˆŒØDFˆÔØ49ˆÔ1ð  8=ˆÔ4Ø,0ˆŒ	ô ×%Ñ%×/Ñ/×1Ò1ð ð 	Ö'ô ×&Ñ&×6Ñ6ØGóð 	Ö'÷K÷ ð ú÷ 5×4Ð4ú÷ "A×!@Ð!@ú××ÐúsV   Á=(MÂ%CM	ÆA
L/ÇLÇL/ÇM	ÇMÌL,Ì&	L/Ì/M Ì:	M	ÍMÍ	MÍM(	c               ó$   € V ^8„  d   QhRRRRRR/# ©rJ   r¯  r  rå  r"   rL   r'   rA   )rN   s   "rH   rO   r¯  a	  s!   € ÷ ñ ˜oð ¸Jð È:ñ rG   c                	óV  € V P                   f   \        R4      hV P                  V,          V n        V P                  V,          V n        V P                  W4      pV P                  \        P                  8X  d
   RV n	        V# V P                  \        P                  8X  d   RV n	        V# )Nz Running CUDAGraph after shutdownTF)r§   r¾   rÎ  r(  rÏ  r  Ú_runr/  r@  rÑ  r0  )r¬   r¯  rå  r  s   &&& rH   rÏ  ÚCUDAGraphTreeManager.runa	  sŒ   € Ø�:‰:ÒÜ Ð!CÓDÐDØ—O‘O KÕ0ˆŒ	Ø×/Ñ/°Õ<ˆŒØ�i‰i˜
Ó0ˆð �9‰9œ×/Ñ/Ô/Ø;?ˆDÔ8ð ˆ
ð �Y‰Yœ/×2Ñ2Ô2Ø;@ˆDÔ8àˆ
rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¯  p	  s   € ÷ -ñ -¨ñ -rG   c                	ó>   € R V n         \        P                  V n        R# rŠ   )rÑ  r/  r0  r(  r³   s   &rH   Úset_to_running_backwardÚ,CUDAGraphTreeManager.set_to_running_backwardp	  s   € Ø7<ˆÔ4Ü#×,Ñ,ˆŽ	rG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   zCallable[[Tensor], bool]rA   )rN   s   "rH   rO   r¯  t	  s   € ÷ 
ñ 
Ð9Qñ 
rG   c                	ó€   € \        V P                  \        \        34      '       d   V P                  P                  # R  # )c                ó   € R # rc  rA   r  s   &rH   r  ÚNCUDAGraphTreeManager._get_cuda_graph_recorded_tensor_checker.<locals>.<lambda>x	  s   € ™5rG   )rc   rÊ  r  r”  rå  r³   s   &rH   Ú'_get_cuda_graph_recorded_tensor_checkerÚ<CUDAGraphTreeManager._get_cuda_graph_recorded_tensor_checkert	  s<   € ô ˜$×+Ñ+¬m¼^Ð-L×MÒMð ×Ñ×<Ñ<ð	
ñ !ð	
rG   c               ó   € V ^8„  d   QhRR/# ©rJ   rL   ry   rA   )rN   s   "rH   rO   r¯  {	  s   € ÷ 7ñ 7 Gñ 7rG   c                	ó>   € \        \        V P                  4      4      # rQ   )ry   r•  rÆ  r³   s   &rH   Únew_warmup_node_idÚ'CUDAGraphTreeManager.new_warmup_node_id{	  s   € Ü”t˜D×4Ñ4Ó5Ó6Ð6rG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   rå  r"   rU   r  rL   rM   rA   )rN   s   "rH   rO   r¯  ~	  s)   € ÷ Sñ SØ%ðSØ/>ðSà	ñSrG   c                	óZ  € V P                  4       p\        V P                  V,          VV P                  4       4      ;p'       dQ   R V P                  V,          V&   WP
                  9   d   R# V P
                  P                  V4       \        V4       R# RV P                  V,          V&   R# )TNF)Ú_get_node_idr   r¹  rå  rÅ  r½  rÈ  r#   )r¬   rå  rU   Únode_idÚmaybe_mutation_strs   &&&  rH   Ú&_update_non_cudagraph_managed_mutationÚ;CUDAGraphTreeManager._update_non_cudagraph_managed_mutation~	  sž   € ð ×#Ñ#Ó%ˆÜ!3Ø×Ñ˜kÕ*ØØ×8Ñ8Ó:ó"
ð 
Ðö 
ð
 NRˆD×4Ñ4°WÕ=¸kÑJà×2Ñ2Ô2ÙØ× Ñ ×$Ñ$ [Ô1Ü/Ð0BÖCàMRˆD×4Ñ4°WÕ=¸kÓJrG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   úGraphID | NonerA   )rN   s   "rH   rO   r¯  �	  s   € ÷ Oñ O˜nñ OrG   c                	óÜ   € V P                   f   R # \        V P                   \        \        34      '       d   V P                   P                  # \        R\        V P                   4       24      h)NzUnknown node type )rÊ  rc   r  r”  rz   r˜  rj   r³   s   &rH   rî  Ú!CUDAGraphTreeManager._get_node_id�	  sZ   € Ø×ÑÒ$ÙÜ˜×)Ñ)¬M¼>Ð+J×KÒKØ×$Ñ$×'Ñ'Ð'äÐ!3´D¸×9JÑ9JÓ4KÐ3LÐMÓNÐNrG   c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   rï  rô  rå  r"   rL   rù   rA   )rN   s   "rH   rO   r¯  ˜	  s$   € ÷ 
ñ 
Ø%ð
Ø4>ð
à	ñ
rG   c                	ó”   € V P                   V,          V,          \        P                  P                  P                  P
                  8„  # rQ   )rÇ  rd   Ú	_inductorrv   r˜   Ú#cudagraph_unexpected_rerecord_limit)r¬   rï  rå  s   &&&rH   Úexceed_rerecord_limitÚ*CUDAGraphTreeManager.exceed_rerecord_limit˜	  s;   € ð ×Ñ˜gÕ& {Õ3Ü�o‰o×$Ñ$×+Ñ+×OÑOñPð	
rG   c               ó$   € V ^8„  d   QhRRRRRR/# rÙ  rA   )rN   s   "rH   rO   r¯   	  s+   € ÷ ZAñ ZA˜ð ZA¸Zð ZAÈJñ ZArG   c                	óÊ  € V P                   '       d   V P                  V4       V P                  '       d   V P                  V4       V P                  '       dZ   V P
                  \        P                  8X  d;   V P                  4       pV P                  V4      '       d   V P                  VR R7       V P                  4       pW P                  V,          9  d   V P                  W!4       V P                  V,          V,          '       g   V P                  WB4      '       d#   V P                  V,          P!                  V4      # W P"                  9   g!   \$        P&                  P(                  '       d3   V P                  '       g!   \$        P&                  P*                  '       dZ   V P
                  \        P                  8X  d   V P-                  4        \.        ;_uu_ 4        V P1                  W4      uuRRR4       # \3        V P4                  \6        4      '       d   \9        R4      hV P4                  f   V P:                  MV P4                  P<                  pV P                   '       Eg(   RpRpWR,           Fí  pVP?                  V4      w  ršV	\@        PB                  8X  d   V PE                  W�4      u # V	\@        PB                  8w  g   KU  V	\@        PF                  8w  d   R p\H        PK                  \L        PN                  4      '       dW   V
! 4       p\H        PQ                  RV PR                  V PU                  V4      V PV                  V,          PX                  V4       Kë  T
pKï  	  V P4                  e@   W P:                  9   d0   V P                  4        V P4                  f   V P[                  W4      # \]        V P                  V,          P^                  4      ^ 8”  db   V P                  W!4       V P                  V P                  4       ,          V,          '       d#   V P                  V,          P!                  V4      # V'       Ed   V P                  4       pV P`                  V,          V;;,          ^,          uu&   V P                  W²4      '       dµ   V'       d   VPb                  MRpVf   \9        R4      h\3        V\d        4      '       d   TMV! 4       p\g        RVPb                   R\h        Pj                  P$                  P&                  Pl                   R	V R
V R2	4       V P                  V,          P!                  V4      # V ;Pn                  ^,          un7        V P                  4        V P4                  e   V P-                  4        \.        ;_uu_ 4        V Pq                  W4      uuRRR4       #   + '       g   i     ELÐ; i  + '       g   i     R# ; i)T)Úskip_dead_output_cleanupNú0expected current_node to not be a CUDAWarmupNodeFz1[%s] Re-recording function=%s, mode=%s, reason=%sz2expected unexpected_rerecord_reason to not be Nonez#skipping cudagraph due to function z$ exceeding max re-recording limit (=z) on cudagraph node z due to Ú.)9Úin_recordingÚtry_end_curr_recordingÚ	in_warmupÚtry_end_curr_warmuprÐ  rÈ  r§  r«  Úget_curr_generationÚin_new_torch_compile_invocationÚtry_end_curr_executionrî  rÅ  rñ  rû  r¹  r  r»  rv   r˜   Úskip_cudagraph_warmupÚforce_cudagraphs_warmupÚ-apply_checkpoint_execution_state_in_allocatorr¾  Ú	run_eagerrc   rÊ  r”  r¾   r¸  r  rI  r    rE  Úexecute_noderT  r  ÚisEnabledForÚloggingÚDEBUGÚdebugr  Úget_func_namerÎ  r&  rÏ  rn   r  rÇ  rz   rW   r#   rd   rù  rú  rÌ  Úrecord_function)r¬   r¯  rå  Úcurr_generationrï  Úchild_nodesÚunexpected_rerecordÚunexpected_rerecord_reasonrP  rA  Ústatus_loggerÚcurr_node_idÚ_idÚreasons   &&&           rH   rÛ  ÚCUDAGraphTreeManager._run 	  s¹  € ð
 ××ÐØ×'Ñ'¨Ô4à�>�>ˆ>Ø×$Ñ$ [Ô1ð ×5×5Ð5Ø—‘¤>×#;Ñ#;Ô;à"×6Ñ6Ó8ˆOð ×3Ñ3°O×DÒDØ×+Ñ+Ø#Ø-1ð ,ô ð
 ×#Ñ#Ó%ˆØ×FÑFÀwÕOÔOØ×7Ñ7¸ÔPð
 ×3Ñ3°GÕ<Ø÷
ô 
à×'Ñ'¨×=Ò=Ø×$Ñ$ [Õ1×7Ñ7¸
ÓCÐCð  ×#;Ñ#;Ô;Ü—}‘}×:×:Ð:ð �~�~ˆ~Ü�}‰}×4×4Ð4ð �‰¤.×":Ñ":Ô:Ø×BÑBÔDç#Ö#Ø—~‘~ jÓ>÷ $Ò#ô �d×'Ñ'¬×8Ò8Ü Ð!SÓTÐTà×+Ñ+Ò3ˆD�JŠJ¸×9JÑ9J×9SÑ9Sð 	ð × × Ñ Ø"'ÐØ)-Ð&Ø$×1Ð1�ð ).×(>Ñ(>¸zÓ(JÑ%�ØÔ1×9Ñ9Ô9Ø×,Ñ,¨UÓ?Ò?àÔ1×9Ñ9Ö9ð Ô!5×!LÑ!LÔLØ.2Ð+ä×'Ñ'¬¯©×6Ò6Ù5B³_Ð2ÜŸ	™	ØOØ ŸO™OØ ×.Ñ.¨{Ó;Ø ŸO™O¨KÕ8×=Ñ=Ø6öð 6CÒ2ñ; 2ðF × Ñ Ò,°Ç
Á
Ô1JØ×+Ñ+Ô-ð ×$Ñ$Ò,ØŸ8™8 JÓ<Ð<ä�4×$Ñ$ [Õ1×DÑDÓEÈÔIØ×;Ñ;¸KÔTØ×;Ñ;¸D×<MÑ<MÓ<OÕPØ÷ô ð  ×,Ñ,¨[Õ9×?Ñ?À
ÓKÐK÷
 #Ð"Ø#×0Ñ0Ó2�Ø×!Ñ! ,Õ/°×<ÀÕAÓ<Ø×-Ñ-¨l×HÒHß-9˜,Ÿ/š/¸t�Cð 2Ò9Ü,ØPóð ô
 &Ð&@Ä#×FÒFñ 3á7Ó9ð ô
 8Ø=¸k¿n¹nÐ=Mð Nä"Ÿ_™_×3Ñ3×:Ñ:×^Ñ^Ð_ð `-Ø-0¨E°¸&¸ÀðDôð  ×,Ñ,¨[Õ9×?Ñ?À
ÓKÐKð ×#Ò# qÕ(Õ#à×'Ñ'Ô)Ø× Ñ Ò,Ø×BÑBÔD÷  ÖØ×'Ñ'¨
Ó@÷  Ò÷A $×#Ð#ú÷@  ×Ðús   Ç"V=Ö!WÖ=W	×W"	c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¯  <
  s   € ÷ !ñ !˜$ñ !rG   c                ó^  € . pV P                   P                  4        F  pVP                  V4       K  	  V'       d\   VP                  4       pVP                  P                  4        F  pVP                  V4       K  	  VP                  4        RVn        Kc  RV n        RV n         RV n        R# )zä
Remove all cached tensors in all nodes. Because cached tensors can hold gradients which in turn
might reference a backward which invokes a CUDA Graph Node, we have to manually clear them on shutdown
to avoid a reference cycle.
N)r¸  rM  rÉ  Úpopr  r  r§   rÊ  )r¬   rÔ  r¸  rÕ  r  s   &    rH   rç   ÚCUDAGraphTreeManager.shutdown<
  sˆ   € ð ˆØ—Z‘Z×&Ñ&Ö(ˆEØ�L‰L˜Öñ )÷ Ø—9‘9“;ˆDØ ŸM™M×0Ñ0Ö2�Ø—‘˜XÖ&ñ 3à×+Ñ+Ô-ØˆDŽJàˆŒ
ØˆŒ
Ø ˆÖrG   c               ó$   € V ^8„  d   QhRRRRRR/# rÙ  rA   )rN   s   "rH   rO   r¯  Q
  s$   € ÷ &5ñ &5Ø)ð&5Ø8Bð&5à	ñ&5rG   c                	óŠ  € \        V P                  \        4      '       d   \        R 4      h\        P
                  P                  P                  \        P                  \        V P                  4      4      ;_uu_ 4        V P                  4       p\        P                  RV P                  V P                  V4      V P                   V,          P"                  VP$                  \'        V4      4       \        P(                  P+                  4        \-        V P.                  V,          VV P                  VV P0                  V P2                  V P4                  V,          V P6                  V P8                  V P                  4
      pV P                  f$   V P:                  V,          P=                  V4       MV P                  P?                  W$4       W@n        \@        PB                  V n"        V PG                  4        \        P(                  P+                  4        VPI                  V4      uuRRR4       #   + '       g   i     R# ; i)r   zA[%s] Recording function=%s, mode=%s, cuda_graph_id=%d, inputs: %sN)%rc   rÊ  r”  r¾   rd   Ú_dynamoÚcallback_handlerÚinstall_callbacksr   ÚCUDAGRAPH_RECORDINGrW   r  Únew_graph_idr  r  r  rÎ  r&  rz   ru   r’   r…  r  r¹  rÁ  r¡   rº  r‚  r(  r¸  rp   rç  r§  rª  rÈ  Úupdate_generationr_  )r¬   r¯  rå  Úgraph_idrÕ  s   &&&  rH   r  Ú$CUDAGraphTreeManager.record_functionQ
  s£  € ô �d×'Ñ'¬×8Ò8Ü Ð!SÓTÐTÜ�]‰]×+Ñ+×=Ñ=Ü×/Ñ/´°T·_±_Ó1E÷
õ 
ð ×(Ñ(Ó*ˆHÜ�I‰IØSØ—‘Ø×"Ñ" ;Ó/Ø—‘ Õ,×1Ñ1Ø—‘Ü! *Ó-ôô �J‰J×"Ñ"Ô$Ü Ø×!Ñ! +Õ.ØØ×!Ñ!ØØ×,Ñ,Ø×!Ñ!Ø×(Ñ(¨Õ5Ø—‘Ø—	‘	Ø—‘óˆDð × Ñ Ò(Ø—
‘
˜;Õ'×.Ñ.¨tÕ4à×!Ñ!×+Ñ+¨KÔ>Ø $ÔÜ,×6Ñ6ˆDŒOØ×"Ñ"Ô$Ü�J‰J×"Ñ"Ô$Ø×(Ñ(¨Ó4÷C
÷ 
÷ 
ó 
ús   Á?F'H1È1I	c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   rÕ  r  r¯  r  rL   r'   rA   )rN   s   "rH   rO   r¯  y
  s$   € ÷ $ñ $Ø!ð$Ø/>ð$à	ñ$rG   c                	óz   € Wn         \        P                  V n        V P	                  4        VP                  V4      # rQ   )rÊ  r§  r«  rÈ  r(  rÏ  )r¬   rÕ  r¯  s   &&&rH   r  Ú!CUDAGraphTreeManager.execute_nodey
  s2   € ð !ÔÜ(×2Ñ2ˆŒØ×ÑÔ Ø�x‰x˜
Ó#Ð#rG   c               ó$   € V ^8„  d   QhRRRRRR/# rÙ  rA   )rN   s   "rH   rO   r¯  �
  s$   € ÷ &ñ &Ø)ð&Ø8Bð&à	ñ&rG   c                	óÄ  € W P                   9   pV P                  V4      pV'       g?   \        P                  R V P                  VV P
                  V,          P                  4       M!\        P                  RV P                  V4       V P                   P                  V4       \        V P                  V,          V P                  V P                  V P                  V P                  V P                  V,          V P                  VV P!                  4       4	      pWPn
        \"        P$                  V n        V P)                  4        W n        VP-                  V4      pRV n        V# )z([%s] Running warmup function=%s, mode=%sz[%s] Running eager function=%sN)r»  r  r  r  r  rÎ  r&  rÈ  r”  r¹  rÊ  rÁ  r§   r¡   rº  r‚  rê  r§  r©  rÈ  r(  rÉ  rÏ  )r¬   r¯  rå  r›  Ú	func_namerÕ  Úresults   &&&    rH   r  ÚCUDAGraphTreeManager.run_eager�
  s!  € ð
 #×&>Ñ&>Ñ>ˆØ×&Ñ& {Ó3ˆ	ßÜ�I‰IØ:Ø—‘ØØ—‘ Õ,×1Ñ1õ	ô �I‰IØ0Ø—‘Øôð
 	× Ñ ×$Ñ$ [Ô1ÜØ×Ñ˜kÕ*Ø×ÑØ×(Ñ(Ø�J‰JØ×ÑØ×$Ñ$ [Õ1Ø�K‰KØØ×#Ñ#Ó%ó

ˆð !ÔÜ(×/Ñ/ˆŒØ×ÑÔ Ø&1Ô#Ø—‘˜*Ó%ˆØ&*ˆÔ#ØˆrG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¯  ©
  s   € ÷ Jñ J¨ñ JrG   c                	óp   € V P                   e(   V P                  P                  V P                   4       R # R # rQ   )rÉ  r»  Údiscardr³   s   &rH   rß   Ú+CUDAGraphTreeManager.mark_warmup_incomplete©
  s-   € Ø×&Ñ&Ò2Ø×$Ñ$×,Ñ,¨T×-HÑ-HÖIñ 3rG   c               ó   € V ^8„  d   QhRR/# rè  rA   )rN   s   "rH   rO   r¯  ­
  s   € ÷ 1ñ 1˜gñ 1rG   c                	ó>   € \        \        V P                  4      4      # rQ   )ry   r•  rÃ  r³   s   &rH   r'  Ú!CUDAGraphTreeManager.new_graph_id­
  s   € Ü”t˜D×.Ñ.Ó/Ó0Ð0rG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   r"   rA   )rN   s   "rH   rO   r¯  °
  s   € ÷ 3ñ 3˜Zñ 3rG   c                	ó>   € \        \        V P                  4      4      # rQ   )r"   r•  rÄ  r³   s   &rH   Únew_func_idÚ CUDAGraphTreeManager.new_func_id°
  s   € Üœ$˜t×0Ñ0Ó1Ó2Ð2rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   rå  r"   rL   rW   rA   )rN   s   "rH   rO   r¯  ³
  s   € ÷ Tñ T¨ð T¸ñ TrG   c                	óR   € \        V P                  V,          P                  R R4      # )rB   Úunknown)rò   r¹  r  ©r¬   rå  s   &&rH   r  Ú"CUDAGraphTreeManager.get_func_name³
  s"   € Ü�t×(Ñ(¨Õ5×;Ñ;¸ZÈÓSÐSrG   c               óH   € V ^8„  d   QhRRRRRRRRR	R
RRRRRRRRRRRRRR/# )rJ   r  r&   rU   r  r  r  r2  r9  r(  r/  r3  r:  r4  r;  r  r<  r5  rù   r6  r  r'  rL   r=  rA   )rN   s   "rH   rO   r¯  ¶
  s‚   € ÷ (ñ (àð(ð  ð(ð )ð	(ð
 )ð(ð ð(ð ,ð(ð 2ð(ð ,ð(ð  $ð(ð #2ð(ð %ð(ð
ñ(rG   c           
     	ó  € V P                  4       pV\        P                  8X  d   Rp
\        V
4      pV'       d   R V n        W@P
                  V&   \        T\        V4      T\        ;QJ d    . R V 4       F  NK  	  5M! R V 4       4      VVV	V4      V P                  V&   WPP                  V&   W°P                  V&   \        P                  ! V P                  VR7      p\        V P                   4      P#                  V4       Wî! V4      3# )Tc              3  ó’   "  € T F=  p\        V\        P                  4      '       g   K%  VP                  '       g   K9  Vx € K?  	  R # 5irQ   )rc   rd   r   rÀ  )r  r�  s   & rH   r  Ú4CUDAGraphTreeManager.add_function.<locals>.<genexpr>Ò
  s'   é € ÐT™Y˜¬*°Q¼¿¹×*E”!È!Ï)Í)—!’!›Yùs   ‚"A©A½
A)rå  rA   )r<  r/  r0  Ú	frozensetrÐ  rº  r)   re   r©  r¹  rÎ  rÏ  rç  r½  rÏ  rö   r¡   rÅ   )r¬   r  rU   r  r2  r(  r3  r4  r  r5  r6  r  rz   Úuser_visible_output_idxs_setrÁ   s   &&&&&&&&&&&&   rH   rA  Ú!CUDAGraphTreeManager.add_function¶
  sò   € ð" ×ÑÓˆØ”?×+Ñ+Ô+Ø')Ð$Ü'0Ð1IÓ'JÐ$ß'Ø8<ˆDÔ5Ø'3× Ñ  Ñ$Ü /ØÜÐ"Ó#ØßŒEÑT™YÓT�E‰EÑT™YÓTÓTØØØ!Ø(ó	!
ˆ×Ñ˜"Ñð #�‰˜ÑØ$.×Ñ˜bÑ!Ü×Ò˜tŸx™x°RÔ8ˆô 	�d×'Ñ'Ó(×=Ñ=¸bÔAØ�2�f“:ˆ~ÐrG   c               ó   € V ^8„  d   QhRR/# rk  rA   )rN   s   "rH   rO   r¯  á
  s   € ÷ ;ñ ;˜dñ ;rG   c                	ó<   € V P                   \        P                  8H  # rQ   )rÈ  r§  rª  r³   s   &rH   r  Ú!CUDAGraphTreeManager.in_recordingà
  s   € à�‰¤.×":Ñ":Ñ:Ð:rG   c               ó   € V ^8„  d   QhRR/# rk  rA   )rN   s   "rH   rO   r¯  å
  s   € ÷ 8ñ 8˜4ñ 8rG   c                	ó<   € V P                   \        P                  8H  # rQ   )rÈ  r§  r©  r³   s   &rH   r  ÚCUDAGraphTreeManager.in_warmupä
  s   € à�‰¤.×"7Ñ"7Ñ7Ð7rG   c               ó   € V ^8„  d   QhRR/# )rJ   rL   zIterator[CUDAGraphNode]rA   )rN   s   "rH   rO   r¯  è
  s   € ÷ ñ Ð2ñ rG   c              #  	óh   "  € V P                   P                  4        F  pT R j  x€L
  K  	  R #  L
5irQ   )r¸  rM  rÑ  s   & rH   Ú	get_rootsÚCUDAGraphTreeManager.get_rootsè
  s(   é € Ø—Z‘Z×&Ñ&Ö(ˆEØ×Òó )Ùùs   ‚#2¥0¦2c               ó   € V ^8„  d   QhRR/# )rJ   rL   r˜  rA   )rN   s   "rH   rO   r¯  í
  s   € ÷ "ñ "ÐCñ "rG   c                	ó   € V P                   # rQ   )Ú_current_noder³   s   &rH   rÊ  Ú!CUDAGraphTreeManager.current_nodeì
  s   € à×!Ñ!Ð!rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   Úvaluer˜  rL   rM   rA   )rN   s   "rH   rO   r¯  ñ
  s   € ÷ 2ñ 2Ð"Gð 2ÈDñ 2rG   c                	óH   € Wn         Vf   \        P                  V n        R # R # rQ   )rV  r§  r¨  rÈ  )r¬   rY  s   &&rH   rÊ  rW  ð
  s   € à"ÔØŠ=Ü,×1Ñ1ˆDŽOñ rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¯  ö
  s   € ÷ 6ñ 6 4ñ 6rG   c                	ó0   € V P                  4       V n        R # rQ   ©r  rË  r³   s   &rH   r(  Ú&CUDAGraphTreeManager.update_generationö
  s   € Ø×3Ñ3Ó5ˆÖrG   c               ó   € V ^8„  d   QhRR/# rb  rA   )rN   s   "rH   rO   r¯  ú
  s   € ÷ ,ñ , ñ ,rG   c                 	ól   € \         P                  ^ 8w  d   \         P                  # \        P                  # r¾  )rÕ   rÖ   r   Ú
generationrA   rG   rH   r  Ú(CUDAGraphTreeManager.get_curr_generationù
  s(   € ä×(Ñ(¨AÔ-Ü×0Ñ0Ð0ä ×+Ñ+Ð+rG   c               ó   € V ^8„  d   QhRR/# rk  rA   )rN   s   "rH   rO   r¯    s   € ÷ 2ñ 2 Dñ 2rG   c                 	ó(   € \         P                  ^ 8g  # r¾  rÙ   rA   rG   rH   Úuser_invoked_mark_stepÚ+CUDAGraphTreeManager.user_invoked_mark_step   s   € ä×,Ñ,°Ñ1Ð1rG   Nc               ó    € V ^8„  d   QhRRRR/# ©rJ   r  rø   rL   rù   rA   )rN   s   "rH   rO   r¯    s   € ÷ @ñ @¸
ð @Èdñ @rG   c                	ó†   € V P                  V4      '       g   R # V P                  4       '       d   R# V P                  '       * # )FT)r  re  rÑ  ©r¬   r  s   &&rH   Úcan_start_new_generationÚ-CUDAGraphTreeManager.can_start_new_generation  s7   € Ø×3Ñ3°O×DÒDÙà×&Ñ&×(Ò(Ùà×?Ñ?Ô?Ð?rG   c               ó    € V ^8„  d   QhRRRR/# rh  rA   )rN   s   "rH   rO   r¯    s   € ÷ 3ñ 3Ø)ð3à	ñ3rG   c                	óH   € Vf   V P                  4       pV P                  V8g  # rQ   r]  rj  s   &&rH   r  Ú4CUDAGraphTreeManager.in_new_torch_compile_invocation  s)   € ð Ò"Ø"×6Ñ6Ó8ˆOØ×Ñ ?Ñ2Ð2rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¯    s   € ÷ Fñ F¸tñ FrG   c                	ó<  € V P                   f   \        R4      hV P                   P                  pV'       g   R # . pV F?  pV F6  w  pppp\        WG,          4      pVf   K  Vw  ršVP	                  W“34        K=  	  KA  	  V'       d   V P                  V4       R # R # ©Nú$expected current_node to not be None)rÊ  r¾   r«  rz  rp   Ú!_clone_live_user_visible_storages)r¬   r«  Ústorages_to_cloneÚoutput_entriesrc  Ú_tensor_weakrefsÚ_cached_tensor_outputsr  rÙ  Ústorage_ptrrK  s   &          rH   Ú-_clone_current_path_live_user_visible_outputsÚBCUDAGraphTreeManager._clone_current_path_live_user_visible_outputs  s¯   € Ø×ÑÒ$Ü Ð!GÓHÐHà×Ñ×>Ñ>ð 	)÷ 0Ùð ð 	ó ?ˆNó  ñØØ Ø&Øä"-¨oÕ.BÓ"C�Ø"Ô.Ø-<Ñ*�KØ%×,Ñ,¨kÐ-JÔKÚó  ñ ?÷ Ø×2Ñ2Ð3DÖEñ rG   c               ó    € V ^8„  d   QhRRRR/# )rJ   ru  zCIterable[tuple[StorageWeakRefPointer, PathUserVisibleStorageGroup]]rL   rM   rA   )rN   s   "rH   rO   r¯  1  s$   € ÷ I:ñ I:ð
ðI:ð 
ñI:rG   c                	ó�  € . p\         P                  P                  V P                  4      ;_uu_ 4        . p. pV Fù  w  rV\         P                  P                  V4      p\         P                  ! VP                  4       3\         P                  VP                  R 7      pVP                  4       p	VP                  \         P                  ! R\         P                  VP                  R 7      P                  V^ VP                  4       3R4      4       VP                  V4       VP                  WyV34       Kû  	  \         P                  ! WC4       V EFP  w  rypVP                  V	4       V U
u. uF  q©P                  4       NK  	  pp
\         P                  P!                  V	P"                  4      '       d*   \         P                  P%                  V	P"                  4       \         P                  P'                  V	P"                  R4       \)        WkRR7       F‡  w  w  pppppV'       d	   Wï,          MRp\+        \,        WÏ,          4      pRWß&   VP/                  4        VP1                  V4       Vf   K^  Vf   Kd  \         P                  P3                  V4       RWï&   K‰  	  EKS  	  RRR4       R# u up
i   + '       g   i     R# ; i))rh   rë   z<CUDAGraph output storage was cloned before a new generation.T)ÚstrictNrA   )r²   )rd   r’   rë   r¡   r-   r}  Úemptyr+  Úuint8rI  rp   Úset_rH  Ú_swap_data_ptr_rr  r€   Ú_has_Standard_DeleterrÁ  Ú_free_And_Remove_DeleterFnÚ&_set_storage_data_ptr_access_error_msgrn  r	   r;   rh  r_  r  )r¬   ru  ÚreplacementsÚsource_tensorsÚreplacement_tensorsry  r[  Úsource_storageÚreplacementÚreplacement_storager  Úplaceholder_weakrefsrc  rŸ  r3  r  Úplaceholder_weakrefru  Ú
output_refs   &&                 rH   rt  Ú6CUDAGraphTreeManager._clone_live_user_visible_storages1  s]  € ð ð 	ô �Z‰Z×Ñ˜t×0Ñ0×1Õ1ØˆNØ"$Ðã(9Ñ$�Ü!&×!5Ñ!5×!HÑ!HÈÓ!U�Ü#ŸkškØ#×*Ñ*Ó,Ð.ÜŸ+™+Ø)×0Ñ0ô�ð
 '2×&AÑ&AÓ&CÐ#Ø×%Ñ%Ü—K’KØ¤%§+¡+°n×6KÑ6Kôç‘d˜>¨1¨~×/DÑ/DÓ/FÐ.HÈ$ÓOôð
 $×*Ñ*¨;Ô7Ø×#Ñ# ^È'Ð$RÖSñ ):ô  × Ò Ð!4ÔEä@LÑ<�°Wà×.Ñ.Ð/BÔCá=Dó(Ù=D¸×1Ñ1Ö3¹Wð %ð (ô —8‘8×1Ñ1Ð2E×2LÑ2L×MÒMÜ—H‘H×7Ñ7Ð8K×8RÑ8RÔSÜ—‘×?Ñ?Ø'×.Ñ.ØRôô +.Ø¸$÷+ñ'ñ Ø#Ø#Ø)ØØ&÷ 7LÐ-Ö2ÐQUð ô "&Ô&;¸_Õ=QÓ!R�JØ+/�OÑ(Ø×5Ñ5Ô7Ø×+Ñ+Ð,?Ô@à,Ô8¸XÔ=QÜŸ™×6Ñ6°xÔ@Ø59Ð-Ó2ô+ñ+ AM÷- 2Ñ1ùò2(÷3 2×1Ð1ús+   ´D8J4Å,J/
ÆC*J4É3J4É9,J4Ê/J4Ê4K	c               ó    € V ^8„  d   QhRRRR/# ©rJ   rå  r"   rL   rM   rA   )rN   s   "rH   rO   r¯  |  s   € ÷ Bñ B°*ð BÀñ BrG   c                óN  € V P                   '       g   \        R4      hV P                  f   \        R4      hV P                  4       '       d   V P	                  RRR7       R# V P                  P                  4       '       d   V P	                  4        R# V P                  V4       R# )zô
Check if the current recording can be terminated, either because all outputs of the
previously recorded node are dead or because it was executed in a different
generation. Will set current_node to None and in_recording to False if successful.
z expected in_recording to be TrueNrs  T)Úclone_live_user_visible_outputsÚdealloc_current_path_weakrefs)r  r¾   rÊ  rk  Ú(clear_current_path_state_and_set_to_nonerÞ  Ú'check_warn_on_unable_to_start_executingrA  s   &&rH   r  Ú+CUDAGraphTreeManager.try_end_curr_recording|  s•   € ð × × Ð Ü Ð!CÓDÐDØ×ÑÒ$Ü Ð!GÓHÐHð ×(Ñ(×*Ò*Ø×9Ñ9Ø04Ø.2ð :ô ñ à×Ñ×1Ñ1×3Ò3Ø×9Ñ9Ô;Ùà×4Ñ4°[ÖArG   rÿ  Fc               ó$   € V ^8„  d   QhRRRRRR/# )rJ   r  rø   rÿ  rù   rL   rM   rA   )rN   s   "rH   rO   r¯  •  s(   € ÷ <ñ <à#ð<ð #'ð	<ð
 
ñ<rG   c               ó.  € V P                   '       d   \        R4      hV P                  f   R# V P                  V4      '       d   V P	                  RR7       R# V'       d   R# V P                  P                  4       '       d   V P	                  4        R# R# )zß
Check if the current executing node can be terminated, either because all outputs of the
previously executed node are dead or because it was executed in a different generation.
Will set current_node to None if successful.
z!expected in_recording to be FalseNT)r“  )r  r¾   rÊ  rk  r•  rÞ  )r¬   r  rÿ  s   &&$rH   r  Ú+CUDAGraphTreeManager.try_end_curr_execution•  sƒ   € ð ××ÐÜ Ð!DÓEÐEØ×ÑÒ$Ùà×(Ñ(¨×9Ò9Ø×9Ñ9Ø04ð :ô ñ ç#Ùà×Ñ×1Ñ1×3Ò3Ø×9Ñ9Ö;ñ 4rG   c               ó    € V ^8„  d   QhRRRR/# r‘  rA   )rN   s   "rH   rO   r¯  ²  s   € ÷ Bñ B¨zð B¸dñ BrG   c                	ó*  € V P                   f   \        R4      hV P                  4       '       d*   V P                  4        V P	                  4        R V n         R # V P                   P                  4       '       d
   R V n         R # V P                  V4       R # rr  )rÊ  r¾   rk  rz  r”  rÞ  r–  rA  s   &&rH   r  Ú(CUDAGraphTreeManager.try_end_curr_warmup²  sz   € Ø×ÑÒ$Ü Ð!GÓHÐHØ×(Ñ(×*Ò*Ø×>Ñ>Ô@Ø×.Ñ.Ô0Ø $ˆDÔÙà×Ñ×1Ñ1×3Ò3Ø $ˆDÔÙà×4Ñ4°[ÖArG   c               ó    € V ^8„  d   QhRRRR/# r‘  rA   )rN   s   "rH   rO   r¯  Á  s   € ÷ #
ñ #
À:ð #
ÐRVñ #
rG   c                ó¬  € WP                   9   g   V P                  4       '       g   R# V P                  f   \        R4      hV P                  P                   Uu. uF"  pVP
                  P                  V8X  g   K   VNK$  	  pp\        V4      ^8:  d   R# \        \        P                  ! W0P                  34       Uu. uF3  pVP                  f   K  VP                  P
                  P                  NK5  	  up4      p\        V4      \        V4      8X  d   R# V P                   P                  V4       \        P                  ! R4       R# u upi u upi )zCWarn if we in a potential loop where we are unable to hit fast pathNrs  zíUnable to hit fast path of CUDAGraphs because outputs from a previous step still require backward. Ensure backward() is invoked or detach outputs. You may also call torch.compiler.cudagraph_mark_step_begin() before each model invocation.)r¼  r  rÊ  r¾   rª  r–  rz   rn   r/   r´  rµ  r—  rÈ  r;  Úwarn)r¬   rå  rÕ  Úexisting_nodesrr   Úparentss   &&    rH   r–  Ú<CUDAGraphTreeManager.check_warn_on_unable_to_start_executingÁ  s*  € ð ×0Ñ0Ô0Ø×7Ñ7×9Ò9áà×ÑÒ$Ü Ð!GÓHÐHð ×)Ñ)×9Ò9ó
á9�Ø×$Ñ$×'Ñ'¨;Ñ6÷ ˆDÙ9ð 	ð 
ô ˆ~Ó !Ô#Ùô ô #Ÿš¨×:KÑ:KÐ9MÔNóáN�AØ—8‘8ô -�—‘×)Ñ)×,Ô,ÙNñó
ˆô ˆw‹<œ3˜~Ó.Ô.Ùà×Ñ×!Ñ! +Ô.Ü�Šðiö	
ùò+
ùòs   ÁEÁ<EÂ>EÃ$EÚis_grad_outputc               ó$   € V ^8„  d   QhRRRRRR/# )rJ   Ústack_tracez
str | Noner¤  rù   rL   rW   rA   )rN   s   "rH   rO   r¯  ç  s$   € ÷ 
ñ 
Øð
Ø48ð
à	ñ
rG   c               	ó^   € V '       d   V P                  4       MR p V'       d   RV  R2# RV  R2# )z[Could not find stack trace]zrError: accessing gradient tensor output of CUDAGraphs that has been overwritten by a subsequent run. Stack trace: a¼  . This can happen with torch.compile(mode='reduce-overhead') and gradient accumulation when a .grad tensor is allocated during CUDAGraph capture. If you need gradient accumulation, allocate stable .grad buffers outside CUDAGraph capture before the compiled backward runs, for example by running an eager warmup iteration or by preallocating zeroed .grad tensors. If you do not need gradient accumulation, set .grad to None before each backward.ziError: accessing tensor output of CUDAGraphs that has been overwritten by a subsequent run. Stack trace: a…  . To prevent overwriting, call torch.compiler.cudagraph_mark_step_begin() before each model invocation. If a tensor is a user-visible output that must remain live across generations, manually clone the tensor outside torch.compile(), or set torch._inductor.config.triton.cudagraph_trees_generation_cloning = 'user_visible' to clone all user-visible tensors that persist across generations.)Ústrip)r¦  r¤  s   &$rH   Úformat_dealloc_msgÚ'CUDAGraphTreeManager.format_dealloc_msgæ  sV   € ÷
 $/ˆK×ÑÔÐ4Rð 	÷ ð5Ø5@°Mð Bdðdð	ðØ'˜=ð )DðDð		
rG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¯    s   € ÷ CUñ CU¨tñ CUrG   c           
     	óˆ  € V P                   f   \        R4      h/ pV P                   P                   EF‡  pVP                  f   \        R4      h\	        VP
                  4      \	        VP                  4      8w  d>   \        P                  R\	        VP
                  4      \	        VP                  4      4       V P                  VP                  P                  ,          \        P                  8H  p\        VP
                  VP                  4       FH  w  rEVf   R MV! 4       pVf   K  \        P                  P!                  VV P#                  WSR7      4       KJ  	  V P$                  '       d   EK@  \        VP&                  VP                  4       F#  w  ruV'       g   K  VV3WP)                  4       &   K%  	  EKŠ  	  \*        \,        ,          ! 4       pV P                   P/                  4        FÐ  pV! 4       p	V	'       g   K  VP)                  4       V9  g   K+  VP1                  VP)                  4       4       VP3                  VP)                  4       R4      w  rSV P#                  WSR7      p
\        P                  P5                  V	4       V P$                  '       d   K±  \        P                  P7                  Wš4       KÒ  	  R # )Nrs  z)expected node.stack_traces to not be Nonez7tensor_weakrefs length (%d) != stack_traces length (%d))r¤  )NF)rÊ  r¾   rª  r2  rn   rŸ  r  ÚwarningrÎ  r–  rz   r/  r0  rn  rd   r€   Ú_set_storage_access_error_msgr©  rÖ  rž  ri   r/   r   rÅ  rÈ  ræ   r„  r…  )r¬   Ústor_dealloc_inforÕ  r¤  r�  r¦  ÚtenrC  ÚdeletedÚ_storage_derefr¥  s   &          rH   r”  Ú2CUDAGraphTreeManager.dealloc_current_path_weakrefs  sG  € Ø×ÑÒ$Ü Ð!GÓHÐHð ACÐØ×%Ñ%×5Õ5ˆDØ× Ñ Ò(Ü$Ð%PÓQÐQô �4×'Ñ'Ó(¬C°×0AÑ0AÓ,BÔBÜ—‘ØMÜ˜×,Ñ,Ó-Ü˜×)Ñ)Ó*ôð —‘ × 5Ñ 5× 8Ñ 8Õ9¼_×=UÑ=UÑUð ô #& d×&:Ñ&:¸D×<MÑ<MÖ"N‘�Øši‘d©Q«S�Ø’;Ùä—‘×6Ñ6ØØ×+Ñ+¨KÐ+ÓWöñ #Oð ×.×.Ð.Úä,/Ø×%Ñ% t×'8Ñ'8ö-Ñ(�÷ #Ùð  Ø"ð=Ð!×"6Ñ"6Ó"8Ó9ô-ñA 6ôV œS–/Ó#ˆØ×,Ñ,×?Ñ?ÖAˆKÙ(›]ˆNß‰~ +×"6Ñ"6Ó"8ÀÖ"GØ—‘˜K×0Ñ0Ó2Ô3à.?×.CÑ.CØ×(Ñ(Ó*¨Mó/Ñ+�ð ×-Ñ-Øð .ó �ô —‘×3Ñ3°NÔCà×2×2Ð2Ùä—‘×?Ñ?ÀÖTó! BrG   r“  r”  c               ó$   € V ^8„  d   QhRRRRRR/# )rJ   r“  rù   r”  rL   rM   rA   )rN   s   "rH   rO   r¯  H  s*   € ÷ !ñ !ð *.ð!ð (,ð	!ð
 
ñ!rG   c               	ó*  € \        V P                  \        4      '       g"   \        R \	        V P                  4       24      hV'       d   V P                  4        V'       d   V P                  4        V P                  P                  4        RV n        R# )ú1expected current_node to be a CUDAGraphNode, got N)rc   rÊ  r  r¾   rj   rz  r”  r&  )r¬   r“  r”  s   &$$rH   r•  Ú=CUDAGraphTreeManager.clear_current_path_state_and_set_to_noneH  st   € ô ˜$×+Ñ+¬]×;Ò;Ü ØCÄDÈ×IZÑIZÓD[ÐC\Ð]óð ÷ +Ø×>Ñ>Ô@ß(Ø×.Ñ.Ô0Ø×Ñ×*Ñ*Ô,Ø ˆÖrG   c               ó   € V ^8„  d   QhRR/# r~   rA   )rN   s   "rH   rO   r¯  Y  s   € ÷ ;ñ ;¸tñ ;rG   c                óh  € \        V P                  \        4      '       g"   \        R\	        V P                  4       24      hV ;P
                  ^,          un        \        P                  RV P
                  4       V P                  P                  pV P                  P                  pVe   Vf   \        R4      h. pV P                  P                  4        \        V P                  P                  4       4      pV Uu. uF	  qU! 4       NK  	  ppV P                  P                  4       p\        P                  P!                  VVVV4       \#        V4       F"  p\        P                  P%                  V4       K$  	  \&        P(                  P*                  '       d“   \-        V P.                  V P0                  V4       V Fi  p	V	! 4       p
V
f   \        R4      h\        P                  P3                  V
4      '       g   \        R4      hV	P5                  4       V9   g   K`  \        R4      h	  R# R# u upi )z˜
Checkpoint the current execution state in the caching allocator so that
additional cudagraph recordings can be made respecting existent live storages.
r¶  zDCheckpointing cuda caching allocator state. Number of checkpoints %dNz(expected state and device to not be Nonez#expected storage_ptr to not be Nonez-expected storage_ptr to have standard deleterz5expected wrapper.data_ptr() not in ptrs_to_deallocate)rc   rÊ  r  r¾   rj   rÍ  r  r  r0  rë   r"  re   rÅ  r  rd   r€   Ú_cuda_setCheckpointPoolStater/   Ú%_cuda_cudaCachingAllocator_raw_deleterv   r˜   rÆ  rÇ  r¡   rÁ  rƒ  ri   )r¬   rq  rë   Ústale_storagesÚlive_storages_wrappersr�  Úlive_storages_weak_refsr  rÛ  Úwrapperry  s   &          rH   r  ÚBCUDAGraphTreeManager.apply_checkpoint_execution_state_in_allocatorY  sï  € ô
 ˜$×+Ñ+¬]×;Ò;Ü ØCÄDÈ×IZÑIZÓD[ÐC\Ð]óð ð 	×(Ò(¨AÕ-Õ(Ü�	‰	ØRØ×,Ñ,ô	
ð
 ×!Ñ!×<Ñ<ˆØ×"Ñ"×)Ñ)ˆØÒ! fÒ&8Ü Ð!KÓLÐLð %'ˆð 	×Ñ×4Ñ4Ô6Ü!% d×&7Ñ&7×&JÑ&JÓ&LÓ!MÐñ <RÓ-RÑ;Q°a¨a®cÑ;QÐÐ-RØ!×.Ñ.×NÑNÓPÐÜ�‰×-Ñ-ØàØØ#ô	
ô Ð0Ö1ˆCÜ�H‰H×:Ñ:¸3Ö?ñ 2ô �=‰=×4×4Ð4ÜØ×!Ñ! 4×#?Ñ#?ÐAWôó 2�Ù%›i�ØÒ&Ü(Ð)NÓOÐOÜ—x‘x×5Ñ5°k×BÒBÜ(ØGóð ð ×#Ñ#Ó%Ð);Ö;Ü(ØOóð ó 2ñ	 5ùò .Ss   Ã=H/c               ó   € V ^8„  d   QhRR/# )rJ   rL   zlist[StorageWeakRefPointer]rA   )rN   s   "rH   rO   r¯  –  s   € ÷ Eñ Eà	$ñErG   c                	óˆ   € V P                   f   . # V P                   P                  4        Uu. uF	  q! 4       NK  	  up# u upi rQ   )rÊ  rÅ  )r¬   r�  s   & rH   Ú.live_cudagraph_pool_storages_in_curr_executionÚCCUDAGraphTreeManager.live_cudagraph_pool_storages_in_curr_execution–  sB   € ð ×ÑÒ$ØˆIð "×.Ñ.×AÑAÔCÓDÑC˜�–ÑCÑDÐDùÒDs   ­?)rV  rÉ  r  rÁ  rË  rÊ  rÍ  rÌ  r¡   rÖ  rÄ  r§   rÃ  rÐ  rÏ  rÎ  r¹  rº  r(  rÅ  rÇ  rÈ  r¸  rÑ  r‚  r»  rÆ  r¼  r½  rQ   )/rB   rC   rD   rE   r{   r­   rÏ  rß  rå  rê  rñ  rî  rû  rÛ  rç   r  r  r  rß   r'  r<  r  rA  ré  r  r  rR  rÊ  Úsetterr(  rS  r  re  rk  r  rz  rt  r  r  r  r–  r©  r”  r•  r  rÃ  rF   rA   rG   rH   rÈ   rÈ   Õ  s‚  † ñõ0q
õfõ-õ
õ7õSõ$Oõ
õZAõx!õ*&5õP$õ&õPJõ1õ3õTõ(ðT ô;ó ð;ð ô8ó ð8õð ô"ó ð"ð ×Ñô2ó ð2õ
6ð ô,ó ð,ð ô2ó ð2÷@÷3õFõ:I:õVBñ2<ð */÷	<ð <õ:Bõ#
ðJ ð
Ø;@ö
ó ð
õ8CUðJ!ð 16ð!ð /4÷	!õ";÷zEñ ErG   rÈ   )NTrA   )rA   rR  )ªr{   Ú
__future__r   rš   Údataclassesrç  r�  r´  r  r!  rÃ  r¨   rz  r;  rÃ   Úcollectionsr   r   Úenumr   r   Útypingr   r	   r
   r   Útorch.fxrd   r   Útorch._custom_class_baser   Útorch._dynamo.callbackr   Ú#torch._dynamo.graph_bytecode_inputsr   r   Útorch._dynamo.mutation_guardr   Útorch._dynamo.utilsr   r   r   Ú3torch._higher_order_ops.cudagraph_conditional_nodesr   r   Útorch._inductor.compile_fxr   r   r   r   r   r   r   r   Útorch._inductor.cudagraph_utilsr   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   Útorch._library.opaque_objectr+   Ú torch.multiprocessing.reductionsr,   Útorch.storager-   Útorch.utilsr.   r÷  Útorch.utils._ordered_setr/   Útorch.utils.weakr0   Úcollections.abcr1   r2   r3   r4   r5   Útorch._guardsr6   Útorch._inductor.utilsr7   Ú
torch.cudar8   Útorch.typesr9   r[   ÚStorageWeakRefPointerÚStorageDataPtrÚNBytesr:   Úbackendsr’   Úis_builtÚtorch._Cr=   r@   r>   Ú_loggingÚgetArtifactLoggerrB   r  ru   r  rv   Ú	dataclassry   r‚   Úcontextmanagerr‡   rŽ   r•   rœ   rž   rÓ   rÍ   r©   r¾  rÕ   rÚ   rß   rí   rä   rö   rÞ   r  r$  r1  r  r;   r{  rz  r‹  rŽ  r’  r©  rñ  re   ÚPathUserVisibleOutputEntryÚPathUserVisibleStorageGrouprù   ÚPathLivenessrW   ÚStackTracesr”  Ú	InputListÚ
OutputListr  rë  rí  r|  rï  rû  r  rW  r¨  rk  rþ  r  rÇ  r§  r/  rÈ   rA   rG   rH   Ú<module>rï     sù  ðñ"õH #ã Û Û Û 	Û Û Û Û 
Û Û Û Û Ý #Ý -ß ß 4Ó 4ã Ý Ý 4Ý 2÷õ ;ß JÑ J÷÷	÷ 	ó 	÷÷ ÷ ñ õ =Ý ;Ý (Ý )Ý /Ý *÷ ßQÕQå'Ý/Ý'Ý!àÐ Ø€Ø	€ÙˆCÐ.Ô/€ð 	‡>�>×Ñ×Ñ×!Ò!÷ð ÷ñ õð ‡n�n×&Ñ& x°Ó>€õõ2 ð ×Ò˜dÔ#÷ð ó $ðõ+ð  ×Ñôó ðð ×Ñô=ó ð=ð ×Ñô	;ó ð	;õ&÷\%ñ \%õ~Oð 	�ŠÓ€Ù �Ô ð —^’^Ó%Ð ÷ñ õ'õ)õ&õ0/õ,÷Aõõ:!ðz ×Ñôó ðñ*'ð (,ð'ð +-ð'ð 13ð'ð +-ð'ð #(ð'ð 13ð'ð $(÷'ð '÷TMHñ MHõ`-õ	"ð ×Ñô4ó ð4ð4 ×Ñô	
ó ð	
õ$ð ˜˜S˜•/€à"ØÐ	 Õ	%Õ&Øˆ˜Õ	ÕØˆ�$�Õ˜$ÕØð	õÐ ð $Ð$>ÀÐ$CÕDÐ ð �D˜•JÕ€à�3˜•:Õ€÷[ñ [ð~ €	Ø€
Ø€	÷ñ ôP˜ô Pñ %Ó&Ð ô
˜oô 
ô
�ô 
÷Vñ Võr"	õ@MõFV÷
õ?õd ôN	�Tô 	ô�dô ÷HEó HErG   