+
    –°³j‘
 ã                  óŠ  € ^ RI Ht ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	t	^ RI
t
^ RIHt ^ RIHtHtHtHtHt ^ RIHt ^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RIHt ^ R	IHt ^ R
IH t  ^ RI!H"t"H#t#H$t$ ^ RI%H&t&H't'H(t(H)t) ^RI*H+t+ ^RI,H-t-H.t.H/t/ ^RI0H1t1 ^RI2H3t3H4t4 ^RI5H6t6H7t7H8t8 ]'       d   ^ RI9H:t: ^RI.H;t; ^RI<H=t= ^RI>H?t?H@t@ ^RIAHBtB ^RICHDtDHEtE ^RIFHGtGHHtHHItIHJtJ ^RI/HKtKHLtLHMtM ^RINHOtOHPtPHQtQHRtRHStSHTtTHUtUHVtVHWtWHXtXHYtY ^RIZH[t[H\t\H]t] ^RI^H_t_ ^RI`HataHbtbHctcHdtd ^RIeHftfHgtg ^RIhHitiHjtjHktkHltlHmtm ^@tn]'       d   ^ R I9HotoHptpHqtq ^ R!IrHsts ^ R"ItHutu ]Pì                  ! ]w4      tx]Pò                  Põ                  ]wR#4      t{]Pò                  Põ                  ]wR$4      t|]Pò                  Põ                  ]wR%4      t}]d! 4       Pü                  t] ! . RVO4      t€RWR& R' llt�]EP                   ! R( R)4      4       tƒ ! R* R+]ƒ4      t„ ! R, R-]ƒ4      t… ! R. R/]„4      t†R0 R1 lt‡]! R2]a]aR37      tˆ]EP                   ! R4 R54      4       t‰ ! R6 R7]4      tŠ ! R8 R9]c]ˆ,          ]]ˆ,          4      t‹]EP                  ! R:R;7       ! R< R=4      4       tŒ]EP                   ! R> R?4      4       t�]EP                  ! R:R;7       ! R@ RA4      4       tŽ]EP                  ! R:R;7       ! RB RC4      4       t�]EP                  ! R:R;7       ! RD RE4      4       t� ! RF RG]=4      t‘ ! RH RI]=4      t’ ! RJ RK]L4      t“]EP                  ! R:R;7       ! RL RM4      4       t”^t•]EP                  ! R:R;7       ! RN RO4      4       t–]EP                  ! R:R;7       ! RP RQ4      4       t—]EP                  ! R:R;7       ! RR RS4      4       t˜ ! RT RU]™4      tšR# )Xé    )ÚannotationsN)ÚCounter)ÚAnyÚcastÚGenericÚ
NamedTupleÚTYPE_CHECKING)ÚTypeVar)Úmetrics)ÚMultiTemplateBuffer)Úfree_unbacked_symbols)Úimmutable_dict)Ú
OrderedSet)ÚFloorDivÚIdentityÚModularIndexing)Úfree_symbol_is_typeÚ
prefix_strÚsymbol_is_typeÚSymT)Úcounters)ÚconfigÚirÚ	scheduler)Úprologue_preserves_zero_mask)Ú	code_hashÚPyCodeCache)Ú	MemoryDepÚStarDepÚWeakDep)ÚCallable©ÚIRNode)ÚWrapperHandler)Ú convert_index_expr_to_value_exprÚ!indexing_dtype_strength_reduction)ÚCoordescTuner)ÚDevicePropertiesÚInductorMeta)Ú
green_textÚlast_power_of_2Únext_power_of_2Úyellow_text)ÚBaseSchedulerNodeÚBaseSchedulingÚ	WhyNoFuse)Úcache_property_on_selfÚdecompose_indexÚexpr_fits_within_32bitÚget_dtype_sizeÚIndentedBufferÚPlaceholderÚprefix_is_reductionÚsympy_index_symbolÚsympy_productÚ
sympy_subsÚunique)ÚopsÚ
OpsWrapperÚV)ÚBlockPatternMatcher)ÚCSEVariableÚindex_prevent_reorderingÚKernelÚPythonPrinter)ÚMultiKernelÚSizeHintMultiKernel)ÚDisableReductionÚEnableReductionÚNodeScheduleEntryÚNodeScheduleMarkerÚSIMDKernelFeatures)ÚIterableÚIteratorÚSequence©ÚTritonKernel)ÚCoalesceVarAnalysisÚ
perf_hintsÚscheduleÚfusionc               ó    € V ^8„  d   QhRRRR/# )é   ÚdefaultÚintÚreturn© )Úformats   "Ú]/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/_inductor/codegen/simd.pyÚ__annotate__r\   k   s   € ÷ ;ñ ;˜3ð ; sñ ;ó    c                ón   € \         P                  P                  P                  P                  pVe   V# T # ©N)ÚtorchÚ	_inductorr   ÚtritonÚ	max_tiles)rV   rc   s   & r[   Úget_max_tilesrd   k   s-   € Ü—‘×&Ñ&×-Ñ-×7Ñ7€IØ!Ò-ˆ9Ð:°7Ð:r]   c                  óÚ   a € ] tR t^ptRtR]P                  P                  R]P                  P                  /R V 3R lllt]	]
R R l4       4       tR R	 lt]	]
R
 R l4       4       tRtV ;t# )ÚIterationRangesa”  
Each range tree represents multiple sets of iteration indexing
in a single tiled dimension in the output kernel.

If you have two loops ranges one (4, 3, 2) and another (4, 6),
then the range tree will be:
        4 (i0)
    3 (i1)  6 (i3)
    2 (i2)
Where i0 is shared between both loops, but then the split into
different indexing vars.  All loop ranges must iterate over
the same number of elements.
ÚdivisorÚlengthc               ó8   € V ^8„  d   QhRRRRRRRRR	RR
RRRRR/# )rU   ÚnameÚstrÚvar_listúlist[sympy.Symbol]Ú
var_rangesúdict[sympy.Symbol, sympy.Expr]Únumelú
sympy.ExprÚprefixÚkernelÚ
SIMDKernelÚrootÚIterationRangesRootrX   ÚNonerY   )rZ   s   "r[   r\   ÚIterationRanges.__annotate__€   sZ   € ÷ ñ àðð %ðð 3ð	ð
 ðð ðð ðð "ðð 
ñr]   c               	ó�   <€ \         S
V `  4        Wn        W n        W0n        W@n        WPn        Wpn        W€n        W`n	        W�n
        R # r_   )ÚsuperÚ__init__rj   rl   rn   rp   rr   rg   rh   rs   ru   )Úselfrj   rl   rn   rp   rr   rs   rg   rh   ru   Ú	__class__s   &&&&&&$$$$€r[   r{   ÚIterationRanges.__init__€   s=   ø€ ô 	‰ÑÔØŒ	Ø ŒØ$ŒØŒ
ØŒØŒØŒØŒØŽ	r]   c               ó   € V ^8„  d   QhRR/# ©rU   rX   ÚboolrY   )rZ   s   "r[   r\   rx   š   s   € ÷ 0ñ 0˜dñ 0r]   c                	ó,   € \        V P                  4      # r_   )r7   rr   ©r|   s   &r[   Úis_reductionÚIterationRanges.is_reduction˜   s   € ô # 4§;¡;Ó/Ð/r]   c               ó   € V ^8„  d   QhRR/# ©rU   rX   úsympy.SymbolrY   )rZ   s   "r[   r\   rx   �   s   € ÷ -ñ -˜ñ -r]   c                	ó,   € \        V P                  4      # r_   ©r8   rj   rƒ   s   &r[   ÚsymbolÚIterationRanges.symbol�   ó   € Ü! $§)¡)Ó,Ð,r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   r   rY   )rZ   s   "r[   r\   rx   ¢   s   € ÷ +ñ +�dñ +r]   c                	ó‚   € \         P                  ! 4        UUu/ uF  w  rW!bK	  	  pppW0P                  ,          # u uppi r_   )r   Úitemsrr   )r|   Úsymtrr   Úprefix_to_symts   &   r[   r‘   ÚIterationRanges.symt    s;   € ô <F×;KÒ;KÔ;MÔNÑ;M©<¨4˜&š,Ñ;MˆÑNØŸk™kÕ*Ð*ùó Os   ™;)	rg   rs   rh   rj   rp   rr   ru   rl   rn   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚsympyÚSÚOner{   Úpropertyr1   r„   r‹   r‘   Ú__static_attributes__Ú__classcell__©r}   s   @r[   rf   rf   p   ss   ø† ñðð —‘—‘ðð �w‰w�{‰{÷ò ð0 Øô0ó ó ð0õ-ð Øô+ó ó ö+r]   rf   c                  ó  a € ] tR t^§tRtR'R V 3R llltR R ltR R ltR R	 ltR
 R lt	R R lt
R R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR  R! ltR" R# ltR$ R% ltR&tV ;t# )(rv   z­
Root of a iteration range tree that represents a single
tiled dimension in the output kernel. It contains multiple
sets of iteration represented with IterationRangesEntry.
c               óD   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRRRRRR/# )rU   rj   rk   rp   rq   rr   ÚindexrW   rs   rt   Ú	pid_cacheúdict[str, str] | NoneÚis_loopr�   Ú
tensor_dimú
int | NoneÚgrid_dimÚhas_zdimrX   rw   rY   )rZ   s   "r[   r\   Ú IterationRangesRoot.__annotate__®   sx   € ÷ *!ñ *!àð*!ð ð*!ð ð	*!ð
 ð*!ð ð*!ð )ð*!ð ð*!ð ð*!ð ð*!ð ð*!ð 
ñ*!r]   c          
     	óè   <€ Vf   / p\         SV `  V. / VVVV R7       W@n        / V n        W`n        V'       d"   V P
                  '       d   V	e   \        R4      hWpn        W€n        W�n	        W n
        R # )N)rj   rl   rn   rp   rr   rs   ru   z.loop ranges must be reduction with no grid_dim)rz   r{   r¢   Únodesr£   r„   ÚAssertionErrorr¥   r¦   r¨   r©   )r|   rj   rp   rr   r¢   rs   r£   r¥   r¦   r¨   r©   r}   s   &&&&&&&$$$$€r[   r{   ÚIterationRangesRoot.__init__®   s�   ø€ ð ÒØˆIÜ‰ÑØØØØØØØð 	ô 	
ð Œ
à=?ˆŒ
ð *3Œ÷
 ˜D×-×-Ð-°(Ò2BÜ Ð!QÓRÐRØŒà$Œà ŒØ Žr]   c               ó   € V ^8„  d   QhRR/# ©rU   rX   rk   rY   )rZ   s   "r[   r\   rª   Ú   s   € ÷ Hñ H˜#ñ Hr]   c                	ó>   € R V P                   : RV P                   R2# )zIterationRangesRoot(ú, z, ...))rj   rp   rƒ   s   &r[   Ú__repr__ÚIterationRangesRoot.__repr__Ú   s   € Ø% d§i¡i¡]°"°T·Z±Z°LÀÐGÐGr]   c               ó   € V ^8„  d   QhRR/# ©rU   rX   rq   rY   )rZ   s   "r[   r\   rª   Ý   s   € ÷ Xñ X˜Jñ Xr]   c                	ój   € \         P                  ! V P                  P                  4        R 2RRR7      # )ÚBLOCKT©ÚintegerÚpositive)r™   ÚSymbolrr   Úupperrƒ   s   &r[   Ú
block_sizeÚIterationRangesRoot.block_sizeÝ   s+   € Ü�|Š|˜tŸ{™{×0Ñ0Ó2Ð3°5Ð9À4ÐRVÔWÐWr]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   rª   à   s   € ÷ &ñ & ñ &r]   c                	ó4   € \        V P                  4       4      # r_   )rk   r¾   rƒ   s   &r[   Úblock_size_strÚ"IterationRangesRoot.block_size_strà   s   € Ü�4—?‘?Ó$Ó%Ð%r]   c               ó   € V ^8„  d   QhRR/# r¶   rY   )rZ   s   "r[   r\   rª   ã   s   € ÷ Tñ T˜jñ Tr]   c                	óN   € \         P                  ! V P                   R 2RRR7      # )ÚoffsetT)rº   Únonnegative)r™   r¼   rr   rƒ   s   &r[   Úblock_offsetÚ IterationRangesRoot.block_offsetã   s"   € ô �|Š|˜tŸ{™{˜m¨6Ð2¸DÈdÔSÐSr]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   rª   è   s   € ÷ $ñ $˜3ñ $r]   c                	ó    € V P                    R 2# )Úmask)rr   rƒ   s   &r[   Ú	mask_nameÚIterationRangesRoot.mask_nameè   s   € Ø—+‘+�˜dÐ#Ð#r]   c               ó    € V ^8„  d   QhRRRR/# )rU   Úmask_varrk   rX   r�   rY   )rZ   s   "r[   r\   rª   ë   s   € ÷ ,ñ , #ð ,¨$ñ ,r]   c                	ó&   € WP                  4       8H  # r_   )rÍ   )r|   rÐ   s   &&r[   Ú	owns_maskÚIterationRangesRoot.owns_maskë   s   € ØŸ>™>Ó+Ñ+Ð+r]   c               ó    € V ^8„  d   QhRRRR/# )rU   Útensor_ndimrW   rX   ztuple[str, ...]rY   )rZ   s   "r[   r\   rª   î   s   € ÷ ñ  cð ¨oñ r]   c                	ó„   € V P                   f   R# R.V,          pV P                  4       W P                   &   \        V4      # )NÚ1rY   )r¦   rÂ   Útuple)r|   rÕ   Úshapes   && r[   Ú
mask_shapeÚIterationRangesRoot.mask_shapeî   s<   € Ø�?‰?Ò"ØˆIà�˜Õ#ˆØ!%×!4Ñ!4Ó!6ˆ�o‰oÑÜ�U‹|Ðr]   c               ó   € V ^8„  d   QhRR/# r€   rY   )rZ   s   "r[   r\   rª   ö   s   € ÷ ñ ¨ñ r]   c                	ó   € R # ©TrY   rƒ   s   &r[   Úsupports_constant_maskÚ*IterationRangesRoot.supports_constant_maskö   ó   € Ùr]   c               ó   € V ^8„  d   QhRR/# r€   rY   )rZ   s   "r[   r\   rª   ù   s   € ÷ ñ ¨4ñ r]   c                	ó   € R # ©FrY   rƒ   s   &r[   Úhas_custom_codegen_headerÚ-IterationRangesRoot.has_custom_codegen_headerù   ó   € Ùr]   c               ó   € V ^8„  d   QhRR/# ©rU   rX   ú1tuple[tuple[sympy.Symbol, sympy.Expr, bool], ...]rY   )rZ   s   "r[   r\   rª   ü   s   € ÷ ñ Ð!Rñ r]   c                	ó   € R# ©NrY   rY   rƒ   s   &r[   Únamed_constantsÚ#IterationRangesRoot.named_constantsü   s   € Øˆ	r]   c               ó   € V ^8„  d   QhRR/# ©rU   rX   rw   rY   )rZ   s   "r[   r\   rª   ÿ   s   € ÷ ñ ˜Tñ r]   c                	óh   € V P                   P                  4        F  pVP                  4        K  	  R # r_   )r¬   ÚvaluesÚcache_clear)r|   Únodes   & r[   ró   ÚIterationRangesRoot.cache_clearÿ   s%   € Ø—J‘J×%Ñ%Ö'ˆDØ×ÑÖó (r]   c               ó   € V ^8„  d   QhRR/# r‡   rY   )rZ   s   "r[   r\   rª     s   € ÷ 9ñ 9˜<ñ 9r]   c                	ó2   € \        V P                   R 24      # )r¢   )r8   rr   rƒ   s   &r[   Ú	index_symÚIterationRangesRoot.index_sym  s   € Ü! T§[¡[ M°Ð"7Ó8Ð8r]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   rg   rq   rh   rX   ÚIterationRangesEntryrY   )rZ   s   "r[   r\   rª     s"   € ÷  ñ  ˜jð  °*ð  ÐAUñ  r]   c                óª  € \         P                  P                  P                  W,          V P                  4      '       d   \        V P                  4       V4      pM\        V P                  4       W4      pW0P                  9  d½   \        V P                   \        \         P                  P                  4       2VVVV 4      pV\         P                  P                  VP                  4       &   V P                   P#                  VP                  4       4       W P$                  VP                  4       &   W@P                  V&   V P                  V,          # )z6
Lookup a given RangeTreeEntry, creating it if needed
)r>   ÚgraphÚsizevarsÚstatically_known_equalsrp   r   rø   r   r¬   rû   rr   Únextrs   Úiter_vars_countÚrange_tree_nodesr‹   rl   Úappendrn   )r|   rg   rh   Úexprrô   s   &&&  r[   ÚlookupÚIterationRangesRoot.lookup  sï   € ô �7‰7×Ñ×3Ñ3°GÕ4DÀdÇjÁj×QÒQÜ˜DŸN™NÓ,¨gÓ6‰Dä" 4§>¡>Ó#3°WÓEˆDà—z‘zÔ!Ü'Ø—;‘;�-¤¤Q§X¡X×%=Ñ%=Ó >Ð?Ð@ØØØØóˆDð 8<ŒA�H‰H×%Ñ% d§k¡k£mÑ4Ø�M‰M× Ñ  §¡£Ô/Ø-3�O‰O˜DŸK™K›MÑ*Ø#�J‰J�tÑØ�z‰z˜$ÕÐr]   c               ó   € V ^8„  d   QhRR/# )rU   rX   rû   rY   )rZ   s   "r[   r\   rª     s   € ÷ 4ñ 4Ð0ñ 4r]   c                ój   € V P                  \        P                  P                  V P                  4      # )z«Return the canonical entry for this root's unsplit logical index.

This is ``lookup(1, numel)``, so the entry is interned in
``range_tree_nodes`` just like split entries.
)r  r™   rš   r›   rp   rƒ   s   &r[   Ú
full_rangeÚIterationRangesRoot.full_range  s!   € ð �{‰{œ5Ÿ7™7Ÿ;™;¨¯
©
Ó3Ð3r]   c               ó    € V ^8„  d   QhRRRR/# )rU   Úlengthsúlist[sympy.Expr]rX   zlist[IterationRangesEntry]rY   )rZ   s   "r[   r\   rª   %  s   € ÷ %ñ %Ø'ð%à	#ñ%r]   c                	óÈ   € \         P                  P                  p. p\        V4       F+  pVP	                  V P                  W$4      4       W$,          pK-  	  . \        V4      O# r_   )r™   rš   r›   Úreversedr  r  )r|   r  rg   Úitervarsrh   s   &&   r[   Úconstruct_entriesÚ%IterationRangesRoot.construct_entries%  sT   € ô —'‘'—+‘+ˆØˆÜ˜wÖ'ˆFØ�O‰O˜DŸK™K¨Ó8Ô9ØÕ&ŠGñ (ð %”˜(Ó#Ð$Ð$r]   c               ó    € V ^8„  d   QhRRRR/# )rU   r  r  rX   rm   rY   )rZ   s   "r[   r\   rª   /  s    € ÷ Eñ EÐ!1ð EÐ6Hñ Er]   c                	óh   € V P                  V4       Uu. uF  q"P                  4       NK  	  up# u upi r_   )r  r‹   )r|   r  Úes   && r[   Ú	constructÚIterationRangesRoot.construct/  s+   € Ø$(×$:Ñ$:¸7Ô$CÓDÑ$C˜q—‘–
Ñ$CÑDÐDùÒDs   ”/c               ó    € V ^8„  d   QhRRRR/# )rU   r¢   rq   rX   z+tuple[list[sympy.Symbol], list[sympy.Expr]]rY   )rZ   s   "r[   r\   rª   2  s   € ÷ (;ñ (;Øð(;à	4ñ(;r]   c           
     ó„  aaa	a
€ R R loVP                    Uu. uF,  p\        P                  P                  P	                  V4      NK.  	  ppV Uu. uF   qD'       g   K  VP
                  V J g   K  VNK"  	  ppVP                  V3R lR7       \        P                  P                  o. o	. o
VV	V
3R lpV F}  p\        P                  P                  P                  VP                  S4      '       g9   V! V P                  S\        VP                  S4      4      4       VP                  oV! V4       K  	  \        P                  P                  P                  V P                   S4      '       g-   V! V P                  S\        V P                   S4      4      4       . \#        S	4      O. \#        S
4      O3# u upi u upi )z,Figure out vars from this tree used in indexc               ó    € V ^8„  d   QhRRRR/# )rU   Úxrû   rX   ztuple[int, bool]rY   )rZ   s   "r[   r\   Ú8IterationRangesRoot.vars_and_sizes.<locals>.__annotate__7  s   € ÷ 
	:ñ 
	:Ð0ð 
	:Ð5Eñ 
	:r]   c                óä   € \         P                  P                  P                  V P                  4      p\         P                  P                  P                  V P
                  4      ^8H  pW'       * 3# )zò
Gets the key for sorting nodes. When two nodes have the
same divisor, the node with length as 1 should be handled
first so the current divisor is not changed after multiplied
node.length. Returns `not length_is_one_hint` for ascending
sort.
)r>   rý   rþ   Úoptimization_hintrg   rh   )r  Údivisor_hintÚlength_is_one_hints   &  r[   Úget_sort_keyÚ8IterationRangesRoot.vars_and_sizes.<locals>.get_sort_key7  sS   € ô Ÿ7™7×+Ñ+×=Ñ=¸a¿i¹iÓHˆLÜ!"§¡×!1Ñ!1×!CÑ!CÀAÇHÁHÓ!MÐQRÑ!RÐØ Ô"8Ð9Ð9r]   c                ó   <€ S! V 4      # r_   rY   )r  r!  s   &€r[   Ú<lambda>Ú4IterationRangesRoot.vars_and_sizes.<locals>.<lambda>E  s	   ø€ ¡¨a¤r]   ©Úkeyc                ó¢   <€ SP                  V P                  4       4       SP                  V P                  4       SV P                  ,          oR # r_   )r  r‹   rh   )rô   rg   Ú
index_varsÚsizess   &€€€r[   ÚaddÚ/IterationRangesRoot.vars_and_sizes.<locals>.addJ  s5   ø€ à×Ñ˜dŸk™k›mÔ,Ø�L‰L˜Ÿ™Ô%Ø §¡Õ+ŠGr]   )Úfree_symbolsr>   rs   r  Úgetru   Úsortr™   rš   r›   rý   rþ   rÿ   rg   r  r   rp   r  )r|   r¢   Úsr¬   Únr+  rô   rg   r!  r)  r*  s   &&     @@@@r[   Úvars_and_sizesÚ"IterationRangesRoot.vars_and_sizes2  sO  û€ õ

	:ð <A×;MÒ;MÓNÑ;M°a”—‘×*Ñ*×.Ñ.¨qÖ1Ñ;MˆÐNÙ!Ó:™E�q£Q”¨1¯6©6°T¨>—�™EˆÐ:Ø�
‰
Ô0ˆ
Ô1Ü—'‘'—+‘+ˆØˆ
Øˆ÷	,ó ˆDÜ—7‘7×#Ñ#×;Ñ;¸D¿L¹LÈ'×RÒRá�D—K‘K ¬°$·,±,ÀÓ)HÓIÔJØŸ,™,�Ù�ŽIñ ô �w‰w×Ñ×7Ñ7¸¿
¹
ÀG×LÒLá�—‘˜G¤X¨d¯j©j¸'Ó%BÓCÔDà&”˜*Ó%Ð&Ð(:¬(°5«/Ð(:Ð:Ð:ùò/ OùÚ:s   ™2F8Á	F=ÁF=Á1F=)r¨   r©   r¢   r¥   r¬   r£   r¦   r_   )r”   r•   r–   r—   r˜   r{   r³   r¾   rÂ   rÈ   rÍ   rÒ   rÚ   rß   rå   rí   ró   rø   r  r	  r  r  r2  r�   rž   rŸ   s   @r[   rv   rv   §   sq   ø† ñ÷*!ò *!õXHõXõ&õTõ
$õ,õõõõõõ9õ õ.4õ%õE÷(;ó (;r]   rv   c                  ó‚   a € ] tR tRtR V 3R lltR R ltR R ltR R	 ltR
 R ltR R lt	R R lt
R R ltRtV ;t# )rû   i]  c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )rU   rj   rk   rg   rq   rh   r  Úparentrf   rX   rw   rY   )rZ   s   "r[   r\   Ú!IterationRangesEntry.__annotate__^  sF   € ÷ ñ àðð ðð ð	ð
 ðð  ðð 
ñr]   c                	ó,  <€ \         SV `  VVP                  V,          VP                  VP                  VP
                  VVVP                  VP                  R 7	       WPn        \        P                  ! R4      ! V P                  4      V n        W@n        R# ))	rj   rp   rl   rn   rr   rg   rh   rs   ru   N)rz   r{   rp   rl   rn   rr   rs   ru   r6  Ú	functoolsÚ	lru_cacheÚ_codegenÚcodegenr  )r|   rj   rg   rh   r  r6  r}   s   &&&&&&€r[   r{   ÚIterationRangesEntry.__init__^  sx   ø€ ô 	‰ÑØØ—,‘, Õ'Ø—_‘_Ø×(Ñ(Ø—=‘=ØØØ—=‘=Ø—‘ð 	ô 
	
ð ŒÜ ×*Ò*¨4Ô0°·±Ó?ˆŒØŽ	r]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   r7  u  s   € ÷ sñ s˜#ñ sr]   c                	óŠ   € R V P                    RV P                   RV P                   RV P                   RV P                   R2# )zIterationRangesEntry(r²   Ú))rj   rg   rh   r  rn   rƒ   s   &r[   r³   ÚIterationRangesEntry.__repr__u  sH   € Ø& t§y¡y k°°D·L±L°>ÀÀDÇKÁKÀ=ÐPRÐSW×S\ÑS\ÐR]Ð]_Ð`d×`oÑ`oÐ_pÐpqÐrÐrr]   c               ó    € V ^8„  d   QhRRRR/# )rU   rj   rk   rX   rw   rY   )rZ   s   "r[   r\   r7  x  s   € ÷ ñ ˜Sð  Tñ r]   c                	óP   a€ V3R  lV n         R V P                   n        SV n        R# )c                 ó   <€ S # r_   rY   ©rj   s   €r[   r$  Ú/IterationRangesEntry.set_name.<locals>.<lambda>y  s   ø€ ™tr]   c                 ó   € R # r_   rY   rY   r]   r[   r$  rF  z  s   € ©4r]   N)r<  ró   rj   )r|   rj   s   &fr[   Úset_nameÚIterationRangesEntry.set_namex  s   ø€ Ü#ˆŒÙ#/ˆ�‰Ô ØˆŽ	r]   c               ó   € V ^8„  d   QhRR/# rð   rY   )rZ   s   "r[   r\   r7  }  s   € ÷ #ñ #˜Tñ #r]   c                	ó:   € V P                   P                  4        R # r_   )r<  ró   rƒ   s   &r[   ró   Ú IterationRangesEntry.cache_clear}  s   € Ø�‰× Ñ Ö"r]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   r7  €  s   € ÷ ñ ˜#ñ r]   c                	óX   € \         P                  P                  V 4       V P                  # r_   )r>   rs   Úcodegen_iteration_ranges_entryrj   rƒ   s   &r[   r;  ÚIterationRangesEntry._codegen€  s   € Ü	�‰×/Ñ/°Ô5Ø�y‰yÐr]   c               ó   € V ^8„  d   QhRR/# )rU   rX   r  rY   )rZ   s   "r[   r\   r7  „  s   € ÷  ñ  Ð"2ñ  r]   c                	ól  € . p\        V P                  \        P                  4      '       d   V# \        V P                  \        \
        34      '       g   \        \        V P                  4      4      hV P                  P                  R ,           F   p\        V\        P                  \        P                  34      '       d   K5  VP                  p\        V4      ^ 8”  g   KS  \        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   K�  VP                  V4       K¢  	  V# )ºé   NNc              3  óV   "  € T F  p\        V\        P                  4      x € K!  	  R # 5ir_   )r   r   ÚSIZE©Ú.0r0  s   & r[   Ú	<genexpr>Ú8IterationRangesEntry.precomputed_args.<locals>.<genexpr>Ž  s!   é € ð ,Ù:A°Q”N 1¤d§i¡i×0Ð0»'ùs   ‚')FT)Ú
isinstancer  r™   r¼   r   r   r­   ÚtypeÚargsÚIntegerr-  ÚlenÚallr  )r|   Úprecomputed_argsÚargÚsymbolss   &   r[   ra  Ú%IterationRangesEntry.precomputed_args„  sÝ   € à-/ÐÜ�d—i‘i¤§¡×.Ò.Ø#Ð#Ü˜$Ÿ)™)¤h´Ð%@×AÒAÜ ¤ d§i¡i£Ó1Ð1Ø—9‘9—>‘> "×%Ð%ˆCÜ˜c¤E§M¡M´5·<±<Ð#@×AÔAØ×*Ñ*�Ü�w“< !Ö#¯«ñ ,Ù:Aó,¯¯ªñ ,Ù:Aó,÷ )ô )ð %×+Ñ+¨CÖ0ñ &ð  Ðr]   c               ó   € V ^8„  d   QhRR/# ©rU   rX   rW   rY   )rZ   s   "r[   r\   r7  ”  s   € ÷ ñ ˜#ñ r]   c                	ó,   € \        V P                  4      # r_   )Úhashrj   rƒ   s   &r[   Ú__hash__ÚIterationRangesEntry.__hash__”  s   € Ü�D—I‘I‹Ðr]   c               ó    € V ^8„  d   QhRRRR/# )rU   ÚotherÚobjectrX   r�   rY   )rZ   s   "r[   r\   r7  —  s   € ÷ 'ñ '˜Fð ' tñ 'r]   c                	óŽ   € \        V\        4      '       g   \        R \        V4       24      hV P                  VP                  8H  # )z#expected IterationRangesEntry, got )r[  rû   r­   r\  rj   )r|   rl  s   &&r[   Ú__eq__ÚIterationRangesEntry.__eq__—  s:   € Ü˜%Ô!5×6Ò6Ü Ð#FÄtÈEÃ{ÀmÐ!TÓUÐUØ�y‰y˜EŸJ™JÑ&Ð&r]   )r<  r  rj   r6  )r”   r•   r–   r—   r{   r³   rH  ró   r;  ra  ri  ro  r�   rž   rŸ   s   @r[   rû   rû   ]  s6   ø† ÷ð õ.sõõ
#õõ õ ÷'ó 'r]   rû   c                  ó’   a € ] tR tRtRtRRRR/R V 3R llltR R	 ltR
 R ltR R ltR R lt	R R lt
R R ltR R ltRtV ;t# )ÚDerivedIterationRangesRooti�  aA  A root with reduced numel/block_size derived from a parent tree.

Used for the grouped reduction output: if the parent R tree covers 1024
columns with local_reduction_size=128, the derived root covers 8 groups. It shares
the parent's loop structure (same is_loop, prefix, loop variable) but
has its own block geometry:

    parent R:  numel=1024, block_size=RBLOCK, offset=roffset
    derived R: numel=8,    block_size=RBLOCK//128, offset=roffset//128

This lets the grouped reduction store at reduced-resolution offsets
while still running inside the parent's reduction loop.
Úname_suffixÚreducedrí   c               ó4   € V ^8„  d   QhRRRRRRRRRRR	R
RR/# )rU   r6  rv   rp   rq   r¾   rÈ   rs  rk   rí   rê   rX   rw   rY   )rZ   s   "r[   r\   Ú'DerivedIterationRangesRoot.__annotate__¬  sQ   € ÷ 0ñ 0à#ð0ð ð	0ð
 ð0ð !ð0ð ð0ð Kð0ð 
ñ0r]   c               	ó  <€ \         SV `  V R VP                   2VVP                  VP                  VP
                  VP                  VP                  VP                  VP                  VP                  R7
       W0n        W@n        W`n        R# )Ú_)
rj   rp   rr   r¢   rs   r£   r¥   r¦   r¨   r©   N)rz   r{   rj   rr   r¢   rs   r£   r¥   r¦   r¨   r©   Ú_block_sizeÚ_block_offsetÚ_named_constants)r|   r6  rp   r¾   rÈ   rs  rí   r}   s   &&$$$$$€r[   r{   Ú#DerivedIterationRangesRoot.__init__¬  s~   ø€ ô 	‰ÑØ�=  &§+¡+ Ð/ØØ—=‘=Ø—,‘,Ø—=‘=Ø×&Ñ&ð —N‘NØ×(Ñ(Ø—_‘_Ø—_‘_ð 	ô 	
ð" &ÔØ)ÔØ /Ör]   c               ó   € V ^8„  d   QhRR/# r¶   rY   )rZ   s   "r[   r\   rv  Ë  s   € ÷  ñ  ˜Jñ  r]   c                	ó   € V P                   # r_   )ry  rƒ   s   &r[   r¾   Ú%DerivedIterationRangesRoot.block_sizeË  s   € Ø×ÑÐr]   c               ó   € V ^8„  d   QhRR/# r¶   rY   )rZ   s   "r[   r\   rv  Î  s   € ÷ "ñ "˜jñ "r]   c                	ó   € V P                   # r_   )rz  rƒ   s   &r[   rÈ   Ú'DerivedIterationRangesRoot.block_offsetÎ  s   € Ø×!Ñ!Ð!r]   c               ó   € V ^8„  d   QhRR/# r‡   rY   )rZ   s   "r[   r\   rv  Ñ  s   € ÷ -ñ -˜<ñ -r]   c                	ó,   € \        V P                  4      # r_   rŠ   rƒ   s   &r[   rø   Ú$DerivedIterationRangesRoot.index_symÑ  r�   r]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   rv  Ô  s   € ÷ #ñ #˜3ñ #r]   c                	ó    € V P                    R 2# )Ú_maskrE  rƒ   s   &r[   rÍ   Ú$DerivedIterationRangesRoot.mask_nameÔ  s   € Ø—)‘)�˜EÐ"Ð"r]   c               ó   € V ^8„  d   QhRR/# r€   rY   )rZ   s   "r[   r\   rv  ×  s   € ÷ ñ ¨ñ r]   c                	ó   € R # rä   rY   rƒ   s   &r[   rß   Ú1DerivedIterationRangesRoot.supports_constant_mask×  s   € ñ r]   c               ó   € V ^8„  d   QhRR/# ré   rY   )rZ   s   "r[   r\   rv  Ü  s   € ÷ %ñ %Ð!Rñ %r]   c                	ó   € V P                   # r_   )r{  rƒ   s   &r[   rí   Ú*DerivedIterationRangesRoot.named_constantsÜ  s   € Ø×$Ñ$Ð$r]   c               ó   € V ^8„  d   QhRR/# r€   rY   )rZ   s   "r[   r\   rv  ß  s   € ÷ ñ ¨4ñ r]   c                	ó   € R # rÞ   rY   rƒ   s   &r[   rå   Ú4DerivedIterationRangesRoot.has_custom_codegen_headerß  rá   r]   )rz  ry  r{  rY   )r”   r•   r–   r—   r˜   r{   r¾   rÈ   rø   rÍ   rß   rí   rå   r�   rž   rŸ   s   @r[   rr  rr  �  sP   ø† ñð0ð %ð0ð NP÷0ò 0õ> õ"õ-õ#õõ
%÷ó r]   rr  c               ó    € V ^8„  d   QhRRRR/# )rU   Úvalueúint | floatrX   rk   rY   )rZ   s   "r[   r\   r\   ã  s   € ÷ ñ ˜ð ¨ñ r]   c                óœ   € V \        R 4      8X  d   R# V \        R4      8X  d   R# \        P                  ! V 4      '       d   R# \        V 4      # )Úinfzfloat("inf")z-infzfloat("-inf")zfloat("nan"))ÚfloatÚmathÚisnanÚrepr)r”  s   &r[   Úconstant_reprrœ  ã  s<   € Ø”�e“ÔÙØ	”%˜“-Ô	ÙÜ	�Š�E×	Ò	ÙÜ�‹;Ðr]   ÚCSEVariableType)ÚboundrV   c                  ó6   € ] tR tRt$ R]R&   R]R&   R]R&   RtR# )	ÚPartialAccumulateið  rk   Úbuffer_nameÚreduction_typer   r”  rY   N©r”   r•   r–   r—   Ú__annotations__r�   rY   r]   r[   r   r   ð  s   ‡ àÓØÓØ‡Jr]   r   c                  ób   € ] tR tRt$ RtR]R&   R]R&   R]R&   R	]R
&   R	]R&   R]R&   R]R&   RtR# )ÚNodeInfoi÷  z>
Pre-computed node information for combo kernel partitioning.
ÚlistÚnode_scheduleÚdictÚtilingzdict | NoneÚtiling_scoresr   rp   ÚrnumelrJ   Úfeaturesr�   Úis_persistent_reductionrY   N©r”   r•   r–   r—   r˜   r¤  r�   rY   r]   r[   r¦  r¦  ÷  s0   ‡ ñð ÓØƒLØÓØƒJØƒKØ Ó Ø!×!r]   r¦  c                  ót  a € ] tR tRt$ Rt]tR]R&   R]R&   RtR]R&   R	]R
&   RgR V 3R lllt	R R lt
R R ltR R ltR t]]R R l4       4       tR R ltR R lt]R R l4       tR R ltR R ltR  R! ltR" R# ltR$ R% ltR& R' ltR( R) ltR* R+ ltR, R- ltR. R/ ltR0 R1 ltR2 R3 ltR4 R5 lt ]!R6 R7 l4       t"]#]$PJ                  PL                  3R8 R9 ll4       t']#]$PJ                  PL                  3R: R; ll4       t(R< R= lt)]#R> R? l4       t*R@ RA lt+RB RC lt,RD RE lt-]!RF RG l4       t.RH RI lt/RJ RK lt0]1Pd                  RL RM l4       t3RN RO lt4RP RQ lt5RR RS lt6RT t7RhRU RV llt8]1Pd                  RW RX l4       t9RY RZ lt:]!R[ 4       t;R\ R] lt<R^ t=R_ t>R` t?Ra t@Rb tARc tBRd Re ltCRftDV ;tE# )irt   i  zg
Common base class for Triton/Halide codegen which both use flattened indexing rather than loop nests.
zCallable[[sympy.Expr], str]ÚsexprÚkexprFr�   Úallow_block_ptrrk   Úkernel_namec               ó8   € V ^8„  d   QhRRRRRRRRR	RR
RRRRR/# )rU   rª  údict[str, sympy.Expr]r­  rJ   r£   r¤   Úoverride_persistent_reductionzbool | NoneÚoverride_cooperative_reductionr«  údict[str, sympy.Expr] | NoneÚmix_order_reductionr�   rX   rw   rY   )rZ   s   "r[   r\   ÚSIMDKernel.__annotate__  sd   € ÷ D?ñ D?à%ðD?ð %ðD?ð )ð	D?ð
 (3ðD?ð )4ðD?ð 4ðD?ð "ðD?ð 
ñD?r]   c                	ó"  <a € Vf   / p\         SS `  4        VS n        VP                  4       S n        \        4       S n        \        4       S n        VP                  4        UU	u/ uF/  w  r‰V\        P                  P                  P                  V	4      bK1  	  up	pS n        . S n        / S n        \         P"                  ! 4       S n        VP'                  4       S n        Ve   TMS P+                  4       S n        VS n        VS n        Ve   TMS P3                  4       S n        VS n        S P9                  4       S n        R S n        \         P"                  ! 4       S n        RS n         \B        PD                  PF                  '       d”   S P                  PH                   Fy  p
\K        V
\L        PN                  4      '       g   K%  \K        V
PP                  \R        PT                  4      '       g   KQ  V
PP                  PW                  4       R8X  g   Kr  RS n          M	  \X        PZ                  R V 3R ll4       pVS n.        S P_                  V4       ^ S n0        R S n1        R S n2        . S n3        S P                  Pi                  4       S n5        R # u up	pi )NFÚdotTc               ó   € V ^8„  d   QhRR/# )rU   r¢   rq   rY   )rZ   s   "r[   r\   Ú)SIMDKernel.__init__.<locals>.__annotate__F  s   € ÷ 	>ñ 	> Zñ 	>r]   c                óÞ   <€ \         P                  P                  P                  V SP	                  4       4      p SP
                   F  pSP                  W4      p K  	  SP                  V 4      # r_   )r>   rý   rþ   Úsimplify_with_rangesrn   Úrange_treesÚcombine_contiguous_dimsÚcombine_modular_indexing_pairs)r¢   Útreer|   s   & €r[   Úsimplify_indexingÚ.SIMDKernel.__init__.<locals>.simplify_indexingE  sY   ø€ ä—G‘G×$Ñ$×9Ñ9¸%ÀÇÁÓARÓSˆEØ×(Ô(�Ø×4Ñ4°UÓA’ñ )ð ×6Ñ6°uÓ=Ð=r]   )6rz   r{   r­  Úget_mutationsÚ	mutationsr5   ÚbodyÚindexing_coder�   r>   rý   rþ   ÚsimplifyÚnumelsrÂ  r  Ú	itertoolsÚcountr  r„   Úinside_reductionÚ should_use_cooperative_reductionÚcooperative_reductionr«  rª  Úshould_use_persistent_reductionÚpersistent_reductionrº  Úwant_no_x_dimÚno_x_dimr   Ústore_output_ctrÚis_native_matmulr   rb   Únative_matmulr¨  r[  r   ÚSchedulerNoderô   r   ÚComputedBufferÚget_reduction_typer9  ÚcacherÆ  Úinitialize_range_treeÚrsplit_sizeÚ
min_xblockÚ
min_rblockÚsaved_partial_accumulateÚselect_index_dtypeÚ_index_dtype)r|   rª  r­  r£   r·  r¸  r«  rº  rr   Úvalrô   rÆ  r}   s   f&&&&&&&    €r[   r{   ÚSIMDKernel.__init__  s  ù€ ð ÒØˆIÜ‰ÑÔØ ˆŒØ!×/Ñ/Ó1ˆŒÜ"Ó$ˆŒ	Ü+Ó-ˆÔàFLÇlÁlÄnô
ÙFT±{°vˆF”A—G‘G×$Ñ$×-Ñ-¨cÓ2Ò2Ánò
ˆŒð 79ˆÔØJLˆÔÜ(ŸšÓ0ˆÔØ (× 5Ñ 5Ó 7ˆÔð .Ò9ñ +à×6Ñ6Ó8ð 	Ô"ð
 <IˆÔØ-3ˆŒð -Ò8ñ *à×5Ñ5Ó7ð 	Ô!ð
 *=ˆÔ Ø×*Ñ*Ó,ˆŒØ%)ˆŒä )§¢Ó 1ˆÔØ %ˆÔÜ�=‰=×&×&Ð&ØŸ™×3Ô3�ä˜t¤Y×%<Ñ%<×=Ô=Ü" 4§9¡9¬b×.?Ñ.?×@Ô@ØŸ	™	×4Ñ4Ó6¸%Ö?à,0�DÔ)Ùñ 4ô 
�‰÷	>ó 
ð	>ð "3ˆÔØ×"Ñ" 9Ô-àˆÔØ&*ˆŒØ&*ˆŒØACˆÔ%Ø ŸM™M×<Ñ<Ó>ˆÖùóg
s   Á$5Jc               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   r»  V  s   € ÷ "ñ "ð 
ñ"r]   c                ó   € \         h)u  Generate template source code with fused prologues and epilogues.

Subclasses override this to implement custom code generation.
The default implementation raises NotImplementedError â€” the actual
standard path lives in ``TritonTemplateKernel.codegen_template_body``.
©ÚNotImplementedError)r|   Ú
schedulingÚtemplate_nodeÚepilogue_nodesÚprologue_nodesÚbuf_name_to_prologue_groupÚprologue_preserves_zero_mask_fnÚrenders   &&&&&&&&r[   Úcodegen_template_bodyÚ SIMDKernel.codegen_template_bodyV  s
   € ô  "Ð!r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   ú	list[Any]rY   )rZ   s   "r[   r\   r»  h  s   € ÷ 
ñ 
 yñ 
r]   c                ó   € . # )a{  Return epilogue nodes that were not fused into the kernel.

These nodes need separate codegen (via ``call_kernel``) and must
be excluded from ``mark_run`` in ``_codegen_single_template``.

The standard path fuses all epilogues, so this returns ``[]``.
``ExternalTritonTemplateKernel`` overrides this for epilogues that
don't read exactly one template output and cannot be fused.
rY   rƒ   s   &r[   Úget_unfused_epiloguesÚ SIMDKernel.get_unfused_epiloguesh  s	   € ð ˆ	r]   c               ó    € V ^8„  d   QhRRRR/# )rU   ÚirW   rX   rk   rY   )rZ   s   "r[   r\   r»  t  s   € ÷ %ñ %°ð %¸ñ %r]   c                	ó   € R V R2# )z<STORE_OUTPUT_Ú>rY   )r|   rú  s   &&r[   Ú_get_store_output_subgraph_nameÚ*SIMDKernel._get_store_output_subgraph_namet  s   € Ø ˜s !Ð$Ð$r]   c                	óx   € \        V P                  4      p\        P                  ! V^,
          ^R7      V n        V# )rT  )ÚstartÚstep)r   r×  rÎ  rÏ  )r|   Útotals   & r[   Úget_store_output_countÚ!SIMDKernel.get_store_output_countw  s.   € Ü�T×*Ñ*Ó+ˆÜ )§¢°e¸aµiÀaÔ HˆÔØˆr]   c               ó   € V ^8„  d   QhRR/# rf  rY   )rZ   s   "r[   r\   r»  ~  s   € ÷ Jñ J Cñ Jr]   c                	ó:   € \        R  V P                   4       4      # )c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5ir_   )r7   )rX  rr   s   & r[   rY  Ú0SIMDKernel.num_reduction_dims.<locals>.<genexpr>  s   é € ÐI¹[°6Ô& v×.Ð.»[ùó   ‚)ÚsumrÍ  rƒ   s   &r[   Únum_reduction_dimsÚSIMDKernel.num_reduction_dims|  s   € ô ÑI¸T¿[º[ÓIÓIÐIr]   c               ó    € V ^8„  d   QhRRRR/# )rU   Údtypeútorch.dtyperX   rk   rY   )rZ   s   "r[   r\   r»  �  s   € ÷ "ñ " +ð "°#ñ "r]   c                	ó   € \         hr_   ré  )r|   r  s   &&r[   Údtype_to_strÚSIMDKernel.dtype_to_str�  ó   € Ü!Ð!r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   r  rY   )rZ   s   "r[   r\   r»  „  s   € ÷ !ñ !°ñ !r]   c                	ó   € V P                   # r_   )rä  rƒ   s   &r[   Úget_index_dtype_as_torch_dtypeÚ)SIMDKernel.get_index_dtype_as_torch_dtype„  s   € Ø× Ñ Ð r]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   r»  ˆ  s   € ÷ Hñ H˜Sñ Hr]   c                	ó@   € V P                  V P                  4       4      # r_   )r  r  rƒ   s   &r[   Úindex_dtypeÚSIMDKernel.index_dtype‡  s   € à× Ñ  ×!DÑ!DÓ!FÓGÐGr]   c               ó   € V ^8„  d   QhRR/# r€   rY   )rZ   s   "r[   r\   r»  ‹  s   € ÷ ñ ˜tñ r]   c                	ó   € R # rä   rY   rƒ   s   &r[   rÕ  ÚSIMDKernel.want_no_x_dim‹  rç   r]   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )rU   r£   r¤   rÐ  r�   r„   rÍ  r¶  rÖ  rX   úlist[IterationRangesRoot]rY   )rZ   s   "r[   r\   r»  Ž  sF   € ÷ 5ñ 5à(ð5ð ð5ð ð	5ð
 &ð5ð ð5ð 
#ñ5r]   c                	ó<  a€ \        V3R  l\         4       4      pV'       * ;'       g    V'       * pR R lp. ROp	\        \        V	4      4      p
RR.pV'       d   TpMV'       d   T
pMW«,           pV! WÆ4      pV! V	\        4      p. p\	        V4       F…  w  pp\        V4      pVP                  V4      pVP                  V4      pVf   TMTpTP                  \        V R2SV,          TTT TT;'       d    V P                  '       * VVRS9   R7
      4       K‡  	  V# )	c              3  ó8   <"  € T F  qS9   g   K  Vx € K  	  R # 5ir_   rY   )rX  rr   rÍ  s   & €r[   rY  Ú3SIMDKernel.construct_range_trees.<locals>.<genexpr>–  s   øé € ð %
Ù!-�v¸6Ñ1A�FŠF£ùó   ƒ�
c               ó   € V ^8„  d   QhRR/# )rU   rX   zdict[Any, int]rY   )rZ   s   "r[   r\   Ú6SIMDKernel.construct_range_trees.<locals>.__annotate__›  s   € ÷ 	ñ 	¨^ñ 	r]   c                ób   a€ \        V3R  lV  4       4       UUu/ uF  w  r#W2bK	  	  upp# u uppi )c              3  ó8   <"  € T F  qS9   g   K  Vx € K  	  R # 5ir_   rY   )rX  rå  rÌ   s   & €r[   rY  ÚOSIMDKernel.construct_range_trees.<locals>.filtered_index_map.<locals>.<genexpr>�  s   øé € Ð2UÁ#¸3ÐPTÉ·3²3Ã#ùr$  )Ú	enumerate)ÚseqrÌ   Úidxrå  s   &f  r[   Úfiltered_index_mapÚ<SIMDKernel.construct_range_trees.<locals>.filtered_index_map›  s4   ø€ ä)2Ô2UÁ#Ó2UÔ)UôÙ)U™X˜S�’Ñ)Uòð ùó s   š+ÚzÚr0_Úr1_r¢   )r£   r¥   r¦   r¨   r©   )r  Úyr/  )
r   Úall_prefixesr§  r  r*  r7   r.  r  rv   rÔ  )r|   r£   rÐ  r„   rÍ  rÖ  Úactive_prefixesÚno_r_dimr-  Ú	grid_dimsÚpointwise_tensor_dimsÚreduction_dimsÚtensor_dimsÚtensor_dim_mapÚgrid_dim_maprÂ  rú  rr   r¦   r¨   r¢   s   &&&&f&               r[   Úconstruct_range_treesÚ SIMDKernel.construct_range_treesŽ  s0  ø€ ô %ô %
Ý!-ó%
ó 
ˆð (Ô'×;Ð;¨|Ô+;ˆõ	ò
 $ˆ	Ü $¤X¨iÓ%8Ó 9ÐØ ˜ˆßØ(‰KßØ/‰Kà/Õ@ˆKñ ,¨KÓIˆÙ)¨)´\ÓBˆàˆÜ" ?Ö3‰IˆAˆvÜ.¨vÓ6ˆLØ'×+Ñ+¨FÓ3ˆJØ#×'Ñ'¨Ó/ˆHØ!Ò)‘A¨xˆEØ×ÑÜ#Ø�h˜eÐ$Ø˜6•NØØØØ'Ø(×JÐJ°×1JÑ1JÔ-JØ)Ø%Ø  F™]ôöñ 4ð& Ðr]   c               ó    € V ^8„  d   QhRRRR/# )rU   r£   zdict[str, str]rX   rw   rY   )rZ   s   "r[   r\   r»  Å  s   € ÷ -ñ -¨~ð -À$ñ -r]   c                	óÒ   € V P                  VV P                  V P                  P                  4       V P                  V P
                  4      pV P                  P                  V4       R # r_   )r<  rÐ  r­  r„   rÍ  rÖ  rÂ  Úextend)r|   r£   rÂ  s   && r[   rÞ  Ú SIMDKernel.initialize_range_treeÅ  sR   € Ø×0Ñ0ØØ×!Ñ!Ø�M‰M×&Ñ&Ó(Ø�K‰KØ�M‰Mó
ˆð 	×Ñ×Ñ Ö,r]   c               ó    € V ^8„  d   QhRRRR/# )rU   ÚindicesúSequence[sympy.Expr]rX   rw   rY   )rZ   s   "r[   r\   r»  Ï  s   € ÷ ñ Ð)=ð À$ñ r]   c                ó   € R# )zZ
Hook called right before codegen with every index that will be
used in the fused kernel.
NrY   )r|   rC  s   &&r[   Úfinalize_indexingÚSIMDKernel.finalize_indexingÏ  s   ‚ r]   c               ó(   € V ^8„  d   QhRRRRRRRR/# ©	rU   rj   rk   r¢   rq   r”  r@   rX   rw   rY   )rZ   s   "r[   r\   r»  Õ  s)   € ÷ *ñ * Cð *°
ð *À;ð *ÐSWñ *r]   c                	óp   € V P                   pR V n          V P                  WV4      W@n         #   Y@n         i ; irä   )rÐ  Ústore)r|   rj   r¢   r”  Úpriors   &&&& r[   Ústore_reductionÚSIMDKernel.store_reductionÕ  s5   € Ø×%Ñ%ˆØ %ˆÔð	*Ø—:‘:˜d¨5Ó1à$)Õ!ø EÕ!ús   •- ­5c               ó   € V ^8„  d   QhRR/# r€   rY   )rZ   s   "r[   r\   r»  Ý  s   € ÷ ñ °$ñ r]   c                	ó   € R # rä   rY   rƒ   s   &r[   rÑ  Ú+SIMDKernel.should_use_cooperative_reductionÝ  rç   r]   c               ó   € V ^8„  d   QhRR/# r€   rY   )rZ   s   "r[   r\   r»  à  s   € ÷ ñ °ñ r]   c                	ó   € R # rä   rY   rƒ   s   &r[   rÓ  Ú*SIMDKernel.should_use_persistent_reductionà  rç   r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   ro   rY   )rZ   s   "r[   r\   r»  ã  s   € ÷ 
ñ 
Ð:ñ 
r]   c                	ót   € \        \        P                  P                  R  V P                   4       4      4      # )c              3  óT   "  € T F  qP                   P                  4       x € K   	  R # 5ir_   )rn   r�   ©rX  rÅ  s   & r[   rY  Ú(SIMDKernel.var_ranges.<locals>.<genexpr>å  s"   é € ð *Ù4D¨D—‘×%Ñ%×'Ð'Ó4Dùs   ‚&()r©  rÎ  ÚchainÚfrom_iterablerÂ  rƒ   s   &r[   rn   ÚSIMDKernel.var_rangesã  s4   € ÜÜ�O‰O×)Ñ)ñ *Ø48×4DÒ4Dó*ó ó
ð 	
r]   c               ó    € V ^8„  d   QhRRRR/# ©rU   r¢   rq   rX   rY   )rZ   s   "r[   r\   r»  ê  s   € ÷ 
ñ 
°Jð 
À:ñ 
r]   c                	ó²  € \        V\        4      '       g   V# VP                  ^ ,          pV P                  P	                  V4      ;pf   V# \        WVP                  /4      p\        P                  P                  P                  V4      p\        VVP                  P                  4       VP                  P                  4       P                  4       /4      # ©r   )r[  r   r]  r  r.  r:   r  r>   rý   rþ   rÄ  ru   rø   r	  r‹   )r|   r¢   r  Ú	tree_nodeÚ	new_indexs   &&   r[   rÄ  Ú)SIMDKernel.combine_modular_indexing_pairsê  sª   € Ü˜%¤×1Ò1ØˆLØ�J‰J�q�MˆØ×.Ñ.×2Ñ2°1Ó5Ð5ˆIÒ>ØˆLÜ˜u¨)¯.©.Ð&9Ó:ˆ	Ü—G‘G×$Ñ$×CÑCÀIÓNˆ	äØØ�^‰^×%Ñ%Ó'¨¯©×)BÑ)BÓ)D×)KÑ)KÓ)MÐNó
ð 	
r]   c               ó$   € V ^8„  d   QhRRRRRR/# ©rU   r¢   rq   rÅ  rv   rX   rY   )rZ   s   "r[   r\   r»  ø  s$   € ÷ >ñ >Øð>Ø':ð>à	ñ>r]   c                	óf  € \        V\        P                  4      '       d0   VP                  \        4      '       d   VP                  V4      w  r4MR p\        P                  P                  P                  W4      ;p'       d    Vw  rg\	        V P                  Wb4      V4      # V P                  W4      # r_   )r[  r™   ÚAddÚhasr   r2  r>   rý   rþ   Úexpand_floor_divÚ_combine_contiguous_dims)r|   r¢   rÅ  Úcandidate_varsrx  Ú
expand_resrb  Údenominators   &&&     r[   rÃ  Ú"SIMDKernel.combine_contiguous_dimsø  sŽ   € ô �eœUŸY™Y×'Ò'¨E¯I©I´h×,?Ò,?Ø $× 3Ñ 3°EÓ :ÑˆN˜Aà!ˆNÜŸ™×)Ñ)×:Ñ:¸5ÓQÐQˆ:ÖQØ%/Ñ"ˆIÜ˜D×9Ñ9¸)ÓJÈKÓXÐXà×0Ñ0°Ó=Ð=r]   c               ó$   € V ^8„  d   QhRRRRRR/# re  rY   )rZ   s   "r[   r\   r»    s$   € ÷ ñ ØðØ':ðà	ñr]   c                óž  € \        V\        P                  \        P                  34      '       d   V# VP	                  V4      w  r4\        V4      ^8:  d   V# \        P                  P                  P                  W4\        V.W44      4      w  rVpWT8X  d   V# VP                  V4      p\        V\        \        W6! V4      4      4      4      p	V	# )z9
More aggressive simplification to merge contiguous dims
)r[  r™   r^  r¼   r2  r_  r>   rý   rþ   Ú_simplify_loopsrA   r  r:   r©  Úzip)
r|   r¢   rÅ  r)  r*  Ú	new_sizesÚreindexÚ_pruneÚnew_index_varsrb  s
   &&&       r[   rj  Ú#SIMDKernel._combine_contiguous_dims  s²   € ô �eœeŸm™m¬U¯\©\Ð:×;Ò;ØˆLØ ×/Ñ/°Ó6Ñˆ
Üˆu‹:˜Œ?ØˆLÜ%&§W¡W×%5Ñ%5×%EÑ%EØÔ7¸¸ÀÓSó&
Ñ"ˆ	˜Fð ÔØˆLØŸ™¨	Ó2ˆÜ˜u¤d¬3¨z¸7À>Ó;RÓ+SÓ&TÓUˆ	ØÐr]   c               ó   € V ^8„  d   QhRR/# )rU   rX   z'contextlib.AbstractContextManager[None]rY   )rZ   s   "r[   r\   r»    s   € ÷ ñ Ð#Jñ r]   c                	óª   a a€ S P                   R,          P                  ;'       g    S P                  o\        P                  V V3R l4       pV! 4       # )rT  c               3  ó4  <"  € S P                   P                  4       '       g$   S P                  '       d   \        R 4      hRx € R# S'       d   S P	                  4        RS n         Rx € S'       d   S P	                  4        RS n        R#   RS n        i ; i5i)z"expected not self.inside_reductionNFT)r­  r„   rÐ  r­   Úcodegen_body)r|   Úshould_flushs   €€r[   ÚctxÚ)SIMDKernel.disable_reduction.<locals>.ctx  sw   øé € à—=‘=×-Ñ-×/Ò/Ø×(×(Ð(Ü(Ð)MÓNÐNÛÙßð ×!Ñ!Ô#Ø$)ˆDÔ!ð-Ûßà×%Ñ%Ô'à(,�Ö%ø¨�Õ%üs.   ƒ2B¶BÁBÁ'B Á3B Â	BÂ	BÂBéÿÿÿÿ)rÂ  r¥   rÒ  Ú
contextlibÚcontextmanager)r|   r}  r|  s   f @r[   Údisable_reductionÚSIMDKernel.disable_reduction  sH   ù€ Ø×'Ñ'¨Õ+×3Ñ3×QÐQ°t×7QÑ7Qˆä	×	"Ñ	"õ	-ó 
#ð	-ñ& ‹uˆr]   c               ó    € V ^8„  d   QhRRRR/# )rU   r  rq   rX   rm   rY   )rZ   s   "r[   r\   r»  2  s   € ÷ 	
ñ 	
 :ð 	
Ð2Dñ 	
r]   c                	ó  € \        V4      \        V P                  4      8w  d.   \        R \        V4       R\        V P                  4       24      h\        WP                  4       UUu. uF  w  r#VP	                  V4      NK  	  upp# u uppi )z4expected len(lengths) == len(self.range_trees), got ú and )r_  rÂ  r­   rr  r  )r|   r  rh   Úrangess   &*  r[   Ú
set_rangesÚSIMDKernel.set_ranges2  s‹   € Üˆw‹<œ3˜t×/Ñ/Ó0Ô0Ü ðÜ˜7“|�n E¬#¨d×.>Ñ.>Ó*?Ð)@ðBóð ô #& g×/?Ñ/?Ô"@ô
á"@‘�ð ×Ñ˜VÖ$Ù"@ò
ð 	
ùó 
s   Á)B	c               ó$   € V ^8„  d   QhRRRRRR/# )rU   ÚgroupsúIterable[sympy.Expr]r  úSequence[Sequence[sympy.Expr]]rX   zStuple[list[list[sympy.Expr]], list[list[Callable[[list[sympy.Expr]], sympy.Expr]]]]rY   )rZ   s   "r[   r\   r»  >  s+   € ÷ R1ñ R1Ø$ðR1Ø/MðR1ð
ñR1r]   c                	óØ  aaaa€ \         ;QJ d    R  V 4       F  '       d   K   RM	  RM! R  V 4       4      '       d   V  Uu. uF  p. NK  	  up. 3# \        P                  P                  oV  Uu. uF  p. NK  	  upoV  Uu. uF  pSP	                  V4      NK  	  upo\
        P                  ! 4       oR VVVV3R llpR R lp. p^ pV EFÁ  p	. p
V	 EF£  pSP                  V^4      '       d   V
P                  R 4       K0  V\        S4      8  d+   SP                  SV,          ^4      '       d   V^,          pK:  V^,           \        S4      8  dý   SP                  VSV,          SV^,           ,          ,          4      '       dÉ   SP                  VSV,          SV^,           ,          ,          4      '       g)   \        VSV,          SV^,           ,          ,          4      hSV,          pSV^,           ,          p\        W¼V,          4      pV
P                  V! WÞ.V! WŒ4      V! V^,           V4      V! V^,           V4      .4      4       EK}  V^,           \        S4      8  dÉ   SP                  VSV,          4      '       g*   SP                  \        VSV,          4      ^4      '       d�   SP                  VSV,          4      '       g   \        VSV,          4      hSV,          p\        VSV,          4      pV
P                  V! V.V! WŒ4      V! V^,           V4      .4      4       EK\  V\        S4      8¼  d   \        V^ 4      hV
P                  \        P                  ! V! W‹4      4      4       EK¦  	  VP                  V
4       EKÄ  	  \         ;QJ d    R S 4       F  '       d   K   RM	  RM! R S 4       4      '       g   \        SV4      hSV3# u upi u upi u upi )	c              3  ó>   "  € T F  p\        V4      ^ 8H  x € K  	  R# 5i©r   N)r_  )rX  rh   s   & r[   rY  Ú5SIMDKernel._split_iteration_ranges.<locals>.<genexpr>D  s   é € Ð6©g FŒs�6‹{˜aÖ«gùs   ‚FTc               ó$   € V ^8„  d   QhRRRRRR/# )rU   rú  rW   r  rq   rX   rY   )rZ   s   "r[   r\   Ú8SIMDKernel._split_iteration_ranges.<locals>.__annotate__L  s!   € ÷ 	#ñ 	#˜ð 	# Jð 	#°3ñ 	#r]   c                óü   <€ SP                  V4      pSP                  SV ,          V4      '       g   \        SV ,          V4      h\        SV ,          V4      SV &   SV ,          P	                  V4       \        S4      # r_   )rÌ  Ústatically_known_multiple_ofÚ	CantSplitr   r  r   )rú  r  Ú
new_rangesÚ	remainingÚsvÚ	var_counts   &&€€€€r[   Ú	add_rangeÚ5SIMDKernel._split_iteration_ranges.<locals>.add_rangeL  si   ø€ Ø—;‘;˜tÓ$ˆDØ×2Ñ2°9¸Qµ<À×FÒFÜ 	¨!¥¨dÓ3Ð3ä# I¨a¥L°$Ó7ˆI�a‰LØ�q�M× Ñ  Ô&Ü˜	“?Ð"r]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   r*  r  Úidxsú	list[int]rX   z(Callable[[list[sympy.Expr]], sympy.Expr]rY   )rZ   s   "r[   r\   r“  U  s$   € ÷ 	ñ 	Ø#ð	Ø+4ð	à5ñ	r]   c                ó²   a a€ \        S4      \        S 4      ^,           8w  d+   \        R\        S4       R\        S 4      ^,            24      hR VV 3R llpV# )z`
Builds the nested expression:
  ((...((s1*v[i1] + v[i2]) * s2 + v[i3]) ... ) * sk + v[i(k+1)])
z*expected len(idxs) == len(sizes) + 1, got r†  c               ó    € V ^8„  d   QhRRRR/# )rU   Ú	flat_varsr  rX   rq   rY   )rZ   s   "r[   r\   ÚOSIMDKernel._split_iteration_ranges.<locals>.make_combined.<locals>.__annotate__b  s   € ÷ ñ Ð"2ð °zñ r]   c                óŠ   <€ V S^ ,          ,          p\        SSR,          4       F  w  r#W!,          W,          ,           pK  	  V# )r   rS  )rr  )r¢  r  r0  r,  rž  r*  s   &   €€r[   ÚgetterÚISIMDKernel._split_iteration_ranges.<locals>.make_combined.<locals>.getterb  s;   ø€ Ø   a¥Õ)�Ü! %¨¨b­Ö2‘F�AØ�8 i¥nÕ4’Dñ 3à�r]   )r_  r­   )r*  rž  r¥  s   ff r[   Úmake_combinedÚ9SIMDKernel._split_iteration_ranges.<locals>.make_combinedU  sY   ù€ ô �4‹yœC ›J¨�NÔ*Ü$ðÜ˜t›9˜+ U¬3¨u«:¸­>Ð*:ð<óð ÷
ñ ð ˆMr]   c                ó6   € \         P                  P                  # r_   )r™   rš   ÚZero)rx  s   &r[   r$  Ú4SIMDKernel._split_iteration_ranges.<locals>.<lambda>p  s   € ´E·G±G·L²Lr]   c              3  óz   "  € T F1  p\         P                  P                  P                  V4      ^8H  x € K3  	  R# 5i)rT  N)r>   rý   rþ   Úguarding_hint_or_throwrW  s   & r[   rY  r‘  Æ  s*   é € ÐVÉIÀq”1—7‘7×#Ñ#×:Ñ:¸1Ó=ÀÖBËIùs   ‚9;)r`  r>   rý   rþ   rÌ  rÎ  rÏ  rÿ   r  r_  Ústatically_known_gtr•  r–  r   ÚoperatorÚ
itemgetter)r‹  r  Úgrouprx  Úgr›  r§  Úreturn_getters_groupsÚcurrent_groupÚlength_groupÚreturn_gettersÚsizeÚsize1Úsize2Úsize3r—  r˜  r™  rš  s   &&             @@@@r[   Ú_split_iteration_rangesÚ"SIMDKernel._split_iteration_ranges=  s�  û€ ÷ ‹3Ñ6©gÓ6�3�3Š3Ñ6©gÓ6×6Ò6Ù$*Ó+¡F˜5“B¡FÑ+¨RÐ/Ð/ä�W‰W×ÑˆÙ:@Ó-A¹&°Q«b¹&Ñ-Aˆ
Ù-3Ó4©V¨�R—[‘[ –^©VÑ4ˆ	Ü—O’OÓ%ˆ	÷	#ó 	#õ	ð* !#ÐØˆÜ#ˆLØˆNÜ$�Ø×-Ñ-¨d°A×6Ò6Ø"×)Ñ)Ñ*@ÔAÙà#¤c¨)£nÔ4¸×9SÑ9SØ˜mÕ,Ø÷:ò :ð
 " QÕ&’Mð ! 1Õ$¤s¨9£~Ô5¸"×:PÑ:PØ˜) MÕ2°Y¸}ÈqÕ?PÕ5QÕQ÷;ò ;ð ×:Ñ:Ø˜i¨Õ6¸À=ÐSTÕCTÕ9UÕU÷ò ô (Ø Ø% mÕ4°yÀÐQRÕARÕ7SÕSóð ð
 & mÕ4�EØ% m°aÕ&7Õ8�EÜ$ T°5­=Ó9�EØ"×)Ñ)á%Ø"˜Ná )¨-Ó ?Ù )¨-¸!Õ*;¸UÓ CÙ )¨-¸!Õ*;¸UÓ Cðó÷
ð # QÕ&¬¨Y«Ô7Ø×*Ñ*¨4°¸=Õ1I×JÒJð ×*Ñ*¬8°D¸)ÀMÕ:RÓ+SÐUV×WÒWð ×:Ñ:Ø˜i¨Õ6÷ò ô (¨¨i¸Õ.FÓGÐGà% mÕ4�EÜ$ T¨9°]Õ+CÓD�EØ"×)Ñ)á%Ø"˜Gá )¨-Ó ?Ù )¨-¸!Õ*;¸UÓ Cðó÷	ð %¬¨I«Ô6Ü'¨¨aÓ0Ð0Ø"×)Ñ)ä ×+Ò+©I°mÓ,JÓK÷ñe %ðl "×(Ñ(¨×8ñq $÷t ‹sÑVÉIÓV�s�sŠsÑVÉIÓV×VÒVô ˜I wÓ/Ð/àÐ0Ð0Ð0ùòW ,ùò .BùÚ4s   ÁOÁ1O"ÂO'c               ó(   € V ^8„  d   QhRRRRRRRR/# )rU   r‹  rŒ  r  r�  Úreduction_numelrq   rX   rY   )rZ   s   "r[   r\   r»  Ó  s2   € ÷ ñ à$ðð 0ðð $ð	ð
 
(ñr]   c                óP  € \         P                  P                  p\        V^,          4      ^ 8X  du   VP	                  V\
        P                  P                  4      '       gE   VP	                  \        V4      \        V^ ,          4      V,          4      '       d   V^ ,          V.3# V# )z1Fill in the reduction numel of lengths if missing)	r>   rý   rþ   r_  rÿ   r™   rš   r›   r9   )Úclsr‹  r  r¾  rþ   s   &&&& r[   Úprepare_split_iteration_lengthsÚ*SIMDKernel.prepare_split_iteration_lengthsÒ  sƒ   € ô —7‘7×#Ñ#ˆÜˆw�q�z‹?˜aÔØ×0Ñ0°Ä%Ç'Á'Ç+Á+×NÒNØ×0Ñ0Ü˜fÓ%Ü˜g a�jÓ)¨OÕ;÷ò ð
 ˜A•J Ð 1Ð2Ð2àˆr]   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rU   r‹  rŒ  r  r�  r¾  rq   rX   r�   rY   )rZ   s   "r[   r\   r»  ç  s2   € ÷ ñ à$ðð 0ðð $ð	ð
 
ñr]   c                	ór   € V P                  WV4      p V P                  W4       R #   \         d     R# i ; i©TF)rÁ  r»  r–  )rÀ  r‹  r  r¾  s   &&&&r[   Úis_compatibleÚSIMDKernel.is_compatibleæ  s>   € ð ×5Ñ5°fÀÓWˆð	Ø×'Ñ'¨Ô8ÙøÜô 	Úð	ús   ”' §6µ6c               ó    € V ^8„  d   QhRRRR/# )rU   r  r�  rX   úlist[list[sympy.Expr]]rY   )rZ   s   "r[   r\   r»  õ  s    € ÷  Vñ  VØ5ð Và	ñ Vr]   c                óZ  € V P                    Uu/ uF  q"P                  VP                  bK  	  ppV P                  '       g9   V F2  p\	        V4      '       g   K  \
        P                  P                  W4&   K4  	  . VP                  4       OpV P                  WQV P                  4      # u upi )aå  
Split and set iteration ranges for the kernel based on the provided lengths.

This method maps the kernel's tiling structure to the node's iteration space,
handling both pointwise and reduction dimensions appropriately.

Args:
    lengths: A sequence of sequences of symbolic expressions representing
            the sizes of different dimensions for each node.

Returns:
    A list of lists of symbolic expressions representing the mapped
    iteration variables for each dimension.
)rÂ  rr   rp   rÐ  r7   r™   rš   r›   rò   Úmap_kernel_groups_to_node_sizesrˆ  )r|   r  Úrtrª  rr   r‹  s   &&    r[   Úsplit_and_set_rangesÚSIMDKernel.split_and_set_rangesõ  sŽ   € ð$ 15×0@Ò0@ÓAÑ0@¨"—)‘)˜RŸX™XÒ%Ñ0@ˆÐAð ×$×$Ð$Û �Ü& v×.Ô.Ü%*§W¡W§[¡[�F“Nñ !ð
 $�6—=‘=“?Ð#ˆð ×3Ñ3°FÀTÇ_Á_ÓUÐUùò Bs   �B(c               ó$   € V ^8„  d   QhRRRRRR/# )rU   r‹  rD  r  r�  rX   rÉ  rY   )rZ   s   "r[   r\   r»    s-   € ÷ Oñ Oà$ðOð 0ðOð
 
 ñOr]   c           
     ó®  € \        V4      \        V4      8X  dS   \        ;QJ d%    R \        W!4       4       F  '       d   K   RM	  RM! R \        W!4       4       4      '       d   V! V!  # V P                  W4      w  rE. \        P
                  P                  V! V!  4      OpV UUu. uF  qw Uu. uF
  qˆ! V4      NK  	  upNK  	  upp# u upi u uppi )aY  
We may want to fuse `for i0 in s0*s1` into a tiled kernel with groups (s0, s1).

To do this we need to split up the iteration space of i0 into something like:
    for i1 in s0:
      for i2 in s1:
        i0 = i1*s1 + i2
        ....

This function matches and resplits lengths to the groups of
this kernel to enable tiled + non-tiled fusions.
c              3  óž   "  € T FC  w  r\         P                  P                  P                  \	        V4      V,
          4      ^ 8H  x € KE  	  R# 5ir�  ©r>   rý   rþ   rÌ  r9   )rX  r  r²  s   &  r[   rY  Ú=SIMDKernel.map_kernel_groups_to_node_sizes.<locals>.<genexpr>*  s=   é € ð /
á,‘�ô �G‰G×Ñ×%Ñ%¤m°AÓ&6¸Õ&:Ó;¸qÖ@Û,ùs   ‚AAFT)r_  r`  rr  r»  rÎ  rZ  r[  )	rÀ  r‹  r  rˆ  r—  r³  r  ÚfnsÚfns	   &&&&     r[   rË  Ú*SIMDKernel.map_kernel_groups_to_node_sizes  sº   € ô& ˆw‹<œ3˜v›;Ô&¯3«3ñ /
ä˜GÔ,ó/
¯3¯3ª3ñ /
ä˜GÔ,ó/
÷ ,
ò ,
ñ ˜wÑ'Ð'à,/×,GÑ,GÈÓ,XÑ)ˆ
ØL”Y—_‘_×2Ñ2±:¸zÑ3JÓKÐLˆÙ8MÔNÑ8M°¨Ó,© "��H–©Ô,Ñ8MÒNÐNùÒ,ùÓNs   Â)CÂ1CÃCÃCc               ó    € V ^8„  d   QhRRRR/# ©rU   r¢   rq   rX   r�   rY   )rZ   s   "r[   r\   r»  4  s   € ÷ 4ñ 4¨*ð 4¸ñ 4r]   c                	ó6   € \        V\        P                  4      # r_   )r   r   ÚTMP©r|   r¢   s   &&r[   Úis_indirect_indexingÚSIMDKernel.is_indirect_indexing4  s   € ä" 5¬$¯(©(Ó3Ð3r]   c               ó    € V ^8„  d   QhRRRR/# rØ  rY   )rZ   s   "r[   r\   r»  8  s   € ÷ &
ñ &
 Jð &
°4ñ &
r]   c                	óî  aa€ V P                  V4      '       d   R # V P                  4       p^.\        V4      ,          pVP                   F¨  pV P                  P                  V4      oSf   K$  \        SP                  \        4      '       g"   \        R\        SP                  4       24      h\        V3R l\        V4       4       R4      pVf   KŠ  W5;;,          SP                  ,          uu&   Kª  	  \        P                  P                   P"                  o\$        ;QJ d0    V3R l\'        VR V 4       4       4       F  '       g   K   R# 	  R # ! V3R l\'        VR V 4       4       4       4      # )FNz5expected entry.parent to be IterationRangesRoot, got c              3  óP   <"  € T F  w  rVSP                   J g   K  Vx € K  	  R # 5ir_   )r6  )rX  rú  rÅ  Úentrys   &  €r[   rY  Ú,SIMDKernel.is_broadcasted.<locals>.<genexpr>L  s"   øé € ÐRÑ"9‘w�q¸TÀUÇ\Á\Ð=Q—’Ó"9ùs   ƒ&œ
&c              3  óJ   <"  € T F  w  rS! V4      S! V4      8g  x € K  	  R # 5ir_   rY   )rX  Ú	idx_rangeÚ
iter_rangerÌ  s   &  €r[   rY  râ  Y  s.   øé € ð 
ñ*Ñ%�	ñ �YÓ¡8¨JÓ#7Ö7ó*ùs   ƒ #c              3  ó8   "  € T F  qP                   x € K  	  R # 5ir_   )rp   rX  s   & r[   rY  râ  \  s   é € ÐC±l¨dŸzžz³lùr	  T)rÜ  Úactive_range_treesr_  r-  r  r.  r[  r6  rv   r­   r\  r   r*  rh   r>   rý   rþ   rÌ  Úanyrr  )r|   r¢   Úactive_treesÚindex_numelsr‹   Útree_posrá  rÌ  s   &&    @@r[   Úis_broadcastedÚSIMDKernel.is_broadcasted8  sI  ù€ à×$Ñ$ U×+Ò+Ùà×.Ñ.Ó0ˆØ�sœS Ó.Õ.ˆØ×(Ô(ˆFØ×)Ñ)×-Ñ-¨fÓ5ˆEØŠ}áÜ˜eŸl™lÔ,?×@Ò@Ü$ðÜ §¡Ó-Ð.ð0óð ô ÜR¤)¨LÔ"9ÓRØóˆHð Òñ Ø×" e§l¡lÕ2Õ"ñ+ )ô2 —7‘7×#Ñ#×,Ñ,ˆß‹sô 
ä),ØÑC±lÓCô*ó
�sŒsð 	
Šsð 	
ˆsô 
ä),ØÑC±lÓCô*ó
ó 
ð 	
r]   c               ó    € V ^8„  d   QhRRRR/# )rU   r¢   rq   rX   rk   rY   )rZ   s   "r[   r\   r»  `  s   € ÷ 7ñ 7 *ð 7°ñ 7r]   c                óÀ   € \        V\        4      '       d*   RRP                  \        V P                  V4      4       R2# V P                  V P                  V4      4      # )a`  
Convert an index expr to a string that can be used in output code.
e.g. a sympy expression "s2" may actually appear as "ks1" in the generated kernel.

Index expressions often need to be passed in as arguments to the triton kernel.
Rename_indexing and codegen_indexing keep track of the needed indices and add
new parameters to the function signature.
Ú[r²   Ú])r[  r§  ÚjoinÚmapÚindex_to_strr²  Úrename_indexingrÛ  s   &&r[   rô  ÚSIMDKernel.index_to_str`  sQ   € ô �eœT×"Ò"Ø�t—y‘y¤ T×%6Ñ%6¸Ó!>Ó?Ð@ÀÐBÐBØ�z‰z˜$×.Ñ.¨uÓ5Ó6Ð6r]   c               ó    € V ^8„  d   QhRRRR/# rØ  rY   )rZ   s   "r[   r\   r»  n  s   € ÷ 
ñ 
¨Jð 
¸4ñ 
r]   c                	óÈ  € \        V \        \        P                  \        P                  34      '       g5   V P
                  '       g#   V P                  '       g   V P                  R Jd   R# V P                  p\        V4      ;'       dZ    \        ;QJ d    R V 4       F  '       d   K   RM	  R M! R V 4       4      ;'       d    \        P                  ! V 4      \        8„  # )TFc              3  ó”   "  € T F>  p\        V\        P                  \        P                  \        P                  34      x € K@  	  R # 5ir_   )r   r   rV  ÚUNBACKED_INTÚPRECOMPUTED_SIZErW  s   & r[   rY  Ú:SIMDKernel._should_precompute_size_expr.<locals>.<genexpr>z  sF   é € ð 
ñ !�Aô ØäŸ	™	Ü×)Ñ)Ü×-Ñ-ð÷ð ó !ùs   ‚AA)r[  rW   r™   r¼   ÚNumberÚ	is_numberÚ	is_symbolÚ
is_integerr-  r�   r`  Ú	count_opsÚ'_MAX_INLINE_PRECOMPUTABLE_SIZE_EXPR_OPS)r¢   rc  s   & r[   Ú_should_precompute_size_exprÚ'SIMDKernel._should_precompute_size_exprm  s´   € ô �uœs¤E§L¡L´%·,±,Ð?×@Ò@Ø��ˆØ��ˆØ×Ñ tÓ+áà×$Ñ$ˆä�‹M÷ Qð Qß“ñ 
ñ !ó
——’ñ 
ñ !ó
ó 
÷Qð Qô —’ Ó&Ô)PÑPð	
r]   c               ó    € V ^8„  d   QhRRRR/# r^  rY   )rZ   s   "r[   r\   r»  ˆ  s   € ÷ 31ñ 31àð31ð 
ñ31r]   c                	ó®  € V P                  V4      p\        V\        P                  P                  P
                  4      p\        VP                  \        P                  4      4      '       g/   \        VP                  \        P                  4      4      '       d4   VP                  \        P                  P                  P
                  4      p\        VP                  \        P                  4      4      '       d·   VP                  \        P                  4       F“  pVP                  p\        V4      ^ 8”  g   K!  \        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   K]  V\        P                  P                  P                  V4      /p\        W4      pK•  	  V P                  V4      p/ pV P                  4        Fn  p\        P                  P                  P!                  VP"                  ^4      '       g   K?  VP$                   F  p\        P&                  P(                  Wh&   K!  	  Kp  	  V'       d   \        WV4      p\+        V\,        4      '       g   TMVP.                  ^ ,          pV P1                  V4      '       d*   \        P                  P                  P                  V4      pV P3                  V4      # )r   c              3  óv   "  € T F/  p\        V\        P                  \        P                  34      x € K1  	  R # 5ir_   )r   r   rV  rû  rW  s   & r[   rY  Ú.SIMDKernel.prepare_indexing.<locals>.<genexpr>œ  s0   é € ð ,á$˜ô # 1¤t§y¡y´$×2GÑ2GÐ&H×IÐIÛ$ùs   ‚79FT)rÆ  r:   r>   rý   rþ   Úprecomputed_replacementsr_  Úatomsr™   ÚfloorÚceilingÚsubsr-  r`  Úlookup_precomputed_sizerç  rÿ   rp   rl   rš   rª  r[  r   r]  r  Úcodegen_indexing)	r|   r¢   Úarc  ÚreplacementsÚ
simp_indexÚsingleton_replacementsrÅ  r‹   s	   &&       r[   Úprepare_indexingÚSIMDKernel.prepare_indexingˆ  sþ  € ð ×&Ñ& uÓ-ˆÜ˜5¤!§'¡'×"2Ñ"2×"KÑ"KÓLˆäˆu�{‰{œ5Ÿ;™;Ó'×(Ò(¬C°·±¼E¿M¹MÓ0J×,KÒ,KØ—J‘JœqŸw™w×/Ñ/×HÑHÓIˆEô ˆu�{‰{œ5Ÿ=™=Ó)×*Ò*Ø—[‘[¤§¡Ö/�ð Ÿ.™.�Ü�w“< !Ö#¯«ñ ,á$ó,¯¯ªñ ,á$ó,÷ )ô )ð %&¤q§w¡w×'7Ñ'7×'OÑ'OÐPQÓ'RÐ#S�LÜ& uÓ;’Eñ 0ð ×+Ñ+¨EÓ2ˆ
à!#ÐØ×+Ñ+Ö-ˆDÜ—7‘7×#Ñ#×;Ñ;¸D¿J¹JÈ×JÒJÙð Ÿ-œ-�Ü16·±·±Ð&Ó.ó (ñ .÷ "Ü# JÓGˆJô )¨´X×>Ò>‰JÀJÇOÁOÐTUÕDVð 	ð ×,Ñ,¨Z×8Ò8ÜŸ™×)Ñ)×AÑAÀ*ÓMˆJà×$Ñ$ ZÓ0Ð0r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   r   rY   )rZ   s   "r[   r\   r»  ½  s   € ÷ 
ñ 
Ð$=ñ 
r]   c                	óŽ   € V P                    Uu. uF*  qP                  '       d   V P                  '       g   K(  VNK,  	  up# u upi r_   )rÂ  r„   rÐ  )r|   Úts   & r[   rç  ÚSIMDKernel.active_range_trees½  s<   € ð ×'Ò'ó
Ù'�!¯~¯~¨~À×AVÕAV�AˆAÑ'ñ
ð 	
ùò 
s
   �%A¹Ac               ó    € V ^8„  d   QhRRRR/# )rU   rÂ  zSequence[IterationRangesRoot]rX   úIterator[None]rY   )rZ   s   "r[   r\   r»  Å  s   € ÷ 1ñ 1Ø8ð1à	ñ1r]   c              #  ó  "  € V P                   p\        V4      V n         V P                  P                  4         Rx € W n         V P                  P                  4        R#   Y n         T P                  P                  4        i ; i5i)z;Temporarily codegen against an alternate range-tree family.N)rÂ  r§  rÆ  ró   )r|   rÂ  Úsaveds   && r[   Úuse_range_treesÚSIMDKernel.use_range_treesÄ  sn   é € ð
 × Ñ ˆÜ Ó,ˆÔð 	×Ñ×*Ñ*Ô,ð	1Ûà$ÔØ×"Ñ"×.Ñ.Ö0øð  %ÔØ×"Ñ"×.Ñ.Õ0üs   ‚7BºA  ¾"BÁ "BÂBc               ó    € V ^8„  d   QhRRRR/# )rU   r  rq   rX   rY   )rZ   s   "r[   r\   r»  Ô  s   € ÷ ñ  Zð °Jñ r]   c                	ób  € \         P                  P                  P                  WP	                  4       4      p\        VP                  \        R 7       FØ  pW P                  9   g   K  / pV P                  V,          P                  4        F.  p\         P                  P                  P                  V4      W4&   K0  	  \        V4      ^ 8”  d>   \        V P                  V,          P                  V4      V P                  V,          n        V P                  V,          P                  4        KÚ  	  V# )r&  )r>   rý   rþ   rÁ  rn   Úsortedr-  rk   r  ra  r  r_  r:   r  r<  )r|   r  Úsymr  Úpss   &&   r[   r  ÚSIMDKernel.codegen_indexingÔ  sæ   € Ü�w‰w×Ñ×4Ñ4°T¿?¹?Ó;LÓMˆÜ˜$×+Ñ+´×5ˆCØ×+Ñ+Ö+ð  "�Ø×/Ñ/°Õ4×EÑEÖG�BÜ'(§w¡w×'7Ñ'7×'OÑ'OÐPRÓ'S�LÓ$ñ Hä�|Ó$ qÔ(Ü6@Ø×-Ñ-¨cÕ2×7Ñ7Ø$ó7�D×)Ñ)¨#Õ.Ô3ð ×%Ñ% cÕ*×2Ñ2Ö4ñ 6ð ˆr]   c               ó   € V ^8„  d   QhRR/# rð   rY   )rZ   s   "r[   r\   r»  å  s   € ÷ <ñ < 4ñ <r]   c                	ó   € \        R 4      h)zNYI: codegen_nan_checkré  rƒ   s   &r[   Úcodegen_nan_checkÚSIMDKernel.codegen_nan_checkå  s   € Ü!Ð":Ó;Ð;r]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   rá  rv   Úcoder5   rX   rw   rY   )rZ   s   "r[   r\   r»  è  s-   € ÷ Jñ Jà"ðJð ðJð 
ñ	Jr]   c                	ó   € \        R 4      h)z$NYI: iteration_ranges_codegen_headerré  )r|   rá  r+  s   &&&r[   Úiteration_ranges_codegen_headerÚ*SIMDKernel.iteration_ranges_codegen_headerè  s   € ô
 "Ð"HÓIÐIr]   c                	ó¨   € \         P                  P                  p\        V P                  P
                  4       F  pVP                  V4       K  	  R # r_   )r>   rý   Úwrapper_coder  r]  Úworkspace_argsÚgenerate_workspace_deallocation)r|   ÚwrapperÚwss   &  r[   Údeallocate_workspacesÚ SIMDKernel.deallocate_workspacesï  s8   € Ü—'‘'×&Ñ&ˆÜ˜4Ÿ9™9×3Ñ3Ö4ˆBØ×3Ñ3°BÖ7ó 5r]   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rU   rj   rk   rô   zIRNode | NoneÚdeallocate_wsr�   rX   rw   rY   )rZ   s   "r[   r\   r»  ô  s,   € ÷ 6ñ 6Øð6Ø,ð6ØDHð6à	ñ6r]   c                	ó   € \        R 4      h)zNYI: call_kernelré  )r|   rj   rô   r8  s   &&&&r[   Úcall_kernelÚSIMDKernel.call_kernelô  s   € ô "Ð"4Ó5Ð5r]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   rÌ   zstr | OpsWrapperr”  r•  rX   zIterator[str]rY   )rZ   s   "r[   r\   r»  ú  s"   € ÷ )ñ )Ð/ð )¸ð )Èñ )r]   c              #  ó  "  € V P                   pV P                  pV'       d   \        P                  ! W4      p\        P
                  ! V4      pWn         W n         Vx € W0n         W@n        R#   Y0n         Y@n        i ; i5i)z:Context manager to add an additional mask to tl.load/storeN)Ú
_load_maskÚ_load_otherr<   Úlogical_andr=   Ú_unwrap)r|   rÌ   r”  rL  Ú	prior_vals   &&&  r[   Ú
mask_loadsÚSIMDKernel.mask_loadsù  sj   é € ð —‘ˆØ×$Ñ$ˆ	ßÜ—?’? 4Ó/ˆDä×!Ò! $Ó'ˆØŒØ Ôð	)àŠJà#ŒOØ(Öøð $ŒOØ(Õüs   ‚AA?ÁA. Á A?Á.A<Á<A?c               ó    € V ^8„  d   QhRRRR/# )rU   r¢   rq   rX   ro   rY   )rZ   s   "r[   r\   r»    s   € ÷ ñ ¨ð Ð8Vñ r]   c                ó,  € V P                   P                  4        UUu/ uF  w  r#W#P                  bK  	  ppp\        W4      p/ pV P                   F9  p\        VP                  4      p\        WX^/4      \        WX^ /4      ,
          Wh&   K;  	  V# u uppi )a  
This gets the stride of the index for each of the tiling variables
(technically, it does it at index 0)

For example, if
xindex = x0 + 512*x1 + 1024*r0
x0 = (xindex//512)
x1 = (xindex % 512)
r0 = rindex // 1024

this function would return
{xindex: 512, rindex: 1024}
)r  r�   r  r:   rÂ  r8   rj   )	r|   r¢   ÚkÚvÚindex_to_tile_indexesÚindex_in_tile_varsÚstridesÚ
range_treer0  s	   &&       r[   Úget_strides_of_loadÚSIMDKernel.get_strides_of_load  s’   € ð 8<×7LÑ7L×7RÑ7RÔ7TÔ UÑ7T©t¨q §F¡F¢Ñ7TÐÑ UÜ'¨ÓEÐØˆØ×*Ô*ˆJÜ" :§?¡?Ó3ˆAÜ#Ð$6¸A¸Ó?Ä*Ø"¨ FóCõ ˆG‹Jñ +ð
 ˆùó !Vs   žBc                	óf   € \        V\        4      '       d   \        \        W4      4      # V ! V4      # r_   )r[  rØ   ró  )rÕ  r”  s   &&r[   Ú_map_tuple_or_scalarÚSIMDKernel._map_tuple_or_scalar#  s(   € ä�eœU×#Ò#Üœ˜R›Ó(Ð(Ù�%‹yÐr]   c               ó   € V ^8„  d   QhRR/# )rU   rX   r§   rY   )rZ   s   "r[   r\   r»  )  s   € ÷ (ñ ( 
ñ (r]   c                	óÄ   € \         P                  ! V P                  P                  4       Uu. uF  pVP	                  4       NK  	  pp\        \        R V4      4      # u upi r_   )rI   Ú
only_nodesr­  r¨  Úestimate_flopsr
  Úfilter)r|   rô   Úflopss   &  r[   rU  ÚSIMDKernel.estimate_flops)  s[   € ô +×5Ò5°d·m±m×6QÑ6QÔRó
áR�ð ×ÑÖ!ÙRð 	ð 
ô ”6˜$ Ó&Ó'Ð'ùò	
s   ­Ac           	     ó`  € . p\        \        V P                  P                  P	                  4       4      4      pV P                  P                  4       w  r4 pV P                  P                  4       p\        P                  P                  P                  \        V P                  P	                  4       4      4      p\        V4       EFZ  w  rxW…9  d   VP                  ^ 4       K  \        P                  P!                  V4      p	\        P                  P                  P                  V	4      p
W¦8”  d‘   \"        \$        ,          ! 4       p^ pWX,           FY  p\'        V\(        \*        34      '       d    VP-                  RV 24       V^,          pK>  VP-                  VP.                  4       K[  	  \        V4      V,          pMT
p\        P                  P1                  V4      p\3        V4      pVP                  VV,          ^\5        Wr8  4      ,           ,          4       EK]  	  \7        V4      # )aË  
Try the best to estimate the total size (in bytes) of the
kernel's inputs and outputs, which is used for estimating the memory
throughput of this kernel. This information is used for checking how
far we are from the peak memory bandwidth. It's important that
we want to avoid overestimating the sizes of the inputs and outputs,
because it can wrongfully give us a very large memory traffic value,
which may be even larger than the theoretical bandwidth and thus
become very misleading. This is particularly problematic for cases
where we slice some inputs. In those cases, we should only count
the size of the "slices" instead of the original inputs, because
only the slices contribute to the real memory traffic.
Úno_index_dep_)r_  r;   r]  Úinplace_buffersrò   Úpython_argdefsr­  Úbuf_accessesr>   rý   rþ   r  r9   rÍ  r*  r  Ú	get_numelr   r   r[  r   r    r+  r¢   Ú	get_dtyper4   rW   r
  )r|   ÚnbytesÚninplace_argsrx  Ú	call_argsr]  Ú	out_numelrú  rb  Ú	arg_numelÚbuf_sizerC  Úno_index_dep_countÚdeprp   r  Ú
dtype_sizes   &                r[   Úestimate_kernel_num_bytesÚ$SIMDKernel.estimate_kernel_num_bytes0  s­  € ð ˆÜœF 4§9¡9×#<Ñ#<×#CÑ#CÓ#EÓFÓGˆØ!ŸY™Y×5Ñ5Ó7Ñˆ�a˜Ø—}‘}×1Ñ1Ó3ˆô —G‘G×$Ñ$×6Ñ6Ü˜$Ÿ+™+×,Ñ,Ó.Ó/ó
ˆ	ô   	×*‰FˆAð Ô&Ø—‘˜aÔ ÙÜŸ™×)Ñ)¨#Ó.ˆIÜ—w‘w×'Ñ'×9Ñ9¸)ÓDˆHØÔ#ô %¤Sž/Ó+�Ø%&Ð"Ø'×,Ð,�CÜ! #¬´Ð'9×:Ò:ØŸ™ mÐ4FÐ3GÐ$HÔIØ*¨aÕ/Ò*àŸ™ C§I¡IÖ.ñ -ô ˜G› yÕ0‘à �Ü—G‘G×%Ñ% cÓ*ˆEÜ'¨Ó.ˆJà�M‰M˜% *Õ,°´C¸Ñ8IÓ4JÕ0JÕK×Lñ; +ô< �6‹{Ðr]   c           	     ó¨  € \        V P                  P                  4      ^8X  dK   \        V P                  P                  4      ^8X  d'   \        V P                  P                  4      ^ 8X  d   R# V P                  P                  4       w  r#rERpV EF€  p\        P                  P                  V4      pV'       g   K-  VP                  4       p	\        V	P                  4      ^8X  g   KY  \        V	P                   U
u. uF  qª^8X  g   K  V
NK  	  up
4      ^8X  d   K�  \        P                  ! V	P                  4      pVf   TpKµ  Wk8w  g   K½  \        RV R2RV RV 2,           4      p\        P!                  V4       V Uu. uFu  p\        P                  P                  V4      '       dL   \        P                  ! \        P                  P#                  V4      P                  4       P                  4      MRNKw  	  ppV Uu. uFa  p\        P                  P                  V4      '       d8   \        P                  P#                  V4      P                  4       P                  MRNKc  	  ppV Uu. uFG  pV\        P                  P$                  9   d   RM"V\        P                  P&                  9   d   RMRNKI  	  ppV U
u. uF  qªP(                  NK  	  pp
\        RV R	V R
V 2RV RV R2,           4      p\        P!                  V4        R# 	  \+        RV R24      p\        P!                  V4       R# u up
i u upi u upi u upi u up
i )zZ
Print message if the kernel have mixed layout inputs.
Only care about 4D tensor for now.
NzExpected stride order z, but found stride orderÚ z for kernel Ú
GraphInputÚIntermediateBufferz  param names z
  buf names z
  strides z	
  sizes z
  sources Ú
z%All the inputs for the triton kernel z have uniform layout)r_  r]  Úinput_buffersÚoutput_buffersr[  r\  r>   rý   Útry_get_bufferÚ
get_layoutr·  r   Úget_stride_orderÚstrider-   ÚlogÚwarningÚ
get_bufferÚgraph_inputsÚname_to_bufferrj   r*   )r|   r´  Úargdefsrb  Ú
_signaturerx  Úuniform_stride_orderÚarg_nameÚbufÚlayoutr  Ústride_orderÚmsgrj   Ústride_order_listÚ	size_listÚsource_listÚargdef_namess   &&                r[   Úwarn_mix_layoutÚSIMDKernel.warn_mix_layoutr  s  € ô �—	‘	×'Ñ'Ó(¨AÔ-Ü�D—I‘I×,Ñ,Ó-°Ô2Ü�D—I‘I×-Ñ-Ó.°!Ô3ñ
 à,0¯I©I×,DÑ,DÓ,FÑ)ˆ˜JØ#Ðä!ˆHÜ—'‘'×(Ñ(¨Ó2ˆCßÙØ—^‘^Ó%ˆFÜ�6—;‘;Ó 1Ö$ä 6§;¢;Ó9¡;˜a°q±&Ÿ˜¡;Ñ9Ó:¸aÔ?ÙÜ!×2Ò2°6·=±=ÓA�Ø'Ò/Ø+7Ò(Ø)Ö9Ü%Ø0Ð1EÐ0FÐF^Ð_Ø˜l˜^¨<¸°}ÐEõFó�Cô —K‘K Ô$ñ %.ó)ñ %.˜Dô Ÿ7™7×1Ñ1°$×7Ò7ô ×+Ò+ÜŸG™G×.Ñ.¨tÓ4×?Ñ?ÓA×HÑHôð "ò	"ñ
 %.ð &ð )ñ %.ó	!ñ %.˜Dô Ÿ7™7×1Ñ1°$×7Ò7ô Ÿ™×*Ñ*¨4Ó0×;Ñ;Ó=×BÒBà!ò"ñ %.ð	 ð !ñ %.ó#ñ %.˜Dð	  ¤1§7¡7×#7Ñ#7Ô7ñ %ð  ¤1§7¡7×#9Ñ#9Ô9ñ 2à!ò	"ñ
 %.ð  ð #ñ 5<Ó#<±G¨q§F¤F±G�LÐ#<Ü%Ø(¨¨°nÀYÀKÈ|Ð\mÐ[nÐoØ& y k°¸k¸]È"ÐMõNó�Cô —K‘K Ô$Úña "ôb Ø3°K°=Ð@TÐUó
ˆô 	�‰�CÖùò[ :ùò)ùò!ùò#ùò $=s2   Ã<L;
Ä	L;
Æ)M Æ,AM È)MÈ-=MÉ0AM
ËMc                	óˆ  € \         P                  ! WR V4      pRV n        \         P                  ! V P                  P
                  V4      p\         P                  ! W44      pRV n        \         P                  ! W%4      p\         P                  ! Wf4      p\         P                  ! WR V4      p\        P                  ! WXV34      # )r
  FT)r<   Ú	reductionrÐ  Ú
index_exprr­  r¾  ÚtruedivÚsubÚmulr=   rA  )	r|   r  r”  Úsum_r¬  ÚmeanÚdxÚdx2Úm2s	   &&&      r[   Úwelford_reduce_fallbackÚ"SIMDKernel.welford_reduce_fallbackº  s�   € Ü�}Š}˜U¨5°%Ó8ˆØ %ˆÔÜ—’ §¡× =Ñ =¸uÓEˆÜ�{Š{˜4Ó(ˆà $ˆÔÜ�WŠW�UÓ!ˆÜ�gŠg�b‹oˆÜ�]Š]˜5¨°Ó4ˆÜ×!Ò! 4¨VÐ"4Ó5Ð5r]   c                	óè   € \         P                  ! WR V4      p\         P                  ! W#4      p\         P                  ! V4      p\         P                  ! WRV4      p\        P
                  ! W634      # )Úfmaxr
  )r<   rŠ  r�  Úexpr=   rA  )r|   r  r”  Úvmaxr�  r˜  Úvsums   &&&    r[   Ú prepare_softmax_twopass_fallbackÚ+SIMDKernel.prepare_softmax_twopass_fallbackÆ  sT   € Ü�}Š}˜U¨6°5Ó9ˆÜ�gŠg�eÓ"ˆÜ�gŠg�c‹lˆÜ�}Š}˜U¨5°#Ó6ˆÜ×!Ò! 4 ,Ó/Ð/r]   c                	ó   € \         hr_   ré  rƒ   s   &r[   Úcodegen_kernelÚSIMDKernel.codegen_kernelÍ  r  r]   c                	ó   € R # r_   rY   rƒ   s   &r[   r{  ÚSIMDKernel.codegen_bodyÐ  ó   € Ùr]   c               ó   € V ^8„  d   QhRR/# )rU   rá  rû   rY   )rZ   s   "r[   r\   r»  Ó  s   € ÷ ñ Ð4Hñ r]   c                	ó   € R # r_   rY   )r|   rá  s   &&r[   rO  Ú)SIMDKernel.codegen_iteration_ranges_entryÓ  r¢  r]   )rä  r>  r?  rÊ  r   rÒ  r­  rË  rÐ  rØ  r  rá  rà  rº  rÉ  rÖ  rÍ  rÔ  r  rÂ  rß  râ  rÆ  r×  rª  r«  )NNNNF©NT)Fr”   r•   r–   r—   r˜   Úpexprr±  r¤  r³  r{   rò  r÷  rý  r  rœ   r1   r  r  r  r  rÕ  r<  rÞ  rF  rM  rÑ  rÓ  rn   rÄ  rÃ  rj  r‚  rˆ  Ústaticmethodr»  Úclassmethodr™   rš   r›   rÁ  rÆ  rÍ  rË  rÜ  rì  rô  r  r  rç  r€  r�  r  r  r(  r-  r5  r:  rC  rM  rP  rU  ri  r‡  r”  r›  rž  r{  rO  r�   rž   rŸ   s   @r[   rt   rt     s	  ø‡ ñð */€EÐ&Ó.Ø&Ó&Ø!€O�TÓ!àÓ÷D?ò D?õL"õ$
õ%òð
 ØôJó ó ðJõ"õ!ð ôHó ðHõõ5õn-õõ*õõõ
õ
õ>õõ(õ2	
ð ôR1ó ðR1ðh ð
 ',§g¡g§k¡kö	ó ðð& ð
 ',§g¡g§k¡kö	ó ðõ VðD ôOó ðOõ84õ&
õP7ð ô
ó ð
õ431õj
ð ×Ñô1ó ð1õõ"<õJò8÷
6ð
 ×Ñô)ó ð)õ"ð0 ñó ðõ
(ò@òDFòP
6ò0ò"ò÷ó r]   rt   T)Úfrozenc                  ó0   € ] tR tRt$ RtR]R&   R]R&   RtR# )Ú_IterationSpacei×  am  Source domain used to remap a pointwise body.

``groups`` are source-domain extents. ``values`` are the matching
codegen-side symbolic index expressions in the outer kernel.

For a grouped reduction body with logical ranges [B, D // G, G],
``groups`` may be [B, D // G, G] while ``values`` are the concrete
symbols the parent kernel is using for those coordinates.
rD  r‹  rò   rY   Nr¯  rY   r]   r[   r¬  r¬  ×  s   ‡ ñð !Ó Ø × r]   r¬  c                  óÜ   € ] tR tRt$ RtR]R&   ]P                  ! ]R7      t	R]R&   Rt
R	]R
&   R R ltR R lt]P                  R R l4       tR R lt]P                  R R l4       tRtR# )Ú_DerivedIterationFamilyiç  a8  Iteration family for a nested-reduction consumer stage.

Two configurations:
  - reduced-output: swaps in a derived grouped-axis tree; ``index_subs``
    rewrites body vars to the active tree symbols
  - parent-full: reuses outer trees; store_cache values are broadcast-lifted
    lazily through CSE when needed
ztuple[IterationRangesRoot, ...]rÂ  )Údefault_factoryro   Ú
index_subsFr�   Ú_headers_emittedc               ó    € V ^8„  d   QhRRRR/# r^  rY   )rZ   s   "r[   r\   Ú$_DerivedIterationFamily.__annotate__ø  s   € ÷ +ñ + ð +°
ñ +r]   c                	ó8   € VP                  V P                  4      # r_   )r  r°  rÛ  s   &&r[   Úremap_indexÚ#_DerivedIterationFamily.remap_indexø  s   € Ø�z‰z˜$Ÿ/™/Ó*Ð*r]   c               ó    € V ^8„  d   QhRRRR/# )rU   rs   úSIMDKernel[Any]rX   r�   rY   )rZ   s   "r[   r\   r³  û  s   € ÷ 
ñ 
 ?ð 
°tñ 
r]   c                	ó@  € \        VP                  4      \        V P                  4      8w  d   R # \        ;QJ d:    R \        VP                  V P                  4       4       F  '       d   K   R # 	  R# ! R \        VP                  V P                  4       4       4      # )Fc              3  ó,   "  € T F
  w  rWJ x € K  	  R # 5ir_   rY   )rX  ÚactiveÚexpecteds   &  r[   rY  Ú7_DerivedIterationFamily.is_active_on.<locals>.<genexpr>þ  s   é € ð 
á$MÑ �ð ÕÛ$Mùs   ‚T)r_  rÂ  r`  rr  ©r|   rs   s   &&r[   Úis_active_onÚ$_DerivedIterationFamily.is_active_onû  s„   € Üˆv×!Ñ!Ó"¤c¨$×*:Ñ*:Ó&;Ô;Ùß‹sñ 
ä$'¨×(:Ñ(:¸D×<LÑ<LÔ$Mó
�sŒsð 	
Šsð 	
ˆsñ 
ä$'¨×(:Ñ(:¸D×<LÑ<LÔ$Mó
ó 
ð 	
r]   c               ó   € V ^8„  d   QhRR/# )rU   rs   r¸  rY   )rZ   s   "r[   r\   r³    s   € ÷ ñ  Oñ r]   c              #  ó¼   "  € V P                  V4      '       d   Rx € R# V P                  V4      ;_uu_ 4        Rx € RRR4       R#   + '       g   i     R# ; i5i)z>Activate this family if it isn't already active on ``kernel``.N)r¿  Úactivater¾  s   &&r[   Úensure_activeÚ%_DerivedIterationFamily.ensure_active  s;   é € ð ×Ñ˜V×$Ò$Ýà—‘˜v×&Õ&Û÷ '×&×&Ò&üs   ‚7A¹A¾
AÁA	Á	Ac               ó    € V ^8„  d   QhRRRR/# )rU   rs   r¸  rX   rw   rY   )rZ   s   "r[   r\   r³    s   € ÷ %ñ % _ð %¸ñ %r]   c                	ó4  € V P                   '       d   R # V P                   Fl  p\        V\        4      '       g   K  VP                  ;'       d    VP
                  pV'       d   VP                  MVP                  pVP                  W$4       Kn  	  RV n         R # r¦  )	r±  rÂ  r[  rr  r¥   rÐ  rË  rÊ  r-  )r|   rs   rÅ  Úemit_in_loop_bodyÚtargets   &&   r[   Úensure_headersÚ&_DerivedIterationFamily.ensure_headers  sv   € ð
 × × Ð ÙØ×$Ô$ˆDÜ˜$Ô :×;Ô;ð %)§L¡L×$LÐ$L°V×5LÑ5LÐ!ß1B˜×-Ò-ÈÏÉ�Ø×6Ñ6°tÖDñ %ð !%ˆÖr]   c               ó    € V ^8„  d   QhRRRR/# )rU   rs   r¸  rX   r  rY   )rZ   s   "r[   r\   r³    s   € ÷ ñ ˜ð °>ñ r]   c              #  	ó¸   "  € V P                  V4       VP                  V P                  4      ;_uu_ 4        R x € R R R 4       R #   + '       g   i     R # ; i5ir_   )rÊ  r  rÂ  r¾  s   &&r[   rÃ  Ú _DerivedIterationFamily.activate  s=   é € ð 	×Ñ˜FÔ#Ø×#Ñ# D×$4Ñ$4×5Õ5Û÷ 6×5×5Ò5üs   ‚5A·A¼
AÁA	Á	A)r±  N)r”   r•   r–   r—   r˜   r¤  ÚdataclassesÚfieldr©  r°  r±  rµ  r¿  r€  r�  rÄ  rÊ  rÃ  r�   rY   r]   r[   r®  r®  ç  st   ‡ ñð 1Ó0à1<×1BÒ1BÐSWÔ1X€JÐ.ÓXØ"Ð�dÓ"õ+õ
ð ×Ñôó ðõ%ð" ×Ñôó ôr]   r®  c                  óX   € ] tR tRt$ RtR]R&   R]R&   R]R&   R]R	&   R
]R&   R
]R&   RtR# )Ú_GroupedReductionVarsi&  aÖ  Variables used to call the grouped-reduction body.

``iter_remapped`` and ``reduce_remapped`` are the iteration/reduction
arguments passed to the grouped reduction's loop body. ``passthrough`` is
the non-grouped axis, while ``group_index`` and ``local_reduction`` are
the two pieces of the grouped parent axis after it is split into
[num_groups, local_reduction_size]. ``group_index_var`` is None only for
the single-group case, where the group index is the constant 0.
r  Úiter_remappedÚreduce_remappedrˆ   Úpassthrough_iter_varzsympy.Symbol | NoneÚgroup_index_varrq   Úgroup_index_exprÚlocal_reduction_varrY   Nr¯  rY   r]   r[   rÒ  rÒ  &  s-   ‡ ñð $Ó#Ø%Ó%Ø&Ó&Ø(Ó(Ø Ó Ø#×#r]   rÒ  c                  óâ  € ] tR tRt$ RtR]R&   R]R&   R]R&   R]R	&   ]R
 R l4       t]R R l4       t	]R R l4       t
]R R l4       t]R R l4       t]R R l4       t]R R l4       t]R R l4       t]R R l4       t]R R l4       t]R R l4       t]R  R! l4       t]R" R# l4       t]R$ R% l4       t]R& R' l4       tR( R) ltR* R+ ltR, R- ltR. R/ ltR0 R1 ltR2 R3 ltR4 R5 ltR6tR7# )8Ú_GroupedReductionLayouti:  zÿGeometry of the grouped reduction (the consumer reduction in a
nested-reduction pair). Describes how the outer kernel's tile is
reshaped into [num_groups, local_reduction_size], which axis is reduced,
and how reduced values are lifted back for epilogues.
rv   Úx_treeÚr_treeúsympy.IntegerÚlocal_reduction_sizer�   Úlocal_reduction_in_rc               ó(   € V ^8„  d   QhRRRRRRRR/# )	rU   rs   r¸  rÞ  rÝ  rß  r�   rX   rÚ  rY   )rZ   s   "r[   r\   Ú$_GroupedReductionLayout.__annotate__H  s2   € ÷ 
ñ 
àð
ð ,ð
ð #ð	
ð
 
!ñ
r]   c                	ó   € \        VP                  4      ^8w  d"   \        R\        VP                  4       24      hVP                  w  rEVP                  R8w  d   \        RVP                  : 24      hV ! VVVVR7      # )rU   z$expected exactly 2 range trees, got r  zexpected x range tree, got )rÛ  rÜ  rÞ  rß  )r_  rÂ  r­   rr   )rÀ  rs   rÞ  rß  rÛ  rÜ  s   &&&&  r[   Úfrom_kernelÚ#_GroupedReductionLayout.from_kernelG  s†   € ô ˆv×!Ñ!Ó" aÔ'Ü Ø6´s¸6×;MÑ;MÓ7NÐ6OÐPóð ð  ×+Ñ+‰ˆØ�=‰=˜CÔÜ Ð#>¸v¿}¹}Ñ>OÐ!PÓQÐQÙØØØ!5Ø!5ô	
ð 	
r]   c               ó   € V ^8„  d   QhRR/# ©rU   rX   rv   rY   )rZ   s   "r[   r\   rá  ]  s   € ÷ Iñ IÐ/ñ Ir]   c                óV   € V P                   '       d   V P                  # V P                  # )z<The tree decomposed into [num_groups, local_reduction_size].)rß  rÜ  rÛ  rƒ   s   &r[   Ú
group_treeÚ"_GroupedReductionLayout.group_tree\  ó"   € ð #×7×7Ð7ˆt�{‰{ÐH¸T¿[¹[ÐHr]   c               ó   € V ^8„  d   QhRR/# ræ  rY   )rZ   s   "r[   r\   rá  b  s   € ÷ Iñ IÐ"5ñ Ir]   c                óV   € V P                   '       d   V P                  # V P                  # )z5The tree that passes through unchanged (not grouped).)rß  rÛ  rÜ  rƒ   s   &r[   Úpassthrough_treeÚ(_GroupedReductionLayout.passthrough_treea  rê  r]   c               ó   € V ^8„  d   QhRR/# rf  rY   )rZ   s   "r[   r\   rá  g  ó   € ÷ 5ñ 5˜Sñ 5r]   c                	ó.   € V P                   '       d   ^# ^ # ©rT  ©rß  rƒ   s   &r[   Úparent_axisÚ#_GroupedReductionLayout.parent_axisf  ó   € ð ×-×-Ð-ˆqÐ4°1Ð4r]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   rá  l  s   € ÷ Lñ L˜cñ Lr]   c                	óv   € R V P                   P                  P                  4       P                  R4      ,           # )Únested_rx  )rè  rr   r½   Úremovesuffixrƒ   s   &r[   Úgroup_prefixÚ$_GroupedReductionLayout.group_prefixk  s+   € à˜4Ÿ?™?×1Ñ1×7Ñ7Ó9×FÑFÀsÓKÕKÐKr]   c               ó   € V ^8„  d   QhRR/# r‡   rY   )rZ   s   "r[   r\   rá  p  s   € ÷ 
ñ 
¨,ñ 
r]   c                	óN   € \         P                  ! V P                   R 2RRR7      # )Ú_LOCAL_REDUCTION_SIZETr¹   ©r™   r¼   rû  rƒ   s   &r[   Úlocal_reduction_size_symÚ0_GroupedReductionLayout.local_reduction_size_symo  s+   € ä�|Š|Ø× Ñ Ð!Ð!6Ð7ØØô
ð 	
r]   c               ó   € V ^8„  d   QhRR/# r‡   rY   )rZ   s   "r[   r\   rá  x  s   € ÷ 
ñ 
 <ñ 
r]   c                	óN   € \         P                  ! V P                   R 2RRR7      # )Ú_REDUCED_BLOCKTr¹   r   rƒ   s   &r[   Úreduced_block_symÚ)_GroupedReductionLayout.reduced_block_symw  s*   € ä�|Š|Ø× Ñ Ð! Ð0ØØô
ð 	
r]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   rá  €  s   € ÷ 0ñ 0˜cñ 0r]   c                	ó6   € V P                   P                  4       # r_   )rè  rÂ   rƒ   s   &r[   Úparent_blockÚ$_GroupedReductionLayout.parent_block  s   € à�‰×-Ñ-Ó/Ð/r]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   rá  „  s   € ÷ 6ñ 6 3ñ 6r]   c                	ó6   € V P                   P                  4       # r_   )rí  rÂ   rƒ   s   &r[   Úpassthrough_blockÚ)_GroupedReductionLayout.passthrough_blockƒ  s   € à×$Ñ$×3Ñ3Ó5Ð5r]   c               ó   € V ^8„  d   QhRR/# r¶   rY   )rZ   s   "r[   r\   rá  ˆ  s   € ÷ Jñ J˜Jñ Jr]   c                	óV   € \        V P                  P                  V P                  4      # r_   )r   rè  rp   rÞ  rƒ   s   &r[   Ú
num_groupsÚ"_GroupedReductionLayout.num_groups‡  s   € ä˜Ÿ™×-Ñ-¨t×/HÑ/HÓIÐIr]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   rá  Œ  s   € ÷ +ñ + ñ +r]   c                	ó,   € \        V P                  4      # r_   )rk   r  rƒ   s   &r[   Únum_groups_strÚ&_GroupedReductionLayout.num_groups_str‹  s   € ä�4×)Ñ)Ó*Ð*r]   c               ó   € V ^8„  d   QhRR/# r°   rY   )rZ   s   "r[   r\   rá  �  s   € ÷ 2ñ 2¨#ñ 2r]   c                	ó,   € \        V P                  4      # r_   )rk   r  rƒ   s   &r[   Úlocal_reduction_size_dimÚ0_GroupedReductionLayout.local_reduction_size_dim�  s   € ä�4×0Ñ0Ó1Ð1r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   ztuple[str, str, str]rY   )rZ   s   "r[   r\   rá  ”  s   € ÷ 
ñ 
Ð3ñ 
r]   c                ó²   € V P                   '       d$   V P                  V P                  V P                  3# V P                  V P                  V P                  3# )z,Shape used before the local group reduction.)rß  r  r  r  rƒ   s   &r[   Úreshape_shapeÚ%_GroupedReductionLayout.reshape_shape“  s\   € ð ×$×$Ð$à×&Ñ&Ø×#Ñ#Ø×-Ñ-ðð ð ×ÑØ×)Ñ)Ø×"Ñ"ð
ð 	
r]   c               ó   € V ^8„  d   QhRR/# rf  rY   )rZ   s   "r[   r\   rá  £  rð  r]   c                ó.   € V P                   '       d   ^# ^# )z@Axis reduced in [passthrough, num_groups, local_reduction_size].ró  rƒ   s   &r[   Úreduce_axisÚ#_GroupedReductionLayout.reduce_axis¢  rö  r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   ztuple[str, str]rY   )rZ   s   "r[   r\   rá  ¨  s   € ÷ =ñ =˜oñ =r]   c                	ó†   € V P                   '       d   V P                  V P                  3# V P                  V P                  3# r_   )rß  r  r  rƒ   s   &r[   Úoutput_shapeÚ$_GroupedReductionLayout.output_shape§  s<   € à×$×$Ð$Ø×*Ñ*¨D×,?Ñ,?Ð@Ð@Ø×#Ñ# T×%;Ñ%;Ð<Ð<r]   c               ó    € V ^8„  d   QhRRRR/# )rU   rÊ  r   rX   rÒ  rY   )rZ   s   "r[   r\   rá  ­  s   € ÷ /
ñ /
àð/
ð 
ñ/
r]   c                	óN  € \         P                  P                  P                  V P                  ^4      '       dF   \
        P                  P                  pRpV P                  P                  4       P                  4       pM5V P                  P                  V P                  V P                  .4      w  r4TpV P                  P                  V P                  P                  .4      ^ ,          pV.p\        VP                   4      ^8X  d   V P"                  '       d   WR.MW%.pMu\        VP                   4      ^8X  dQ   V P"                  '       d   WPP                  ,          V,           M"W P                  P                  ,          V,           .pM\%        R4      h\'        VVVVVV4      # )rT  Nz1nested grouped reduction expects 1 or 2 iter vars)r>   rý   rþ   rÿ   r  r™   rš   rª  rè  r	  r‹   r  rÞ  rí  rp   r_  Ú	iter_varsrß  r­   rÒ  )r|   rÊ  r×  rÖ  rØ  Úpassthrough_varrÔ  rÓ  s   &&      r[   Úconstruct_group_reduction_varsÚ6_GroupedReductionLayout.construct_group_reduction_vars­  ss  € ô �7‰7×Ñ×3Ñ3°D·O±OÀQ×GÒGô  %Ÿw™wŸ|™|ÐØ"ˆOØ"&§/¡/×"<Ñ"<Ó">×"EÑ"EÓ"GÑà37·?±?×3LÑ3LØ—‘ $×";Ñ";Ð<ó4Ñ0ˆOð  /Ðð ×/Ñ/×9Ñ9Ø×"Ñ"×(Ñ(Ð)ó
à
õˆð /Ð/ˆÜˆt�~‰~Ó !Ô#ð ×,×,Ð,ð !Ñ3à&Ð8ñ ô
 �—‘Ó  AÔ%ð ×,×,Ð,ð  §/¡/Õ1Ð4DÖDà%×(=Ñ(=×(CÑ(CÕCÀoÕUð‰Mô !Ð!TÓUÐUä$ØØØØØØó
ð 	
r]   c               ó    € V ^8„  d   QhRRRR/# )rU   Úgroup_reduction_varsrÒ  rX   r®  rY   )rZ   s   "r[   r\   rá  Þ  s   € ÷ 3
ñ 3
à3ð3
ð 
!ñ3
r]   c                	óš  a € R  V 3R llpV! S P                   4      pV! S P                  4      p/ pS P                  '       d   TMTpS P                  '       d   TMTpVP                  4       P	                  4       WQP
                  &   VP                  e+   VP                  4       P	                  4       WQP                  &   \        VW43R7      # )c               ó    € V ^8„  d   QhRRRR/# )rU   rÅ  rv   rX   rY   )rZ   s   "r[   r\   ÚH_GroupedReductionLayout.make_reduced_output_family.<locals>.__annotate__â  s   € ÷ 	ñ 	Ð+ð 	Ð0Cñ 	r]   c           	     ór  <€ V SP                   Jd   V # SP                  SP                  R 3pSP                  \	        V P                  4       SP                  4      R 3p\        V \	        V P                  SP                  4      SP                  \	        V P                  4       SP                  4      W3R7      # )T)rp   r¾   rÈ   rí   )	rè  r  rÞ  r  r   r¾   rr  rp   rÈ   )rÅ  Úlocal_reduction_size_constÚreduced_block_constr|   s   &  €r[   ÚbuildÚA_GroupedReductionLayout.make_reduced_output_family.<locals>.buildâ  s³   ø€ Ø˜4Ÿ?™?Ó*ð �à×-Ñ-Ø×)Ñ)Øð*Ð&ð ×&Ñ&Ü˜Ÿ™Ó*¨D×,IÑ,IÓJØð#Ðô
 .ØÜ˜tŸz™z¨4×+DÑ+DÓEØ×1Ñ1Ü%Ø×%Ñ%Ó'¨×)FÑ)Fóð "<Ð Qôð r]   )r°  rÂ  )rÛ  rÜ  rß  r	  r‹   rÕ  rÖ  r®  )r|   r/  r6  Úreduced_x_treeÚreduced_r_treer°  Úreduced_passthrough_treeÚreduced_group_trees   f&      r[   Úmake_reduced_output_familyÚ2_GroupedReductionLayout.make_reduced_output_familyÞ  sÌ   ø€ ÷	ð 	ñ4 ˜tŸ{™{Ó+ˆÙ˜tŸ{™{Ó+ˆØ57ˆ
à"×7×7Ð7‰N¸^ð 	!ð #×7×7Ð7‰N¸^ð 	ð %×/Ñ/Ó1×8Ñ8Ó:ð 	×<Ñ<Ñ=ð  ×/Ñ/Ò;ð #×-Ñ-Ó/×6Ñ6Ó8ð ×;Ñ;Ñ<ô 'Ø!Ø'Ð8ô
ð 	
r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   r®  rY   )rZ   s   "r[   r\   rá    s   € ÷ 
ñ 
à	 ñ
r]   c                	óF   € \        V P                  V P                  3R 7      # ))rÂ  )r®  rÛ  rÜ  rƒ   s   &r[   Úmake_parent_full_familyÚ/_GroupedReductionLayout.make_parent_full_family  s!   € ô 'ØŸ™ d§k¡kÐ2ô
ð 	
r]   c               ó    € V ^8„  d   QhRRRR/# )rU   r/  rÒ  rX   r¬  rY   )rZ   s   "r[   r\   rá    s   € ÷ 
ñ 
Ø$9ð
à	ñ
r]   c                	óæ  € V P                   '       g]   V P                  V P                  V P                  P                  .pVP
                  VP                  VP                  .p\        VV4      # \        V P                  P                  V P                  P                  .V P                  P                  4       P                  4       V P                  P                  4       P                  4       .4      # r_   )rß  r  rÞ  rí  rp   r×  rØ  rÕ  r¬  rÛ  rÜ  r	  r‹   )r|   r/  Úsource_groupsÚsource_valuess   &&  r[   Úparent_full_iteration_valuesÚ4_GroupedReductionLayout.parent_full_iteration_values  sÎ   € ð ×(×(Ð(à—‘Ø×)Ñ)Ø×%Ñ%×+Ñ+ðˆMð %×5Ñ5Ø$×8Ñ8Ø$×9Ñ9ðˆMô
 #ØØóð ô Ø�[‰[×Ñ §¡× 1Ñ 1Ð2Ø�[‰[×#Ñ#Ó%×,Ñ,Ó.°·±×0FÑ0FÓ0H×0OÑ0OÓ0QÐRó
ð 	
r]   c               ó$   € V ^8„  d   QhRRRRRR/# ©rU   rs   rO   r”  r@   rX   rY   )rZ   s   "r[   r\   rá  1  s(   € ÷ 
ñ 
àð
ð ð
ð 
ñ	
r]   c                	ó°   € VP                   f   \        R4      hVP                  e   \        VP                  4      ^8  d   V# V P	                  VVRR7      # )Núvalue must have a known dtypeF©Úmaterialize_singleton©r  r­   rÙ   r_  Ú%_broadcast_value_to_parent_resolution©r|   rs   r”  s   &&&r[   Ú*maybe_broadcast_value_to_parent_resolutionÚB_GroupedReductionLayout.maybe_broadcast_value_to_parent_resolution1  s\   € ð
 �;‰;ÒÜ Ð!@ÓAÐAØ�;‰;Ò¤# e§k¡kÓ"2°QÔ"6ð ˆLà×9Ñ9ØØØ"'ð :ó 
ð 	
r]   c               ó$   € V ^8„  d   QhRRRRRR/# rI  rY   )rZ   s   "r[   r\   rá  D  s(   € ÷ 
ñ 
àð
ð ð
ð 
ñ	
r]   c                	óÂ   € VP                   f   \        R4      hVP                  e   \        VP                  4      ^8¼  g   \        R4      hV P	                  VVRR7      # )NrK  z;grouped reduction input must have a known parent-tile shapeTrL  rN  rP  s   &&&r[   Úensure_parent_tile_resolutionÚ5_GroupedReductionLayout.ensure_parent_tile_resolutionD  sg   € ð
 �;‰;ÒÜ Ð!@ÓAÐAØ—‘Ò'¬C°·±Ó,<ÀÔ,AÜ ØMóð ð ×9Ñ9ØØØ"&ð :ó 
ð 	
r]   c               ó(   € V ^8„  d   QhRRRRRRRR/# )rU   rs   rO   r”  r@   rM  r�   rX   rY   )rZ   s   "r[   r\   rá  V  s2   € ÷ 4ñ 4àð4ð ð4ð
  $ð4ð 
ñ4r]   c          	     	ó°  € VP                   f   \        R4      hVP                  e   \        VP                  4      ^8¼  g   \        R4      h\	        VP                  V P
                  ,          4      pW@P                  8X  g   VR8X  d   V'       g   V# V P                  pV P                  pVR8X  d)   V P                  '       d   V^3pWV3pWV3p	Me^V3pWe3pWe3p	MZV P                  '       d%   V P                  p
V P                  pWZ^3pWZV3pWV3p	M$V P                  p
V P                  pV
^V3pW«V3pWe3p	VP                  VVVV	VP                   V	R7      pV# )NrK  z)value must have a known parent-tile shaper×   )r”  Úpre_broadcast_shapeÚbroadcast_shapeÚfinal_shaper  Ú	out_shape)r  r­   rÙ   r_  rk   rô  r
  r  rß  r  r  Úemit_broadcast_via_reshape)r|   rs   r”  rM  Ú
parent_dimÚpassthrough_extentÚparent_extentrY  rZ  r[  r  rÞ  s   &&&$        r[   rO  Ú=_GroupedReductionLayout._broadcast_value_to_parent_resolutionV  s„  € ð �;‰;ÒÜ Ð!@ÓAÐAØ—‘Ò'¬C°·±Ó,<ÀÔ,AÜ Ð!LÓMÐMÜ˜Ÿ™ T×%5Ñ%5Õ6Ó7ˆ
Ø×*Ñ*Ô*Ø˜#Ô×&;àˆLà!×3Ñ3ÐØ×)Ñ)ˆØ˜Ôð ×(×(Ð(Ø'9¸1Ð&=Ð#Ø#5Ð"E�Ø1ÐA‘à'(Ð*<Ð&=Ð#Ø#0Ð"E�Ø,ÐA‘Ø×&×&Ð&Ø×,Ñ,ˆJØ#'×#@Ñ#@Ð à#5À1Ð"EÐØ1Ð?SÐTˆOØ-Ð=‰Kà×,Ñ,ˆJØ#'×#@Ñ#@Ð à#-¨qÐ2DÐ"EÐØ)ÐASÐTˆOØ(Ð=ˆKØ×1Ñ1ØØ 3Ø+Ø#Ø—+‘+Ø!ð 2ó 
ˆð ˆr]   rY   N)r”   r•   r–   r—   r˜   r¤  r©  rã  rœ   rè  rí  rô  rû  r  r  r
  r  r  r  r  r  r"  r&  r,  r<  r@  rF  rQ  rU  rO  r�   rY   r]   r[   rÚ  rÚ  :  s}  ‡ ñð  ÓØÓØ'Ó'ØÓàô
ó ð
ð( ôIó ðIð ôIó ðIð ô5ó ð5ð ôLó ðLð ô
ó ð
ð ô
ó ð
ð ô0ó ð0ð ô6ó ð6ð ôJó ðJð ô+ó ð+ð ô2ó ð2ð ô
ó ð
ð ô5ó ð5ð ô=ó ð=õ
/
õb3
õj
õ
õ.
õ&
÷$4ñ 4r]   rÚ  c                  ó<   € ] tR tRt$ RtR]R&   R]R&   R R ltR	tR
# )Ú_ParentFullLoadTransformi�  z9Lift loaded values to parent-full resolution when needed.rO   rs   rÚ  r€  c               ó    € V ^8„  d   QhRRRR/# )rU   r”  r@   rX   rY   )rZ   s   "r[   r\   Ú%_ParentFullLoadTransform.__annotate__”  s   € ÷ 
ñ 
˜;ð 
¨;ñ 
r]   c                	óN   € V P                   P                  V P                  V4      # r_   )r€  rQ  rs   )r|   r”  s   &&r[   ÚapplyÚ_ParentFullLoadTransform.apply”  s#   € Ø�{‰{×EÑEØ�K‰KØó
ð 	
r]   rY   N)r”   r•   r–   r—   r˜   r¤  rg  r�   rY   r]   r[   rc  rc  �  s   ‡ áCàÓØ#Ó#÷
ñ 
r]   rc  c                  ó^   a € ] tR tRtRtRR/R V 3R llltR R ltR	 R
 ltR R ltRt	V ;t
# )Ú_GroupedReductionOpsHandleri›  zõGroup-reduction stage handler: reshape parent-full tile into
[num_groups, local_reduction_size], reduce over the local axis, and store
the result.
Loads go through the normal CSE path, then may be lifted to parent
resolution before the reshape.
Úload_transformNc               ó(   € V ^8„  d   QhRRRRRRRR/# )	rU   rs   rO   r€  rÚ  Úfamilyr®  rk  ú_ParentFullLoadTransform | NonerY   )rZ   s   "r[   r\   Ú(_GroupedReductionOpsHandler.__annotate__£  s4   € ÷ .ñ .ð ð.ð
 (ð.ð (ð.ð 8ñ.r]   c               	óV   <€ \         SV `  V4       W n        W0n        W@n        WPn        R # r_   )rz   r{   Ú_kernelÚ_layoutÚ_familyÚ_load_transform)r|   Úinnerrs   r€  rm  rk  r}   s   &&&$$$€r[   r{   Ú$_GroupedReductionOpsHandler.__init__£  s'   ø€ ô 	‰Ñ˜ÔØŒØŒØŒØ-Ör]   c               ó$   € V ^8„  d   QhRRRRRR/# ©rU   rj   rk   r¢   rq   rX   r@   rY   )rZ   s   "r[   r\   ro  ²  ó!   € ÷ ñ ˜ð  Zð °Kñ r]   c                	óŽ   € V P                   P                  W4      pV P                  e   V P                  P                  V4      pV# r_   )Ú_innerÚloadrt  rg  )r|   rj   r¢   r”  s   &&& r[   r|  Ú _GroupedReductionOpsHandler.load²  s?   € Ø—‘× Ñ  Ó-ˆØ×ÑÒ+ð ×(Ñ(×.Ñ.¨uÓ5ˆEØˆr]   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )	rU   r  r  Ú	src_dtyper¢  rk   r”  r@   rX   rY   )rZ   s   "r[   r\   ro  º  s<   € ÷ 
ñ 
àð
ð ð
ð ð	
ð
 ð
ð 
ñ
r]   c                ó~  € V P                   pV P                  P                  WT4      pV P                  P	                  V4       VP                  W@P                  P                  V4      pV;P                  ^,          un        VP                  VVV P                  P                  VV P                  P                  4      # )zAReshape the parent-full tile and reduce over the local reduction.)rq  rr  rU  rs  rÊ  Úemit_reshaper  Únum_reductionÚemit_reducer"  r&  )r|   r  r  r¢  r”  rG  Úreshapeds   &&&&&  r[   rŠ  Ú%_GroupedReductionOpsHandler.reductionº  s‘   € ð �L‰LˆØ—‘×:Ñ:¸1ÓDˆð 	�‰×#Ñ# AÔ&Ø—>‘> %¯©×)CÑ)CÀYÓOˆØ	�Š˜1Õ�Ø�}‰}ØØØ�L‰L×$Ñ$ØØ�L‰L×%Ñ%ó
ð 	
r]   c               ó(   € V ^8„  d   QhRRRRRRRR/# rI  rY   )rZ   s   "r[   r\   ro  Ò  s)   € ÷ ;ñ ; Cð ;°
ð ;À;ð ;ÐSWñ ;r]   c                	ó  € V P                   P                  V4      pV P                   P                  V P                  4      ;_uu_ 4        V P                  P                  WV4       R R R 4       R #   + '       g   i     R # ; ir_   )rs  rµ  rÄ  rq  r{  rK  )r|   rj   r¢   r”  Úremapped_indexs   &&&& r[   rM  Ú+_GroupedReductionOpsHandler.store_reductionÒ  sR   € ØŸ™×1Ñ1°%Ó8ˆØ�\‰\×'Ñ'¨¯©×5Õ5Ø�K‰K×Ñ˜d°EÔ:÷ 6×5×5Ò5ús   Á	A0Á0B	)rs  rq  rr  rt  )r”   r•   r–   r—   r˜   r{   r|  rŠ  rM  r�   rž   rŸ   s   @r[   rj  rj  ›  s0   ø† ñð.ð ;?÷.ò .õõ
÷0;ó ;r]   rj  c                  óV   a € ] tR tRtRtRR/R V 3R llltR R ltRR	 R
 lltRtV ;t	# )Ú_PointwiseRemapHandleriØ  a,  Pointwise bodies at a remapped iteration range.

Unlike _GroupedReductionOpsHandler (which performs reshape+reduce), this
handler runs pure pointwise bodies for reduced/parent-full prologues
and epilogues. Loads use the normal CSE path after index remapping, then
may be lifted to parent resolution.
rk  Nc               ó$   € V ^8„  d   QhRRRRRR/# )rU   rs   rt   rm  r®  rk  rn  rY   )rZ   s   "r[   r\   Ú#_PointwiseRemapHandler.__annotate__á  s*   € ÷ .ñ .ð ð.ð
 (ð.ð 8ñ.r]   c               	óJ   <€ \         SV `  V4       W n        W0n        W@n        R # r_   )rz   r{   rq  rs  rt  )r|   ru  rs   rm  rk  r}   s   &&&$$€r[   r{   Ú_PointwiseRemapHandler.__init__á  s"   ø€ ô 	‰Ñ˜ÔØŒØŒØ-Ör]   c               ó$   € V ^8„  d   QhRRRRRR/# rx  rY   )rZ   s   "r[   r\   r�  î  ry  r]   c                	óV  € V P                   P                  V4      pV P                   P                  V P                  4      ;_uu_ 4        V P                  P                  W4      pR R R 4       V P                  e   V P                  P                  X4      pX#   + '       g   i     L;; ir_   )rs  rµ  rÄ  rq  r{  r|  rt  rg  )r|   rj   r¢   rˆ  r”  s   &&&  r[   r|  Ú_PointwiseRemapHandler.loadî  sx   € ØŸ™×1Ñ1°%Ó8ˆØ�\‰\×'Ñ'¨¯©×5Õ5Ø—K‘K×$Ñ$ TÓ:ˆE÷ 6à×ÑÒ+Ø×(Ñ(×.Ñ.¨uÓ5ˆEØˆ÷	 6×5ús   Á	BÂB(	c          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )rU   rj   rk   r¢   rq   r”  r@   Úmoder   rX   rw   rY   )rZ   s   "r[   r\   r�  ö  sC   € ÷ 
Fñ 
Fàð
Fð ð
Fð ð	
Fð
 ð
Fð 
ñ
Fr]   c           	     	ó  € V P                   pV P                  P                  V4      pV P                  P                  V4      ;_uu_ 4        V P                  P                  WW4R 7       RRR4       R#   + '       g   i     R# ; i))r”  N)rq  rs  rµ  rÄ  r{  rK  )r|   rj   r¢   r”  r”  rG  rˆ  s   &&&&&  r[   rK  Ú_PointwiseRemapHandler.storeö  s\   € ð �L‰LˆØŸ™×1Ñ1°%Ó8ˆØ�\‰\×'Ñ'¨×*Õ*Ø�K‰K×Ñ˜d°EÐÔE÷ +×*×*Ò*ús   ÁA3Á3B	)rs  rq  rt  r_   )
r”   r•   r–   r—   r˜   r{   r|  rK  r�   rž   rŸ   s   @r[   r‹  r‹  Ø  s-   ø† ñð.ð ;?÷.ò .õ÷
Fõ 
Fr]   r‹  c                  ó  € ] tR tRt$ Rt]tR]R&   R tR t	]	t
]	tR tR tR	 tR
 tRoR R lltR tR tRpR R lltR R ltR R lt]R R l4       tR R lt]R R l4       tR R ltRR/R  R! lltRpR" R# lltR$ R% lt]R& R' l4       tRqR( R) llt R* R+ lt!R, R- lt"R.R//R0 R1 llt#R2 R3 lt$R4 t%R5R/R6 lt&R7 R8 lt'R9 R: lt(R; R< lt)R5RR=R/R> R? llt*R@ t+RA RB lt,RC RD lt-]RE RF l4       t.]RG RH l4       t/RI RJ lt0RK RL lt1]RM RN l4       t2RrRO RP llt3RQ t4]]5Pl                  ! ^ 4      RR RS l4       4       t7]RT RU l4       t8]RV RW l4       t9]RX RY l4       t:]RZ R[ l4       t;]R\ R] l4       t<]R^ R_ l4       t=]R` Ra l4       t>]]?P€                  P‚                  R3Rb Rc ll4       tB]]?P€                  P‚                  R3Rd Re ll4       tC]]?P€                  P‚                  R3Rf Rg ll4       tDRh tERi Rj ltFRsRk Rl lltGRm tHRntIR# )tÚSIMDSchedulingi  zc
Single Instruction Multiple Data parent class used for fusion across
multiple different backends.
z	type[Any]Úkernel_typec                	óZ   € \         ;QJ d    . R  V 4       F  NK  	  5# ! R  V 4       4      # )c              3  ó†   "  € T F7  p\         P                  P                  P                  \	        V4      4      x € K9  	  R # 5ir_   rÒ  rW  s   & r[   rY  Ú*SIMDScheduling.group_fn.<locals>.<genexpr>  s-   é € ÐPÉ%ÀQ”Q—W‘W×%Ñ%×.Ñ.¬}¸QÓ/?×@Ð@Ë%ùs   ‚?A©rØ   )r|   r*  s   &&r[   Úgroup_fnÚSIMDScheduling.group_fn  s$   € ßŒuÑPÉ%ÓP�uÐPˆuÑPÉ%ÓPÓPÐPr]   c                ó‚  aaaa€ \        V\        P                  4      '       g!   \        V\        P                  4      '       d    \        P                  P                  W4      # VP                  w  pw  poVP                  w  pw  oo\        W4      pVP                  4       '       d8   VP                  4       '       g"   VP                  4       '       d   V! R4       R# MLVP                  4       '       d7   VP                  4       '       g!   VP                  4       '       d   V! R4       R# VP                  4       '       Ed]   VP                  4       '       EdF   VS8H  ;'       d    SS8H  pV'       g   ^ RIH	p VP                  W4      pV'       g1   ^ RIH
p VP                  W4      ;'       d    VP                  W4      pV'       g   V! RVSSS4       V'       d¿   VP                  4       '       g   VP                  4       '       d“   VP                  4       '       g   Y!r!VP                  VS4      o\        ;QJ d.    VV3R lVP                  4        4       F  '       d   K   RM#	  RM! VV3R lVP                  4        4       4      '       g   V! R4       R# V# VP                  4       '       EgØ   VP                  4       '       EgÁ   VS8X  d   SS8X  g®   VP!                  4       '       g   V! R	VSSS4       R# VP                  4        Fu  p	V	P!                  4       '       d    M^V	P#                  4       VP%                  4       ,          '       g   KH  V	P                  w  pw  r«WJ8X  d
   SV8X  d   Kh  V! R
VV
SV4        R# 	  W3 F  pVP!                  4       '       g   K   R# 	  VP                  VS4      pVP                  VS4      pV P'                  VP                  4       VP                  4       ,           VS4      p\(        P*                  P,                  '       dg   Rp\/        V4      ^8”  d,   \/        V4      ^8”  d   YÞu;8H  ;'       d    V8H  Mu pMWß8H  pM\/        V4      ^8”  d   Wï8H  pV'       g   V! RVVV4       R# R# VP                  4       '       EgV   VP                  4       '       Ed?   S^8X  d   S^8w  g   \1        RS RS 24      hVSS,          8X  dþ   \        ;QJ d.    VV3R lVP                  4        4       F  '       d   K   RM#	  RM! VV3R lVP                  4        4       4      '       g   V! R4       R# \(        P*                  P2                  '       dt   VP!                  4       '       g^   \5        VP                  V\6        P8                  P:                  4      P=                  4       4      V^3SS^339   pV'       g	   V! R4       V# R# VS8w  d	   V! R4       VS8H  # VP                  4       '       d   VP                  4       '       d   \1        R4      hV P?                  W!4      # )zŠ
Hook called by Scheduler to determine if the Triton backend
can fuse node1 and node2.  These nodes might already be
FusedSchedulerNodes.
z&Split scan cannot fuse with reductionsF)ÚMixOrderReduction)ÚNestedReductionz1numel/rnumel mismatch (reduce) (%s, %s), (%s, %s)c              3  óŒ   <"  € T F9  p\         P                  SP                  4       VP                  4       SR 7      x € K;  	  R# 5i©)r¾  N)rt   rÆ  rò   Ú
get_ranges)rX  Ún2Úrnumel1rª  s   & €€r[   rY  Ú*SIMDScheduling.can_fuse.<locals>.<genexpr>S  sD   øé € ð ñ 0˜ô ×,Ñ,ØŸ™›¨¯©«È'ð -÷ ð ó 0ùs   ƒAATz/invalid loop order and tiling for native matmulz5numel/rnumel mismatch (non-reduce) (%s, %s), (%s, %s)z:numel/rnumel mismatch prologue mismatch (%s, %s), (%s, %s)ztiling mismatch (%s, %s, %s)z,expected rnumel1 == 1 and rnumel2 != 1, got r†  c              3  óp   <"  € T F+  p\         P                  SS3VP                  4       4      x € K-  	  R # 5ir_   )rt   rÆ  r¥  )rX  r1  Únumel2Úrnumel2s   & €€r[   rY  r¨  ¤  s3   øé € ð á.˜ô ×,Ñ,¨f°gÐ->ÀÇÁÃ×OÐOÛ.ùs   ƒ36z"nodes numel/rnumel incompatibilityzinvalid tiling for reductionznodes numel incompatibilityz:expected node1 to be a reduction and node2 not a reduction) r[  r   ÚForeachKernelSchedulerNodeÚcan_fuser±  r0   Úis_split_scanr„   Útorch._inductor.schedulerr¡  r¢  Ú_is_dependent_reduction_pairrØ  Ú
get_tilingr`  Ú	get_nodesÚis_templateÚused_buffer_namesÚget_buffer_namesÚselect_tilingr   rb   Ú tiling_prevents_pointwise_fusionr_  r­   Ú tiling_prevents_reduction_fusionrØ   r™   rš   r›   rò   Úcan_fuse_horizontal)r|   Únode1Únode2rx  Únumel1ÚwhyÚreduction_can_fuser¡  r¢  rô   Ú	pro_numelÚ
pro_rnumelr1  Útiling1Útiling2Útiling3ÚcondÚis_reduction_tiling_validrª  r§  r«  rª  s   &&&               @@@@r[   r­  ÚSIMDScheduling.can_fuse  sp  û€ ô �eœY×AÑA×BÒBÄjØ”9×7Ñ7÷G
ò G
ô ×7Ñ7×@Ñ@ÀÓNÐNà$Ÿ{™{ÑˆÑˆF�GØ$Ÿ{™{ÑˆÑˆF�GÜ˜Ó%ˆà×Ñ× Ò ¨×)<Ñ)<×)>Ò)>Ø×!Ñ!×#Ò#ÙÐ<Ô=Ùð $ð × Ñ ×"Ò"¨5×+>Ñ+>×+@Ò+@Ø×!Ñ!×#Ò#ÙÐ<Ô=Ùà×Ñ×Ó E×$6Ñ$6×$8Ó$8Ø!'¨6Ñ!1×!HÐ!H°gÀÑ6HÐß%ÝGà%6×%?Ñ%?ÀÓ%MÐ"ç%õ
 Fà%4×%QÑ%QØó&÷ &=ð &=à%×.Ñ.¨uÓ<ð #÷ &ÙØGØØØØô÷ "Ø×&Ñ&×(Ò(¨E×,BÑ,B×,DÒ,Dð ×-Ñ-×/Ò/Ø#(˜5ð ×)Ñ)¨&°'Ó:�ß“sõ ð $Ÿo™oÔ/ó	—s—s’sõ ð $Ÿo™oÔ/ó	÷ ò ñ ÐIÔJÙ à%Ð%à×!Ñ!×#Ó#¨E×,>Ñ,>×,@Ó,@Ø˜fÔ$¨°GÔ);Ø×(Ñ(×*Ò*ÙØOØØØØôñ !ð !&§¡Ö 1˜à×+Ñ+×-Ò-Ù!ð  $×5Ñ5Ó7¸%×:PÑ:PÓ:R×RÔRÙ$Ø59·Z±ZÑ2˜Ñ2˜IØ &Ô 3¸À:Ö8MÙØ \Ø &Ø )Ø 'Ø *ôò $)ñ# !2ð& “^�Ø—=‘=—?”?Úñ $ð
 ×&Ñ& v¨wÓ7ˆGØ×&Ñ& v¨wÓ7ˆGØ×(Ñ(Ø—‘Ó! E§O¡OÓ$5Õ5°v¸wóˆGô �}‰}×=×=Ð=Ø�Ü�w“< !Ô#Ü˜7“| aÔ'Ø&×<Ô<°WÔ<™à&Ñ1™Ü˜“\ AÔ%Ø"Ñ-�DßÙØ6ØØØô	ñ !áà×!Ñ!×#Ó#¨×(:Ñ(:×(<Ó(<Ø˜q”L W°¤\Ü$ðØ"˜) 5¨¨	ð3óð ð ˜ 'Õ)Ô)ß“sõ à"Ÿ_™_Ô.ó—s—s’sõ à"Ÿ_™_Ô.ó÷ ò ñ Ð<Ô=Ù ä—M‘M×B×BÐBØ!×-Ñ-×/Ò/ä05Ø×(Ñ(¨´·±·±Ó=×DÑDÓFó1ð   ˜Ø ¨!Ð,ðñ1Ð-÷ 5ÙÐ:Ô;Ø4Ð4Ùà˜ÔÙÐ1Ô2Ø˜VÑ#Ð#à×!Ñ!×#Ò# u×'9Ñ'9×';Ò';Ü ØLóð ð ×'Ñ'¨Ó5Ð5r]   c           
     	ó.  aaaaaaa€ . o\         \        P                  ,          ! 4       o\        4       o\        4       oR oVV3R lpVV3R lpV3R lpVVVV3R lp\        P                  VVVV3R l4       pVV3R lp	V Fç  p
V
S9   d   K  SP                  V
4       V! V
4      '       da   V	! V
S4      '       d   V! 4       ;_uu_ 4         R R R 4       S'       d%   V! V
4      '       g   S;'       g    \        S4      oMR oV! V
4       K‹  V! V
4      '       d,   V! 4       ;_uu_ 4        SP                  V
4       R R R 4       KÄ  \        RS RS R	V
P                  ^,           24      h	  S#   + '       g   i     Lª; i  + '       g   i     EK  ; i)
Nc                óŒ   <€ V P                   w  pw  r#VS8H  ;'       d    VS8H  ;'       g    VSS,          8H  ;'       d    V^8H  # rò  ©r±  ©r1  rx  Ú
node_numelÚnode_rnumelrp   r¬  s   &   €€r[   Úfits_in_main_bodyÚ@SIMDScheduling.generate_node_schedule.<locals>.fits_in_main_bodyÐ  sQ   ø€ Ø+,¯7©7Ñ(ˆAÑ(�
Ø %Ñ'×AÐA¨K¸6Ñ,A÷ ð Ø˜e f�nÑ,×AÐA°ÀÑ1Aðr]   c                ód   <€ V P                   w  pw  r#VS8H  ;'       d    V^8H  ;'       d    S^8g  # rò  rÉ  rÊ  s   &   €€r[   Úfits_outside_reductionÚESIMDScheduling.generate_node_schedule.<locals>.fits_outside_reductionÖ  s8   ø€ Ø+,¯7©7Ñ(ˆAÑ(�
Ø Ñ&×KÐK¨;¸!Ñ+;×KÐKÀÈ!ÁÐKr]   c                ój   <€ V P                   P                   F  pVP                  S9   g   K   R # 	  R# rÅ  )Úread_writesÚreadsrj   )r1  ÚreadÚcurrent_loop_buffer_usages   & €r[   Úexpect_improved_memory_usageÚKSIMDScheduling.generate_node_schedule.<locals>.expect_improved_memory_usageÚ  s,   ø€ ØŸ™×+Ô+�Ø—9‘9Ð 9Ö9Úñ ,ñ r]   c                óÂ  <€ SP                  V 4       SP                  V 4       SP                  V P                  P                   Uu. uF  qP
                  NK  	  up4       V P                  4       '       d    \        V \        P                  4      '       d€   \        V P                  \        P                  4      '       dV   \        V P                  P                  \        P                  4      '       g"   SP                  V P                  4       4       R # SP                  V P                  P                    Uu. uF  qP
                  NK  	  up4       R # u upi u upi r_   )r+  r  ÚupdaterÓ  rÔ  rj   r„   r[  r   rÚ  rô   r   rÛ  ÚdataÚScanÚget_nameÚwrites)r1  r  rÖ  Údoner¨  Únot_ready_yet_nodess   & €€€€r[   Úschedule_node_in_loopÚDSIMDScheduling.generate_node_schedule.<locals>.schedule_node_in_loopà  sä   ø€ Ø�H‰H�QŒKØ× Ñ  Ô#Ø%×,Ñ,¸a¿m¹m×>QÒ>QÓ-RÑ>Q¸¯f¬fÑ>QÑ-RÔSð
 —‘× Ò Ü˜q¤)×"9Ñ"9×:Ò:Ü˜qŸv™v¤r×'8Ñ'8×9Ò9Ü" 1§6¡6§;¡;´·±×8Ò8à#×'Ñ'¨¯
©
«Ö5à)×0Ñ0À!Ç-Á-×BVÒBVÓ1WÑBV¸Q·&´&ÑBVÑ1WÖXùò .Sùò 2Xs   ÁEÄ:Ec               3  óz  <"  € S'       d#   SR,          \         J d   SP                  4        MSP                  \        4       S'       d6   SP	                  S\        4       SP	                  S^,           \         4       RoRx € SP                  \         4       SP                  4        S P                  4        R# 5i)rT  Nr  )rG   Úpopr  rF   ÚinsertÚclear)rÖ  Úmaybe_split_indexr¨  rà  s   €€€€r[   Úend_current_reduction_loopÚISIMDScheduling.generate_node_schedule.<locals>.end_current_reduction_loopñ  sŒ   øé € ÷  ¨rÕ!2´oÓ!EØ×!Ñ!Õ#à×$Ñ$Ô%5Ô6ß Ø×$Ñ$Ð%6Ô8HÔIØ×$Ñ$Ð%6¸Õ%:¼OÔLØ$(Ð!ÛØ× Ñ ¤Ô1Ø×%Ñ%Ô'Ø%×+Ñ+Ö-ùs   ƒAB;ÁA0B;c                óÎ   <€ S^8X  d   R# SV P                   ,          '       g   R# V'       d$   \        VR,          \        \        34      '       d   \	        R4      h\        S4      # )rT  FzAexpected non-empty node_schedule not ending in a reduction toggler  )Ú	ancestorsr[  rG   rF   r­   r�   )rô   r¨  rà  r¬  s   &&€€r[   Ú#requires_closing_previous_reductionÚRSIMDScheduling.generate_node_schedule.<locals>.requires_closing_previous_reduction	  s_   ø€ Ø˜Œ{ÙØ&¨¯©×7Ô7ÙçÜ"Ø! "Õ%¬Ô9IÐ'J÷ò ô %ØWóð ô Ð+Ó,Ð,r]   zunexpected group: (r²   z) != )
r   r   r.   r€  r�  r+  r_  r  rê  r±  )r|   r¬   rp   r¬  rÍ  rÐ  r×  rá  rè  rì  rô   rÖ  rß  rç  r¨  rà  s   &&ff       @@@@@r[   Úgenerate_node_scheduleÚ%SIMDScheduling.generate_node_scheduleÇ  s`  þ€ Ø#%ˆÜœ)×5Ñ5Ö6Ó8ˆô 0:«|ÐÜ5?³\Ð!Ø(,Ðö	ö	Lõ	÷	Yð 	Yô" 
×	"Ñ	"÷	.ó 
#ð	.ö	-ó  ˆDØ�tŒ|ÙØ�H‰H�TŒNá  ×&Ò&Ù6°t¸]×KÒKÙ3×5Õ5Ø÷ 6÷ -Ñ5QÐRV×5WÒ5Wà(9×(OÐ(O¼SÀÓ=OÑ%ð )-Ð%á% dÖ+Ù'¨×-Ò-Ù/×1Õ1Ø!×(Ñ(¨Ô.÷ 2Ñ1ô *Ø)¨%¨°°6°(¸%ÀÇ
Á
È1Å¸ÐOóð ñ- ð4 Ð÷' 6×5ú÷ 2×1Ñ1ús   ÃE/Ä*FÅ/E?ÆFc                	ó  € VP                   VP                  r2VP                  VP                  4       ,          '       g(   VP                  VP                  4       ,          '       d   \	        R 4      hV P                  W#4       R# )z3node1 and node2 must not be ancestors of each otherN)rº  r»  rë  Úget_operation_namesr­   Ú_codegen_mix_order_reduction)r|   rô   rº  r»  s   &&  r[   Úcodegen_mix_order_reductionÚ*SIMDScheduling.codegen_mix_order_reduction-	  s]   € Ø—z‘z 4§:¡:ˆuð —‘ 5×#<Ñ#<Ó#>×>Ô>Ø—_‘_ u×'@Ñ'@Ó'B×BÔBä Ð!VÓWÐWà×)Ñ)¨%Ö7r]   c                	ó¶   € VP                  4       p. p. pV F=  pVP                  4       '       d   VP                  V4       K,  VP                  V4       K?  	  W43# r_   )r²  r„   r  )r|   rô   r¬   Ú
reductionsÚ	epiloguess   &&   r[   Ú#_split_mix_order_reduction_epilogueÚ2SIMDScheduling._split_mix_order_reduction_epilogue9	  sX   € à—‘Ó ˆØˆ
Øˆ	ÛˆDØ× Ñ ×"Ò"Ø×!Ñ! $Ö'à× Ñ  Ö&ñ	 ð
 Ð$Ð$r]   c                óÖ  € VP                   VP                  rTVP                  pV P                  VRVRV/.RVRRRRRR/4      ^ ,          pVP                  '       g   \        R	4      hVP                  '       g   \        R
4      hW'n        V P                  Wg4       VP                  P                  \        VP                  4      VP                  R,          ,          VP                  R,          VP                  ,           ^,
          VP                  ,          ,          R\        P                  R7      w  r‰p
V
^ 8w  d   \        RV
: 24      hV;_uu_ 4        VP!                  4        RRR4       \"        P$                  ! 4       p\&        P(                  ! V4      ;_uu_ 4        T;_uu_ 4        V'       d'   VP+                  \,        P.                  ! RR7      4       VP1                  4       pRRR4       RRR4       V'       d*   XP3                  \5        \6        P8                  4      R4      pWyX3#   + '       g   i     LÒ; i  + '       g   i     L`; i  + '       g   i     Lk; i)z
for_benchmark:
    True if the generated code is for benchmarking. We need make
    sure benchmark harness code is generated.
r  r0  r­  r«  Nrº  Tr·  z$expected kernel.persistent_reductionz#expected kernel.mix_order_reductionF)r  zws_off=©Úbenchmark_kernelÚtriton_)rp   r¾  r¨  Úcreate_kernel_choicesrÔ  r­   rº  rß  Ú!codegen_node_schedule_with_kernelr]  Ú	workspacer_  râ  rÍ  r`   r˜  r{  r€  Ú	ExitStackr>   Úset_kernel_handlerÚenter_contextr   Úpatchrž  Úreplacerk   r6   ÚKERNEL_NAME)r|   Úkernel_featuresÚ
split_sizeÚfor_benchmarkrp   r¬  r¨  rs   rx  Úws_nameÚws_offÚstackÚsrc_codes   &&&&         r[   Ú-_generate_kernel_code_for_mix_order_reductionÚ<SIMDScheduling._generate_kernel_code_for_mix_order_reductionE	  sð  € ð (×-Ñ-¨×/NÑ/NˆvØ'×5Ñ5ˆà×+Ñ+ØØ�5˜% Ð(Ð)à˜OØ Ø% tØ/°ð	ó	
ð õ	ˆð ×*×*Ð*Ü Ð!GÓHÐHØ×)×)Ð)Ü Ð!FÓGÐGØ'ÔØ×.Ñ.¨}ÔEð $Ÿ[™[×2Ñ2Ü�×/Ñ/Ó0Ø�m‰m˜EÕ"õ#à—‘˜cÕ" V×%7Ñ%7Õ7¸!Õ;À×@RÑ@RÕRõTð Ü—+‘+ð 3ó 
Ñˆ�Fð �QŒ;Ü  G F¡9 Ó.Ð.ßŠVØ×ÑÔ!÷ ô ×$Ò$Ó&ˆÜ×!Ò! &×)Õ)¯5ª5ßØ×#Ñ#¤F§L¢LÀ$Ô$GÔHØ×,Ñ,Ó.ˆH÷ ,1×)÷
 ð
  ×'Ñ'¬¬K×,CÑ,CÓ(DÀiÓPˆHØ Ð(Ð(÷ �Vú÷ ,1¯5ú×)×)ús0   ÅH2Æ#IÆ.?I	Ç-IÈ2I	ÉIÉIÉI(	NFc               ó$   € V ^8„  d   QhRRRRRR/# )rU   Ú
node_nameszOrderedSet[str] | NoneÚskip_perf_cacher�   rX   ztuple[float, str]rY   )rZ   s   "r[   r\   ÚSIMDScheduling.__annotate__}	  s*   € ÷ "ñ "ð +ð	"ð
 ð"ð 
ñ"r]   c                	ó   € \         hr_   ré  )r|   ÚmodÚn_spills_thresholdr  r  s   &&&&&r[   Úbenchmark_codegened_moduleÚ)SIMDScheduling.benchmark_codegened_module}	  s
   € ô "Ð!r]   c                	óÎ  a aa%a&€ \         P                  P                  S4      w  o&pVV&3R  lpV! 4       p\        ;P                  ^,          un        S P                  V4      w  rg. pV FD  p	V	P                  4        V	P                  4       p
V
P                  4        VP                  V
4       KF  	  S P                  SP                  4       V,           S&V4      p\        VS&V4      o%\        P                  P                  P                   '       gˆ   \        P"                  P$                  fl   \        P"                  P&                  '       g-   \        P(                  '       g   \        P*                  '       d    V%V 3R lp\,        P.                  ! VV^4      pS P1                  S%VRR7      w  rÞp\3        V^ ,          P4                  P6                  4      p/ pV'       Ed†   V EF@  p	V	P9                  4       ^ ,          P4                  P;                  4       pV	P9                  4       ^ ,          P<                  ^ ,          P4                  P9                  4       ^ ,          P4                  P;                  4       pVVV&   S P                   '       g   \?        R4      hS P                   P@                  PC                  V	P9                  4       ^ ,          P<                  ^ ,          P4                  P;                  4       4       \D        PF                  PH                  PC                  V4       EKC  	  VPJ                   F.  pVPM                  VPN                  VPN                  4      Vn'        K0  	  S PQ                  WûV4      pVVn)        \U        V4      Vn*        \D        PV                  ! V4      ;_uu_ 4        S%PY                  4        FI  pVP9                  4       ^ ,          P4                  P;                  4       V9  g   K9  VP[                  4        KK  	  RRR4       \D        PF                  P\                  P_                  R4       S Pa                  VR4       VPc                  VPR                  RR7       \D        PF                  ;PH                  VPH                  ,          un$        \D        PF                  ;Pd                  VPd                  ,          un2        \g        V4      \g        VPJ                  4      8w  d.   \?        R\g        V4       R	\g        VPJ                  4       24      h\D        PF                  P\                  Pi                  S&V,           ^,
          V,          4      p\k        VPJ                  4       EF   w  ppVPN                  pR
V RV R2pV RV 2pR
V RV 2pRRRR/pVPM                  VPl                  VPl                  4      pV RV RV RV RV RV RV R2p\D        PF                  Po                  V4      p V es   V Pq                  4       Pr                   U!u. uF,  p!\D        PF                  P\                  Pi                  V!4      NK.  	  p"p!RRPu                  V"4       R2p#VRV# R2,          p\D        PF                  Pw                  V4      ;p$\        Px                  8w  d   VRV$ R2,          p\D        PF                  P\                  P{                  V4       \D        PF                  P\                  P|                  PC                  V4       EK£  	  VP                  4        V'       d   S P�                  V4       S Pƒ                  4        R#   + '       g   i     ELN; iu up!i )c                 ó†  <€ \         P                  P                  e   \         P                  P                  # \        P                  ! SP                  4       4      p V P                  pV^,          p\        P                  P                  P                  S4      p\        \        W2,          4      ^4      p\        V^€4      pV# r_   )r   rb   Úmix_order_reduction_split_sizer(   ÚcreateÚ
get_deviceÚmulti_processor_countr>   rý   rþ   r  Úmaxr+   Úmin)Údevice_propÚnum_smÚestimated_num_splitsÚ
numel_hintr  rº  rp   s        €€r[   Ú_pick_split_sizeÚESIMDScheduling._codegen_mix_order_reduction.<locals>._pick_split_size‰	  s–   ø€ ä�}‰}×;Ñ;ÒGÜ—}‘}×CÑCÐCô +×1Ò1°%×2BÑ2BÓ2DÓEˆKØ ×6Ñ6ˆFØ#)¨A¥:Ð ô
 Ÿ™×)Ñ)×;Ñ;¸EÓBˆJÜœ_¨ZÕ-OÓPÐRTÓUˆJÜ˜Z¨Ó-ˆJØÐr]   Nc                óˆ   <€ SP                  SV R R7      w   r\        P                  ! V4      pSP                  V4      w  rAV# )T©r  r	  )r  r   r|  r  )Úcandidate_split_sizerx  r  r  Úmsr  r|   s   &    €€r[   Ú_benchÚ;SIMDScheduling._codegen_mix_order_reduction.<locals>._bench»	  sO   ø€ Ø!%×!SÑ!SØ#Ø3Ø"&ð "Tó "‘��1ô
 "×&Ò& xÓ0�Ø×7Ñ7¸Ó<‘�Ø�	r]   Fr(  ú!expected self.scheduler to be setz!# Call mix order reduction kernel)r8  zKexpected len(converted_nodes) == len(kernel.saved_partial_accumulate), got r†  Ú(z) * (r@  z * z + 1) * r   Úaminr  Úamaxz = rð  z : z].view(r²   z).z(dim=0)rñ  z.view(z.to()Br   r¡  Úget_numel_rnumelr   ró  rø  Úcancel_reduction_splitÚextract_pw_from_reductionÚswap_pw_red_dimensionr  rî  r²  rJ   r`   ra   r   Údeterministicrb   r  Ú'mix_order_reduction_autotune_split_sizeÚmax_autotuneÚcoordinate_descent_tuningr'   Úautotune_single_fieldr  r�   rô   Ú_split_sizeÚget_outputsrÝ  Úusersr­   Úremoved_opsr+  r>   rý   Úremoved_buffersrâ  r.  r¡  Údefine_kernelr´  r   r  Úscheduler_nodesÚmark_runr0  Úmake_commentÚcodegen_commentr:  Úinplaced_to_remover_  Úcodegen_python_sizevarr*  r¢  rx  rs  r·  rò  r_  r˜  Ú	writelineÚ	allocatedr5  Ú_codegen_nodesÚfree_buffers_in_scheduler)'r|   rº  r»  r¬  r%  r  Únode2_reductionsÚnode2_epilogueÚconverted_nodesÚsubnodeÚ	convertedr¨  r+  rs   r
  r  Úis_split_reductionÚrenameÚbufnameÚusernameÚpartial_accumr´  rô   Únsplitr,  r¡  Ú
stride_strr   ÚendÚreduction_type2opÚopnameÚfinal_reduceÚbufferr0  r[  Úfinal_shape_strÚbuffer_dtyper  rp   s'   ff&                                  @@r[   rò  Ú+SIMDScheduling._codegen_mix_order_reduction†	  s2  û€ Ü!×3Ñ3×DÑDÀUÓK‰ˆˆvö	ñ$ &Ó'ˆ
ô 	×+Ò+¨qÕ0Õ+ð ,0×+SÑ+SØó,
Ñ(Ðð ˆÛ'ˆGØ×*Ñ*Ô,Ø×9Ñ9Ó;ˆIØ×+Ñ+Ô-Ø×"Ñ" 9Ö-ñ	 (ð
 ×3Ñ3Ø�O‰OÓ Õ/°¸ó
ˆô -¨]¸EÀ6ÓJˆô —‘×&Ñ&×4×4Ð4Ü—‘×<Ñ<ÒDä—‘×E×EÐEÜ×&×&Ð&Ü×3×3Ð3öô '×<Ò<ØØØóˆJð %)×$VÑ$VØØ!Øð %Wó %
Ñ!ˆ˜ô "Ð"2°1Õ"5×":Ñ":×"FÑ"FÓGÐØˆßÐÜ+�Ø!×-Ñ-Ó/°Õ2×7Ñ7×@Ñ@ÓB�à×'Ñ'Ó)¨!Õ,ß‘U˜1õç‘TŸ+™+›-¨õ+÷ ‘TŸ(™(›*ð	 ð #+��w‘Ø—~—~�~Ü(Ð)LÓMÐMØ—‘×*Ñ*×.Ñ.Ø×'Ñ'Ó)¨!Õ,×2Ñ2°1Õ5×:Ñ:×CÑCÓEôô —‘×'Ñ'×+Ñ+¨G×4ñ ,ð  "(×!@Ô!@�Ø,2¯J©JØ!×-Ñ-¨}×/HÑ/Hó-�Ö)ñ "Að
 ×(Ñ(¨À&ÓIˆØ(ˆÔÜ$ XÓ.ˆÔä×!Ò! &×)Õ)Ø'×7Ñ7Ö9�ð ×#Ñ#Ó% aÕ(×-Ñ-×6Ñ6Ó8ÀÖFØ—M‘M–Oñ :÷ *ô 	
�‰×Ñ×)Ñ)Ð*MÔNØ×Ñ˜]¨DÔ1à×Ñ˜6×-Ñ-¸UÐÔCÜ	�‰×Ò 6×#9Ñ#9Õ9ÕÜ	�‰×"Ò" f×&?Ñ&?Õ?Õ"ô ˆÓ¤3 v×'FÑ'FÓ#GÔGÜ ð=ä�Ó'Ð(¨¬c°&×2QÑ2QÓ.RÐ-SðUóð ô
 —‘×%Ñ%×<Ñ<Ø�ZÕ !Õ#¨
Õ2ó
ˆô #,¨F×,KÑ,K×"LÑˆC�Ø'×3Ñ3ˆKà˜V˜H E¨&¨°Ð3ˆJØ�e˜3˜z˜lÐ+ˆEØ�c�U˜( : ,Ð/ˆCà�vØ�vð!Ðð '×*Ñ*Ø×,Ñ,¨m×.JÑ.JóˆFð *˜]¨#¨g¨Y°a¸°w¸cÀ#ÀÀgÈfÈXÐUWÐX^ÐW_Ð_aÐbhÐaiÐipÐqˆLô —W‘W×'Ñ'¨Ó4ˆFØÒ!ð $×.Ñ.Ó0×5Ò5óá5˜ô —G‘G×(Ñ(×?Ñ?ÀÖBÙ5ð ð ð %& d§i¡i°Ó&<Ð%=¸QÐ"?�Ø &¨Ð(9¸Ð ;Õ;�ô !"§¡× 1Ñ 1°+Ó >Ð>�Ä5Ç;Á;ÔNØ $ | n°AÐ 6Õ6�Ü�G‰G× Ñ ×*Ñ*¨<Ô8ô �G‰G× Ñ ×*Ñ*×.Ñ.¨{×;ñC #MðF 	×$Ñ$Ô&çØ×Ñ Ô/à×&Ñ&Ö(÷C *×)Ð)üòXs   Î+A]Ï5]Ø2]"Ý]	c                ó:  a#€ VP                   VP                  r2VP                  w  pw  rVVP                  w  pw  rxVP                  p	VP                  \
        P                  P                  P                  J p
\        V	4      p\
        P                  P                  VVVP                  4      pVf   \        R4      h\        V4      p\
        P                  P                  P                  pV UUu. uF&  w  ppWòP!                  4       9   g   K  VVJ g   K$  VNK(  	  ppp\#        V4      pVP!                  4        Uu. uF  pVV9  g   K  VNK  	  ppVP$                  o#V P'                  VP!                  4       VV4      p\(        ;QJ d    V#3R lV 4       F  '       g   K   RM	  RM! V#3R lV 4       4      '       g   \        R4      hS#P*                  pV P'                  VVV4      p\,        P.                  P0                  P2                  P4                  '       d   VP7                  4       MRp\9        VVVV4      pV P;                  VVVV4      w  ppRV9   g   RV9   d   \        R	4      h\<        ;P>                  ^,          un        R
VRRRV/p\A        RV PC                  VV.V4      ^ ,          4      pV
'       d	   VVn"        MVVn#        V PI                  VV4       V;_uu_ 4        VPK                  4        \L        PO                  VV	V
4      pVPQ                  V4      pVPS                  V4      pVPU                  4       p V PW                  VVPX                  VPZ                  4      p!VP]                  V4      p"V P_                  VVV V!\a        VV4      R7       V Pc                  VVS#VVPX                  VPZ                  V!V"VVV 4       VPK                  4        RRR4       V Pe                  VVV. VOVOVO4       R# u uppi u upi   + '       g   i     L7; i)ae  
Generate a single kernel with an outer reduction, a group
reduction, and optional reduced/parent-full epilogues for
dependent cross-axis reductions that share a large input.

The local group may split either parent tree. For example,
group-in-R reshapes [XBLOCK, RBLOCK] into [XBLOCK, RBLOCK/G, G],
while group-in-X reshapes it into [XBLOCK/G, G, RBLOCK].
Nz2expected nested pointwise domains to be classifiedc              3  ó*   <"  € T F  qSJ x € K
  	  R # 5ir_   rY   )rX  ÚsnÚgrouped_reductions   & €r[   rY  Ú:SIMDScheduling.codegen_nested_reduction.<locals>.<genexpr>f
  s   øé € ÐFÑ5E¨rÐ*Õ*Ó5Eùs   ƒTFz1grouped reduction must appear in grouped scheduler2  r/  z2nested reduction does not support tiled reductionsr­  r¸  r«  rO   ©rk  )3rº  r»  r±  Ú
group_sizeÚgrouped_axisr   r¢  ÚGroupedAxisÚRrW   Ú _classify_nested_pointwise_nodesÚdomain_contextr­   r©  ÚPointwiseDomainÚLOCAL_REDUCTION_INPUTr²  r   ra  rî  rè  Ú_bodyr`   ra   r   rb   Úcoalesce_tiling_analysisÚget_coalesce_analysisrJ   Úget_tiling_and_scoresr   Úcodegen_nested_reductionr   rþ  rá  rà  rÿ  r{  rÚ  rã  r,  r<  r@  Ú!_local_reduction_iteration_valuesrÓ  rÔ  rF  Ú_codegen_remapped_pointwiserc  Ú _codegen_nested_grouped_scheduleÚ!_finalize_nested_reduction_kernel)$r|   rô   Ú
outer_nodeÚgrouped_noderx  Úouter_numelÚouter_rnumelÚgrouped_numelÚgrouped_rnumelrÞ  rß  Úlocal_reduction_size_hintÚnested_pointwise_domainsÚpointwise_domain_by_nodeÚlocal_reduction_domainr`  ÚdomainÚouter_local_reduction_pointwiseÚ#outer_local_reduction_pointwise_setÚouter_codegen_nodesÚgrouped_scheduleÚgrouped_reduction_bodyÚcombined_scheduleÚcoalesce_analysisr  rª  Útiling_scoreÚkernel_kwargsrs   r€  r/  Úreduced_output_familyÚparent_full_familyÚlocal_reduction_sourceÚparent_full_sourcera  s$   &&                                 @r[   rp  Ú'SIMDScheduling.codegen_nested_reduction2
  sq  ø€ ð $(§:¡:¨t¯z©z�LØ)3×)9Ñ)9Ñ&ˆÑ&ˆKØ-9×-?Ñ-?Ñ*ˆÑ*ˆMØ#Ÿ™Ðà×Ñ¤×!:Ñ!:×!FÑ!F×!HÑ!HÐHð 	ô %(Ð(<Ó$=Ð!ä×%Ñ%×FÑFØØØ×#Ñ#óð 	!ð $Ò+Ü Ð!UÓVÐVô Ð)Ó*ð 	!ô ×%Ñ%×5Ñ5×KÑKð 	ñ
 7ô+
á6‘
��FØ×)Ñ)Ó+Ñ+ô à06Ð:PÐ0P÷ ˆBÙ6ð 	(ñ +
ô
 /9Ø+ó/
Ð+ð
 !×*Ñ*Ô,ó
á,�ØÐ<Ñ<÷ ˆBÙ,ð 	ð 
ð
 6:×5KÑ5KÐØ48×4OÑ4OØ×"Ñ"Ó$ØØó5
Ð÷
 ‹sÔFÑ5EÓF�s�sŠsÔFÑ5EÓF×FÒFÜ Ð!TÓUÐUØ!2×!8Ñ!8Ðà ×7Ñ7ØØØó
Ðô �‰×%Ñ%×,Ñ,×E×EÐEð ×,Ñ,Ô.àð 	ô
 -ØØØØó	
ˆð  $×9Ñ9ØØØØó	 
Ñˆ�ð �&Œ=˜C 6œMÜ Ð!UÓVÐVä×(Ò(¨AÕ-Õ(à˜Ø,¨eØ˜\ð)
ˆô
 ØØ×&Ñ&ØØ�Øóð õ	ó
ˆ÷  Ø 9ˆFÕà 9ˆFÔð
 	×.Ñ.Ð/@À&ÔIçŠVð ×ÑÔ!ä.E×.QÑ.QØØ$Ø$ó/ˆFð ×5Ñ5Ð6LÓMð !ð ×1Ñ1Ð2FÓGð "ð ×.Ñ.Ó0ð ð ×6Ñ6Ø*Ø(×6Ñ6Ø(×8Ñ8óð #ð 39×2UÑ2UØ$ó3Ðð ×,Ñ,ØØ/Ø"Ø&Ü7¸ÀÓGð -ô ð ×1Ñ1ØØ Ø!ØØ$×2Ñ2Ø$×4Ñ4Ø&Ø"Ø(Ø%Ø"ôð ×ÑÔ!÷k ðn 	×.Ñ.ØØØðØ"ðà0ðð "ðö			
ùóG+
ùò
÷H �Vús+   Ã0O?ÄO?ÄO?Ä<	PÅ
PË*C1P
Ð
P	c               ó   € V ^8„  d   QhRR/# rð   rY   )rZ   s   "r[   r\   r  á
  s   € ÷ Fñ Fð 
ñFr]   c           	     	óÔ  € T P                  T;'       g    T4      p\        P                  ! V4      ;_uu_ 4        \        P                  ! R/ VB ;_uu_ 4        VP                  4       pR R R 4       R R R 4       V P                  XVV4      Vn        \        V4      Vn        \        P                  ! V4      ;_uu_ 4        VP                  4        F  pVP                  4        K  	  R R R 4       V Uu. uF  p\        V\        4      '       g   K  VNK  	  ppV P                  W4       R #   + '       g   i     LÊ; i  + '       g   i     LÕ; i  + '       g   i     Lr; iu upi rì   )Ú_collect_config_patchesr>   r  r   r  rž  r?  r´  r   r²  rA  r[  r.   Ú_launch_kernel_and_cleanup)	r|   rs   r…  rô   Úconfig_patch_scheduleÚconfig_patchesr  r`  Úbase_scheduler_nodess	   &&&&&    r[   rt  Ú0SIMDScheduling._finalize_nested_reduction_kernelá
  s  € ð ×5Ñ5Ø!×6Ð6Ð%6ó
ˆô ×!Ò! &×)Õ)¬6¯<ª<Ñ+I¸.×+IÓ+IØ×,Ñ,Ó.ˆH÷ ,J×)à!×/Ñ/ØØØó
ˆÔô
 % XÓ.ˆÔä×!Ò! &×)Õ)Ø—n‘nÖ&�Ø—‘–ñ '÷ *ñ
 /ó 
Ù.�T´*¸TÔCT×2U�DˆDÑ.ð 	ð  
ð 	×'Ñ'¨ÖE÷! ,J×+Iú×)×)ú÷ *×)üò 
s;   ºD?ÁD,	Á*D?Ã (EÃ4E%ÄE%Ä,D<Ä7D?Ä?E	ÅE"	c               ó0   € V ^8„  d   QhRRRRRRRRRR	R
R/# )rU   ra  úscheduler.SchedulerNoder€  rÚ  r‹  r¬  rŒ  r}  zBdict[BaseSchedulerNode, scheduler.NestedReduction.PointwiseDomain]rX   rw   rY   )rZ   s   "r[   r\   r  ý
  sP   € ÷ Pñ Pð 3ð	Pð
 (ðPð !0ðPð ,ðPð#
ðPð 
ñPr]   c           	     ón  € VP                   p\        VP                   Uu. uF  pVP                  V,          NK  	  upV4      p\	        W4      pV EFZ  pV\
        \        39   d   K  \        V\        P                  4      '       g   \        R\        V4       24      hVVJ d   V P                  VVVVVV
4       Kl  V	P                  V4      pVf   \        RV 24      hV\        P                  P                  P                   J d   V P#                  VV.V
V4       KÎ  V\        P                  P                  P$                  J d   TpMJV\        P                  P                  P&                  J d   V P)                  VVV4      pM\        RV 24      hV P#                  VV.VVVR7       EK]  	  R# u upi )a  Interpret the local reduction schedule with nested emitters.

``local_reduction_source`` is the pre-reduction domain, including the
local reduction lane. ``parent_full_source`` is the outer tile domain
used by pointwise consumers after reduced values are broadcast back.
zexpected SchedulerNode, got Nz#no pointwise domain classified for z$unexpected nested pointwise domain: rc  )rl  r¬  r*  rn   rc  rF   rG   r[  r   rÚ  r­   r\  Ú_codegen_grouped_reductionr.  r¢  rj  ÚREDUCEDrr  rk  ÚPARENT_FULLÚ(_select_full_resolution_pointwise_source)r|   rs   rƒ  ra  r€  rÓ  rÔ  r‹  rŒ  r}  r‰  rŠ  r„  rH  Úreduced_sourceÚparent_full_load_transformr`  r  Úsources   &&&&&&&&&&&&       r[   rs  Ú/SIMDScheduling._codegen_nested_grouped_scheduleý
  sÌ  € ð, "3×!8Ñ!8ÐÜ(ð 0×9Ò9óá9�Að '×1Ñ1°!×4Ð4Ù9ñð ó
ˆô &>¸fÓ%MÐ"Ü"ˆBØÔ&¬Ð8Ô8ñ
 Ü˜b¤)×"9Ñ"9×:Ò:Ü$Ð'CÄDÈÃHÀ:Ð%NÓOÐOØÐ&Ó&Ø×/Ñ/ØØØØ!Ø#Ø)ôñ Ø-×1Ñ1°"Ó5ˆFØŠ~Ü$Ð'JÈ2È$Ð%OÓPÐPØœ×2Ñ2×BÑB×JÑJÓJØ×0Ñ0ØØ�DØ)Ø"ô	ñ àÜ×,Ñ,×<Ñ<×RÑRóSð 0‘Øœ9×4Ñ4×DÑD×PÑPÓPØ×FÑFØØ&Ø*ó‘ô %Ð'KÈFÈ8Ð%TÓUÐUØ×,Ñ,ØØ�Ø"ØØ9ð -÷ óW #ùòs    F2c               ó(   € V ^8„  d   QhRRRRRRRR/# )rU   r`  r—  rŒ  r¬  r‹  rX   rY   )rZ   s   "r[   r\   r  O  s2   € ÷ 
ñ 
à#ð
ð ,ð
ð !0ð	
ð
 
ñ
r]   c                	ón  € VP                  4       pV P                  VP                  V4      '       d   V# V P                  VP                  V4      '       d   V# \        P	                  VP                  V4      '       d   V# \        P	                  VP                  V4      '       d   V# \        R V 24      h)z5unsupported full-resolution nested pointwise ranges: )r¥  Ú_ranges_exactly_match_groupsr‹  rt   rÆ  r­   )r|   r`  rŒ  r‹  Ú	sn_rangess   &&&& r[   rœ  Ú7SIMDScheduling._select_full_resolution_pointwise_sourceO  s¦   € ð —M‘M“Oˆ	Ø×,Ñ,Ð-?×-FÑ-FÈ	×RÒRØ%Ð%Ø×,Ñ,Ð-C×-JÑ-JÈI×VÒVØ)Ð)Ü×#Ñ#Ð$6×$=Ñ$=¸y×IÒIØ%Ð%Ü×#Ñ#Ð$:×$AÑ$AÀ9×MÒMØ)Ð)ÜØCÀIÀ;ÐOó
ð 	
r]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   r‹  rD  r‡  r�  rX   r�   rY   )rZ   s   "r[   r\   r  c  s&   € ÷ 
ñ 
Ø$ð
à.ð
ð 
ñ
r]   c                	óò   € Vw  r#V'       * ;'       df    \        V4      \        V 4      8H  ;'       dG    \        ;QJ d%    R  \        W 4       4       F  '       d   K   R# 	  R# ! R  \        W 4       4       4      # )c              3  óx   "  € T F0  w  r\         P                  P                  P                  W4      x € K2  	  R # 5ir_   )r>   rý   rþ   rÿ   )rX  rå  r±  s   &  r[   rY  Ú>SIMDScheduling._ranges_exactly_match_groups.<locals>.<genexpr>k  s2   é € ð á)AÑ%�Jô —‘× Ñ ×8Ñ8¸×KÐKÛ)Aùs   ‚8:FT)r_  r`  rr  )r‹  r‡  Úiter_rangesÚreduction_rangess   &&  r[   r£  Ú+SIMDScheduling._ranges_exactly_match_groupsb  s‚   € ð
 )/Ñ%ˆà Ô ÷ ð Ü�KÓ ¤C¨£KÑ/÷ð ç“ñ ä),¨[Ô)Aó—”ð	
ò ð	
ð ñ ä),¨[Ô)Aóó ð	
r]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   ra  r—  r€  rÚ  rX   rw   rY   )rZ   s   "r[   r\   r  q  s*   € ÷ ñ ð 3ðð (ð	ð 
ñr]   c           
     	óp  € VP                   p\        W4      p\        \        P                  ! 4       VVVVR 7      p	\        P
                  ! V	4      ;_uu_ 4        VP                  V4      ;_uu_ 4        V! VVRR7       RRR4       RRR4       R#   + '       g   i     L; i  + '       g   i     R# ; i))rs   r€  rm  rk  T)Úallow_same_symbol_in_indexN)rl  rc  rj  r>   Úget_ops_handlerÚset_ops_handlerÚset_current_node)
r|   rs   ra  r€  rÓ  rÔ  r‰  r„  rk  Úhandlers
   &&&&&&&   r[   r™  Ú)SIMDScheduling._codegen_grouped_reductionq  sŽ   € ð "3×!8Ñ!8ÐÜ1°&ÓAˆÜ-Ü×ÒÓØØØ(Ø)ô
ˆô ×Ò˜w×'Õ'¨×)@Ñ)@ÐAR×)SÕ)SÙ"ØØØ+/õ÷ *T×'Ñ'×)S×)Sú×'×'Ð'ús$   ÁB$Á3B	Á?B$ÂB!ÂB$Â$B5	c               ó$   € V ^8„  d   QhRRRRRR/# )rU   rŸ  r¬  r  r�  rX   rÉ  rY   )rZ   s   "r[   r\   r  ‹  s(   € ÷ 
ñ 
àð
ð 0ð
ð 
 ñ	
r]   c                	óì   a€ VP                   pVP                  o\        V4      \        S4      8w  d$   \        R \        V4       R\        S4       24      hR V3R llp\        P                  VVV4      # )zgroups/values length mismatch: z != c               ó    € V ^8„  d   QhRRRR/# )rU   r—  rD  rX   rÉ  rY   )rZ   s   "r[   r\   ÚHSIMDScheduling._map_iteration_values_to_node_sizes.<locals>.__annotate__—  s   € ÷ 	ñ 	Ø-ð	à#ñ	r]   c                 ób   <€ \        SV 4       UUu. uF  w  r\        W4      NK  	  upp# u uppi r_   )rr  r2   )r—  r”  r‡  rò   s   *  €r[   Úsplit_values_by_rangesÚRSIMDScheduling._map_iteration_values_to_node_sizes.<locals>.split_values_by_ranges—  s8   ø€ ô
 &)¨°Ô%<ôá%<‘M�Eô   Ö.Ù%<òð ùó s   ‘+)r‹  rò   r_  r­   rt   rË  )rÀ  rŸ  r  r‹  rº  rò   s   &&&  @r[   Ú#_map_iteration_values_to_node_sizesÚ2SIMDScheduling._map_iteration_values_to_node_sizesŠ  st   ø€ ð —‘ˆØ—‘ˆÜˆv‹;œ#˜f›+Ô%Ü Ø1´#°f³+°¸dÄ3ÀvÃ;À-ÐPóð ÷	ð 	ô ×9Ñ9ØØØ"ó
ð 	
r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   r¬  rY   )rZ   s   "r[   r\   r  ¥  s   € ÷ Kñ Kð
 
ñKr]   c                	ó¢   € . VP                   OVP                  OpV Uu. uF  qQP                  V,          NK  	  pp\        V. VOVO4      # u upi r_   )r*  Úreduce_varsrn   r¬  )r|   rÊ  rÓ  rÔ  Ú	body_varsrH  r‹  s   &&&&   r[   rq  Ú0SIMDScheduling._local_reduction_iteration_values¥  sX   € ð 9�d—n‘nÐ8 t×'7Ñ'7Ð8ˆ	Ù.7Ó8©i¨—/‘/ !×$Ð$©iˆÐ8Ü˜vÐ'I¨Ð'I¸Ð'IÓJÐJùò 9s   ŸArk  c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rU   rm  r®  rŸ  r¬  rk  rn  rX   rw   rY   )rZ   s   "r[   r\   r  ¯  s4   € ÷ (ñ (ð (ð	(ð
  ð(ð 8ð(ð 
ñ(r]   c               ó:  € VP                  V4      ;_uu_ 4        V F·  pV P                  VVP                  4       4      w  rxV'       d   \        RV 24      h\	        \
        P                  ! 4       VVVR7      p	\
        P                  ! V	4      ;_uu_ 4        VP                  V4      ;_uu_ 4        VP                  V4       RRR4       RRR4       K¹  	  RRR4       R#   + '       g   i     L&; i  + '       g   i     Ké  ; i  + '       g   i     R# ; i)z¯Emit pointwise nodes under an explicit nested iteration family.

`source` describes the coordinate space whose values are remapped into
each pointwise node's own loop ranges.
z0pointwise node must have no reduction vars, got )rs   rm  rk  N)
rÃ  r¼  r¥  r­   r‹  r>   r°  r±  r²  rl  )
r|   rs   Úpointwise_nodesrm  rŸ  rk  r`  r*  Úreduction_varsr³  s
   &&&&&$    r[   rr  Ú*SIMDScheduling._codegen_remapped_pointwise¯  sÝ   € ð �_‰_˜V×$Õ$Û%�Ø,0×,TÑ,TØØ—M‘M“Oó-Ñ)�	÷ "Ü(ØJÈ>ÐJZÐ[óð ô 1Ü×%Ò%Ó'Ø!Ø!Ø#1ô	�ô ×&Ò& w×/Õ/°×1HÑ1HÈ×1LÕ1LØ—H‘H˜YÔ'÷ 2M×/Ñ/ñ &÷ %Ñ$÷  2M×1Lú×/×/Ð/ú÷! %×$Ð$ús<   šA>D	ÂC5Â2C"ÃC5ÃD	Ã"C2Ã-C5Ã5DÄ 	D	Ä	D	c               ó    € V ^8„  d   QhRRRR/# )rU   r¬   z!Sequence[scheduler.SchedulerNode]r†  úCoalesceVarAnalysis | NonerY   )rZ   s   "r[   r\   r  Ð  s   € ÷ 
ñ 
à0ð
ð 6ñ
r]   c                	ó�  € V P                   '       g   \        R 4      hV Uu. uF/  q3P                  4       V P                   P                  9  g   K-  VNK1  	  ppV'       g   R# \	        VR R7      P
                  w  pw  rVV P                  WV4      p\        P                  RV4       V P                  \        WuWb4      4      # u upi )r-  Nc                ó4   € \        V P                  4       4      # r_   ©rW   r„   ©r  s   &r[   r$  Ú/SIMDScheduling._codegen_nodes.<locals>.<lambda>Ü  s   € ´c¸!¿.¹.Ó:JÔ6Kr]   r&  zSchedule:
 %s)r   r­   rÝ  r=  r  r±  rî  Úschedule_logÚdebugÚcodegen_node_schedulerJ   )r|   r¬   r†  rô   rx  rp   r¬  r¨  s   &&&     r[   rH  ÚSIMDScheduling._codegen_nodesÐ  s³   € ð
 �~�~ˆ~Ü Ð!DÓEÐEá"ó
Ù"�T§m¡m£o¸T¿^¹^×=WÑ=WÑ&W�DˆD™Uð 	ð 
÷ ÙÜ  Ñ,KÔL×RÑRÑˆ‰?ˆEà×3Ñ3°EÀ&ÓIˆÜ×ÑÐ+¨]Ô;à×)Ñ)Ü˜}°VÓOó
ð 	
ùò
s   ¢*CÁCc               ó   € V ^8„  d   QhRR/# )rU   rô   z6scheduler.FusedSchedulerNode | scheduler.SchedulerNoderY   )rZ   s   "r[   r\   r  å  s   € ÷ =ñ =ØJñ=r]   c                óx  € V P                   '       g   \        R4      hVP                  4        Uu. uF0  pVP                  4       V P                   P                  9  g   K.  VNK2  	  pp\        V4      ^ 8X  d   R# \        P                  P                  P                  P                  '       dw   \        V4      \        XP                  4       4      8w  d?   V P                   '       g   \        R4      h\         P                  ! V P                   V4      pVP                  4       pMRpV P                  W#4      # u upi )z9
Given a set of pre-fused nodes, generate a SIMD kernel.
r-  N)r   r­   r²  rÝ  r=  r_  r`   ra   r   rb   rm  ÚFusedSchedulerNodern  rH  )r|   rô   r¬   r†  s   &&  r[   Úcodegen_nodeÚSIMDScheduling.codegen_nodeå  sö   € ð �~�~ˆ~Ü Ð!DÓEÐEð Ÿ™Ô(ó
á(�Ø�}‰}‹ d§n¡n×&@Ñ&@Ñ@÷ ˆDÙ(ð 	ð 
ô
 ˆu‹:˜Œ?Ùä�?‰?×!Ñ!×(Ñ(×A×AÐAÜ�5‹zœS §¡Ó!1Ó2Ô2Ø—~—~�~Ü(Ð)LÓMÐMÜ ×3Ò3°D·N±NÀEÓJ�Ø $× :Ñ :Ó <Ñà $Ðà×"Ñ" 5Ó<Ð<ùò#
s   °+D7Á D7c               ó$   € V ^8„  d   QhRRRRRR/# )rU   rp   rq   ÚbufferszCIterable[ir.Buffer | ir.TensorBox | ir.TorchBindObject | ir.IRNode]rX   r�   rY   )rZ   s   "r[   r\   r    s&   € ÷ "ñ "Øð"àTð"ð 
ñ"r]   c                	ó”  € \         P                  ! \         P                  4      P                  p\	        V 4      '       g   R # V Uu. uF9  pVP                  4       '       g   K  VP                  4       P                  4       NK;  	  ppV Fš  pVP                  4       '       d   K  \        V\        P                  4      '       g   K=  VP                  4       pTV Uu. uF9  pVP                  4       '       g   K  VP                  4       P                  4       NK;  	  up,          pKœ  	  \        ;QJ d    R V 4       F  '       d   K   R M	  RM! R V 4       4      '       g   R # \        P                  P                  P!                  W4       V F,  p\        P                  P                  P!                  Wb4       K.  	  R# u upi u upi )Fc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5ir_   )r3   )rX  r·  s   & r[   rY  Ú8SIMDScheduling.can_use_32bit_indexing.<locals>.<genexpr>  s   é € ÐF¹I°DÔ)¨$×/Ð/»Iùr	  T)r`   ÚiinfoÚint32r  r3   Úhas_tensor_outputrs  Ústorage_sizer[  r   ÚMutationOutputÚget_mutation_buffersr`  r>   rý   rþ   Ú	check_leq)rp   rÙ  Úint_maxr  Ú	buf_sizesÚmutated_bufsr·  s   &&     r[   Úcan_use_32bit_indexingÚ%SIMDScheduling.can_use_32bit_indexing   sW  € ô
 —+’+œeŸk™kÓ*×.Ñ.ˆä% e×,Ò,Ùñ ó
á�Ø×$Ñ$×&ô ,ˆC�N‰NÓ×)Ñ)Ö+Ùð 	ð 
ó ˆCØ×(Ñ(×*Ô*¬z¸#¼r×?PÑ?P×/QÔ/QØ"×7Ñ7Ó9�Øá+óá+˜Ø×,Ñ,×.ô 4�C—N‘NÓ$×1Ñ1Ö3Ù+ñõ ’	ñ ÷ ‹sÑF¹IÓF�s�sŠsÑF¹IÓF×FÒFÙô 	
�‰×Ñ×"Ñ" 5Ô2ÛˆDÜ�G‰G×Ñ×&Ñ& tÖ5ñ áùò/
ùòs   ÁG Á#"G ÃGÃ8"Gc               ó(   € V ^8„  d   QhRRRRRRRR/# )	rU   rs   r   r¨  úlist[NodeScheduleEntry]Úonly_gen_src_coder�   rX   rw   rY   )rZ   s   "r[   r\   r  %  s8   € ÷ @ñ @àð@ð /ð@ð  ð	@ð
 
ñ@r]   c                óÊ  € V P                  W!4       V'       gU   \        P                  ! V4      ;_uu_ 4        \        P                  ! V4       F  pVP                  4        K  	  RRR4       \        P                  ;P                  VP                  ,          un        \        P                  ;P                  VP                  ,          un        R#   + '       g   i     Lt; i)zU
Process a kernel by generating code for its node schedule and updating graph state.
N)	rÿ  r>   r  rI   rT  rA  rý   r>  rD  )r|   rs   r¨  rë  rô   s   &&&& r[   Úprocess_kernelÚSIMDScheduling.process_kernel%  s‹   € ð 	×.Ñ.¨}ÔEß Ü×%Ò% f×-Õ-Ü.×9Ò9¸-ÖH�DØ—M‘M–Oñ I÷ .ô 	
�‰×Ò 6×#9Ñ#9Õ9ÕÜ	�‰×"Ò" f×&?Ñ&?Õ?×"÷	 .×-ús   ¸.CÃC"	c               ó    € V ^8„  d   QhRRRR/# )rU   r¨  rõ  rX   zdict[str, Any]rY   )rZ   s   "r[   r\   r  6  s   € ÷ ñ °Yð À>ñ r]   c                óÖ   € / pV F`  p\        V\        4      '       g   K  VP                  f   K+  VP                  P                  4       pV'       g   KO  VP	                  V4       Kb  	  V# )z¼Collect and merge config_patches from all operations in the node schedule.

This enables scoped config (e.g., coordinate_descent_tuning) for kernels
that contain decomposition operations.
)r[  r.   rô   Úget_config_patchesrÚ  )r|   r¨  Úmerged_patchesrô   Úpatchess   &&   r[   r�  Ú&SIMDScheduling._collect_config_patches6  sW   € ð *,ˆÛ!ˆDÜ˜$Ô 1×2Ô2°t·y±yÔ7LØŸ)™)×6Ñ6Ó8�ß‘7Ø"×)Ñ)¨'Ö2ñ	 "ð
 Ðr]   c               ó   € V ^8„  d   QhRR/# )rU   r  rJ   rY   )rZ   s   "r[   r\   r  D  s   € ÷ J)ñ J)Ð5Gñ J)r]   c           
     óÄ  € VP                   pV P                  VVP                  VP                  VP                  4      w  r4V P                  VV.RVRV/4      pV F  pV P                  W&4       K  	  \        P                  ! V4       V P                  V4      pV FŸ  p\        P                  ! V4      ;_uu_ 4        \        P                  ! R/ VB ;_uu_ 4        VP                  4       pRRR4       RRR4       V P                  XW&4      p	\         P#                  RV	4       W–n        \'        V4      Vn        K¡  	  ?\)        V4      ^8”  d   \        V4      p
MVw  p
\        P                  ! V
4      ;_uu_ 4        VP+                  4        F  pVP-                  4        K  	  RRR4       V Uu. uF  p\/        V\0        4      '       g   K  VNK  	  ppV P3                  V
VRR7       \        P4                  P6                  P8                  '       dû   \        P:                  '       då   V^ ,          P<                  P?                  4       pVP+                  4        F¯  pVPA                  4       pWí9  d   K  VPB                  f   \E        R4      hVPB                  PG                  4       pVf   KT  \H        R,          R	;;,          ^,          uu&   \        P4                  P6                  PK                  R
VPL                  : RV R24       K±  	  V PO                  4        R#   + '       g   i     EL=; i  + '       g   i     ELI; i  + '       g   i     EL¬; iu upi )z,
Generate code for nodes in kernel_features
r­  r«  Nz+Generating kernel code with kernel_name: %sF)Úfree_buffersz!expected node.node to not be NoneÚinductorÚintermediate_hookszrun_intermediate_hooks(r²   r@  rY   )(r¨  ro  rp   r¾  r†  rþ  rÿ  rD   Úmerge_workspaces_inplacer�  r>   r  r   r  rž  r?  rv  rÐ  r´  r   r_  r@  rA  r[  r.   r‘  rý   r0  Úsupports_intermediate_hooksÚgenerate_intermediate_hooksr]  Úlive_output_buffersrÝ  rô   r­   Úget_origin_noder   rF  rj   rI  )r|   r  r¨  rª  r‡  Úkernelsrs   r“  r  r´  Úfinal_kernelrô   r”  Ú	live_outsrj   Úorigin_nodes   &&              r[   rÑ  Ú$SIMDScheduling.codegen_node_scheduleD  sÃ  € ð (×5Ñ5ˆà#×9Ñ9ØØ×!Ñ!Ø×+Ñ+Ø×-Ñ-ó	 
Ñˆð ×,Ñ,ØØˆHØ˜¨/¸<ÐHó
ˆó
 ˆFØ×2Ñ2°=ÖIñ ä×,Ò,¨WÔ5ð ×5Ñ5°mÓDˆãˆFÜ×%Ò% f×-Õ-¬v¯|ª|Ñ/M¸n×/MÓ/MØ!×0Ñ0Ó2�÷ 0N×-à×,Ñ,¨X°}ÓMˆKÜ�I‰IÐCÀ[ÔQØ!,ÔÜ(¨Ó2ˆFÖñ ð ô ˆw‹<˜!ÔÜ& wÓ/‰Là%‰Oˆ\ä×!Ò! ,×/Õ/Ø'×7Ñ7Ö9�Ø—‘–ñ :÷ 0ñ +ó9
Ù*�T¬j¸Ô?P×.Q�DˆD™]ð 	ð 9
ð 	×'Ñ'ØØ Øð 	(ô 	
ô �G‰G× Ñ ×<×<Ð<Ü×2×2Ð2ð   �
Ÿ™×;Ñ;Ó=ˆIØ'×7Ñ7Ö9�Ø—}‘}“�ØÔ(ÙØ—9‘9Ò$Ü(Ð)LÓMÐMØ"Ÿi™i×7Ñ7Ó9�ØÔ*Ü˜ZÕ(Ð)=×>À!ÕCÓ>Ü—G‘G×(Ñ(×2Ñ2Ø1°+×2BÑ2BÑ1EÀRÈÀvÈQÐOöñ :ð 	×&Ñ&Ö(÷a 0N×/MÐ/Mú×-×-Ð-ú÷ 0×/Ð/üò
9
s<   Â=L5ÃL!Ã-L5Å>(M	Æ2MÇMÌ!L2Ì,	L5Ì5MÍ	M	r÷  Tc               ó$   € V ^8„  d   QhRRRRRR/# )rU   r”  úlist[BaseSchedulerNode]r÷  r�   rX   rw   rY   )rZ   s   "r[   r\   r  �  s*   € ÷ -ñ -ð 6ð-ð
 ð-ð 
ñ-r]   c               	ó   € V P                  W!P                  4       \        P                  P                  '       d)   \
        P                  P                  P                  4        \        P                  P                  '       dU   \        P                  P                  '       d5   \
        P                  P                  P                  VP                  V4       VP                  VP                  4       \        P                  P                  '       d)   \
        P                  P                  P                  4        \        P                  '       d   VP                  4        \        P                  '       d   VP                  VP                  4       \
        P                  ;P                   VP                   ,          un        \
        P                  ;P"                  VP"                  ,          un        V'       d   V P%                  4        R # R # r_   )rC  r´  r   ÚcppÚenable_kernel_profiler>   rý   r0  Ú write_kernel_context_guard_beginÚenable_kernel_context_guardÚwrite_kernel_context_guardr:  Úwrite_kernel_context_guard_endÚnan_assertsr(  r‡  r>  rD  rI  )r|   rs   r”  r÷  s   &&&$r[   r‘  Ú)SIMDScheduling._launch_kernel_and_cleanup�  s@  € ð 	×ÑÐ1×3EÑ3EÔFÜ�:‰:×+×+Ð+Ü�G‰G× Ñ ×AÑAÔCÜ�:‰:×+×+Ð+´·
±
×0V×0VÐ0VÜ�G‰G× Ñ ×;Ñ;Ø×"Ñ"Ø$ôð 	×Ñ˜6×-Ñ-Ô.Ü�:‰:×+×+Ð+Ü�G‰G× Ñ ×?Ñ?ÔAä××ÐØ×$Ñ$Ô&Ü×!×!Ð!Ø×"Ñ" 6×#5Ñ#5Ô6ä	�‰×Ò 6×#9Ñ#9Õ9ÕÜ	�‰×"Ò" f×&?Ñ&?Õ?Õ"çØ×*Ñ*Ö,ñ r]   c               ó    € V ^8„  d   QhRRRR/# )rU   r  rJ   rX   zlist[SIMDKernel]rY   )rZ   s   "r[   r\   r  ®  s   € ÷ 
ñ 
Ø1ð
à	ñ
r]   c                	ó(   € V P                   ! V/ VB .# r_   )r™  )r|   r  Úkernel_argsrˆ  s   &&&&r[   rþ  Ú$SIMDScheduling.create_kernel_choices®  s'   € ð ×ÒØðàñð
ð 	
r]   c           
     	ó¨  € T;_uu_ 4        \         P                  ! 4       p/ pV FÄ  pV\        J d"   VP                  VP	                  4       4       K.  V\
        J d   VP                  4        KJ  VP                  4        VP                  VP                  4       4      pVP                  \        P                  VP                  P                  V4      P                  4       4      4       KÆ  	  VP!                  VP#                  4       4       V F¤  pV\        J d"   VP                  VP	                  4       4       K.  V\
        J d   VP                  4        KJ  \%        VP                  4       \'        VP                  4       VP                  VP                  4       4      pVP)                  V4       K¦  	  R R R 4       R #   + '       g   i     R # ; ir_   )r€  r  rF   r  r‚  rG   ÚcloseÚdecide_inplace_updaterÍ  r¥  rÚ  r©  Úfromkeysrl  Úindexing_from_argsrò   rF  Úkeysr&   r%   r<  )r|   r¨  rs   r  Úall_indexingrô   r)  s   &&&    r[   rÿ  Ú0SIMDScheduling.codegen_node_schedule_with_kernel¸  sN  € ßŠVÜ×(Ò(Ó*ˆEØˆLó &�ØÔ+Ó+Ø×'Ñ'¨×(@Ñ(@Ó(BÖCØœ_Ó,Ø—K‘K–Mà×.Ñ.Ô0Ø!'×!<Ñ!<¸T¿_¹_Ó=NÓ!O�JØ ×'Ñ'ÜŸ™Ø ŸJ™J×9Ñ9¸*ÓE×LÑLÓNóöñ &ð ×$Ñ$ \×%6Ñ%6Ó%8Ô9ó &�ØÔ+Ó+Ø×'Ñ'¨×(@Ñ(@Ó(BÖCØœ_Ó,Ø—K‘K–Mô 6°d·j±jÔAÜ4°T·Z±ZÔ@Ø!'×!<Ñ!<¸T¿_¹_Ó=NÓ!O�JØ—L‘L Ö,ñ &÷- �V�VŠVús   ‹F+G Ç G	rë  c          	     ó  € / pVP                  4       p. p	V F   p
V
P                  4       pV	P                  V
4       W¸,          '       g   K4  \        V4      ^8w  d   \	        R\        V4       24      hW—\        \        V4      4      &   VP                  P                  \        \        V4      4      4       . p	K¢  	  \        V	4      ^ 8w  d   \	        R\        V	4       24      hVP                   F)  pVP                  P                  P                  VR4       K+  	  VP                  V VVVV\        V4      p\        P                  '       dM   VP!                  4       R,          pVP#                  4        RV RVP%                  V4      P'                  4        2p. VOVNVOpV'       d   V# \)        VP+                  4        Uu. uF  p\-        V4      NK  	  up4      p\.        P0                  ! V4      ;_uu_ 4        VP3                  4        V F%  p\-        V4      V9  g   K  VP3                  4        K'  	  V F  pVP3                  4        K  	  RRR4       V P5                  WßV4      Vn        V# u upi   + '       g   i     L.; i)z;
Helper method to codegen a single template kernel variant
zexpected len(names) == 1, got z#expected empty prologue_group, got Ng    eÍÍAro  )r´  rµ  r  r_  r­   r   ÚiterÚprologue_fused_inputsr+  r]  rp  rä  rò  r   r   rü  ri  Úimports_for_benchmark_kernelÚcodegen_kernel_benchmarkÚgetvaluer   r÷  Úidr>   r  rA  r?  r´  )r|   rs   rñ  rì  rí  rî  rë  rï  Útemplate_readsÚprologue_groupÚprologueÚnamesÚbuf_namer  Únum_gbr¨  r1  Úunfused_setrô   s   &&&&&&$            r[   Ú_codegen_single_templateÚ'SIMDScheduling._codegen_single_templateÛ  sF  € ð &(Ð"Ø&×8Ñ8Ó:ˆØˆÛ&ˆHØ×-Ñ-Ó/ˆEØ×!Ñ! (Ô+à×%Ö%Ü�u“: ”?Ü(Ð+IÌ#ÈeË*ÈÐ)VÓWÐWØ@N¬4´°U³Ó+<Ñ=Ø×,Ñ,×0Ñ0´´d¸5³kÓ1BÔCØ!#’ñ 'ô ˆ~Ó !Ô#Ü Ø5´c¸.Ó6IÐ5JÐKóð ð ×4Ô4ˆHØ�K‰K×%Ñ%×)Ñ)¨(°DÖ9ñ 5ð ×/Ñ/ØØØØØ&Ü(Øó
ˆô ×"×"Ð"Ø×5Ñ5Ó7¸#Õ=ˆFà×6Ñ6Ó8Ð9¸Ø�*˜BØ×2Ñ2°6Ó:×CÑCÓEÐFðHð ð J˜.ÐI¨-ÐI¸.ÐIˆçØˆOô !°×1MÑ1MÔ1OÓ!PÑ1O¨A¤" Q¦%Ñ1OÑ!PÓQˆÜ×!Ò! &×)Õ)Ø×"Ñ"Ô$Û&�Ü�d“8 ;Ö.Ø—M‘M–Oñ 'ó '�Ø—‘–ñ '÷ *ð "×/Ñ/°ÈÓPˆÔàˆùò "Qß)×)ús   ÇI0Ç:#I5È"-I5É5J	c               ó    € V ^8„  d   QhRRRR/# )rU   rô   r   rX   útuple[tuple[int, ...], ...]rY   )rZ   s   "r[   r\   r  *  s   € ÷ ñ Ø'ðà	$ñr]   c                	ól  aa€ ^RI Ho V3R lo. p\        VP                  4      V.,            Fy  p\	        V\        \
        34      '       dD   TP                  \
        ;QJ d    . V3R lV 4       F  NK  	  5M! V3R lV 4       4      4       Kb  VP                  S! V4      4       K{  	  \        V4      # )rU   r"   c                ó  <€ \        V S4      '       g   R # \        V \        P                  4      '       d   V P                  4       p V P	                  4       ;pf   R # \
        ;QJ d    . R V 4       F  NK  	  5# ! R V 4       4      # )Nc              3  ó$   "  € T F  qx € K  	  R # 5ir_   rY   rW  s   & r[   rY  ÚKSIMDScheduling._get_multikernel_shapes.<locals>.get_size.<locals>.<genexpr>6  s   é € Ð)¡D˜qœ£Dùs   ‚)r[  r   ÚBaseViewÚunwrap_viewÚmaybe_get_sizerØ   )rb  r·  r#   s   & €r[   Úget_sizeÚ8SIMDScheduling._get_multikernel_shapes.<locals>.get_size/  si   ø€ Ü˜c 6×*Ò*ÙÜ˜#œrŸ{™{×+Ò+Ø—o‘oÓ'�Ø×*Ñ*Ó,Ð,�Ò5Ùß”5Ñ)¡DÓ)—5Ð)�5Ñ)¡DÓ)Ó)Ð)r]   c              3  ó4   <"  € T F  pS! V4      x € K  	  R # 5ir_   rY   )rX  Ú_argr4  s   & €r[   rY  Ú9SIMDScheduling._get_multikernel_shapes.<locals>.<genexpr>;  s   øé € Ð @¹C°D¡¨$§ »Cùs   ƒ)r   r#   r§  Úinputsr[  rØ   r  )r|   rô   Úoutrb  r#   r4  s   &&  @@r[   Ú_get_multikernel_shapesÚ&SIMDScheduling._get_multikernel_shapes*  sƒ   ù€ õ 	 õ	*ð ˆÜ˜Ÿ™Ó$¨ v×-Ð-ˆCÜ˜#¤¤e˜}×-Ò-Ø—
‘
Ÿ5œ5Ô @¹CÓ @Ÿ5™5Ô @¹CÓ @Ó@ÖAà—
‘
™8 C›=Ö)ñ	 .ô
 �S‹zÐr]   c               ó    € V ^8„  d   QhRRRR/# )rU   rô   r   rX   r�   rY   )rZ   s   "r[   r\   r  @  s   € ÷ 
ñ 
Ð/Bð 
Àtñ 
r]   c                	óŒ   € V P                  V4      p\        ;QJ d    R  V 4       F  '       g   K   R# 	  R# ! R  V 4       4      # )c              3  óŠ   "  € T F9  p\         ;QJ d    R  V 4       F  '       g   K   RM	  RM! R  V 4       4      x € K;  	  R# 5i)c              3  ó¤   "  € T FF  p\        V\        P                  4      ;'       d     \        V\        P                  4      '       * x € KH  	  R # 5ir_   ©r[  r™   ÚExprr^  rW  s   & r[   rY  ÚFSIMDScheduling._kernel_has_dynamic_shapes.<locals>.<genexpr>.<genexpr>C  s;   é € ð á�Aô ˜1œeŸj™jÓ)×NÐN´*¸QÄÇÁÓ2NÔ.NÔNÛùs
   ‚%A¨(ATFN)rè  )rX  rÙ   s   & r[   rY  Ú<SIMDScheduling._kernel_has_dynamic_shapes.<locals>.<genexpr>B  sG   é € ð 
ñ
  �÷	 ‹Cñ áó�C�CŠCñ áó÷ ð ó  ùs   ‚A’A¦ATF)r;  rè  )r|   rô   Úshapess   && r[   Ú_kernel_has_dynamic_shapesÚ)SIMDScheduling._kernel_has_dynamic_shapes@  sQ   € Ø×-Ñ-¨dÓ3ˆß‹sñ 
ñ
  ó
�sŒsð 	
Šsð 	
ˆsñ 
ñ
  ó
ó 
ð 	
r]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   rô   r   ÚhintrW   rX   r,  rY   )rZ   s   "r[   r\   r  J  s$   € ÷ 
ñ 
Ø'ð
Ø/2ð
à	$ñ
r]   c                óŠ   a€ V P                  V4      p\        ;QJ d    . V3R lV 4       F  NK  	  5# ! V3R lV 4       4      # )z[
Returns cache key for hint-based multi-graph; key is tuple of shapes with hint filled in.
c              3  óˆ   <"  € T F7  p\         ;QJ d    . V3R  lV 4       F  NK  	  5M! V3R  lV 4       4      x € K9  	  R# 5i)c              3  ó¬   <"  € T FI  p\        V\        P                  4      '       d#   \        V\        P                  4      '       g   SMTx € KK  	  R # 5ir_   rA  )rX  r0  rI  s   & €r[   rY  ÚASIMDScheduling._make_shape_cache_key.<locals>.<genexpr>.<genexpr>R  sG   øé € ð ñ �Aô ˜a¤§¡×,Ò,´ZÀÄ5Ç=Á=×5QÒ5Qñ àôó ùs   ƒAANr�  )rX  rÙ   rI  s   & €r[   rY  Ú7SIMDScheduling._make_shape_cache_key.<locals>.<genexpr>Q  sJ   øé € ð 
ñ  �÷ ŒEô ñ ó	�E‰Eô ñ ó	÷ ð ó  ùs
   ƒA“/A)r;  rØ   )r|   rô   rI  rE  s   &&f r[   Ú_make_shape_cache_keyÚ$SIMDScheduling._make_shape_cache_keyJ  sK   ø€ ð ×-Ñ-¨dÓ3ˆßŒuô 
ñ  ó
�uð 	
ˆuô 
ñ  ó
ó 
ð 	
r]   Úhint_overridec               ó    € V ^8„  d   QhRRRR/# )rU   rQ  r§   rX   ú
str | NonerY   )rZ   s   "r[   r\   r  [  s$   € ÷ rñ rð "ðrð 
ñrr]   c          
     ón  € VP                   w  pw  rxV^8w  d   \        RV 24      h\        VP                  \        4      '       EdS   VP                  P
                  '       Ed6   \        VP                  P
                  4      ^8”  Ed   V P                  VP                  4      '       Edï   / p	. p
VP                  P
                  P                  4        F¾  w  ppV! VP                  VR7      w  rÞV'       dX   V P                  VVVVVRR7      p\        V\        4      '       g   \        R\        V4       24      hV
P                  V4       K{  Vf   K�  V P                  VVVVVRR7      pVf   RMV P                  VP                  V4      pWÙV&   KÀ  	  V'       d   RP                  V
4      # \        P                   ! \#        V	P%                  4       4      4       \'        V	4      p. VOVNVOpV P)                  VVP*                  4       VP-                  VP*                  4       \.        P0                  ;P2                  VP2                  ,          un        \.        P0                  ;P4                  VP4                  ,          un        V P7                  4        R# VP                  P9                  VP                  VR7      w  rÞV'       d   V P                  VVVVVRR7      # V P                  VVVVVRR7      p. VOVNVOpV P)                  VVP*                  4       VP-                  VP*                  VP                  4       \.        P0                  ;P2                  VP2                  ,          un        \.        P0                  ;P4                  VP4                  ,          un        V P7                  4        R# )	z¦
Codegen a triton template with multi-kernel dispatch support

If `only_gen_src_code=True` the src code will be returned instead of being
codegenned into the wrapper
zexpected rnumel == 1, got )rQ  T)rë  z#expected src_code to be a str, got NFz

)r±  r­   r[  rô   r   Ú_make_kernel_rendersr_  rF  r�   r)  rk   r\  r  rO  rò  rD   rú  r§  rò   rE   rC  r´  r:  r>   rý   r>  rD  rI  Úmake_kernel_render)r|   rì  rí  rî  rë  rQ  rx  Ú_numelr¬  rÿ  Ú	src_codesÚ	size_hintrV  rs   rñ  r  Úshape_cache_keyÚmulti_kernelr¨  s   &&&&$$             r[   Úcodegen_templateÚSIMDScheduling.codegen_template[  sj  € ð  ,×1Ñ1ÑˆÑˆFØ�QŒ;Ü Ð#=¸f¸XÐ!FÓGÐGô �}×)Ñ)Ô+>×?Ó?Ø×"Ñ"×7×7Ñ7Ü�M×&Ñ&×;Ñ;Ó<¸qÕ@Ø×/Ñ/°×0BÑ0B×CÓCàˆGØˆIð
 ×#Ñ#×8Ñ8×>Ñ>Ö@ñØØ"á!3Ø!×&Ñ&°mô"‘�÷ %Ø#×<Ñ<ØØØ%Ø&Ø&Ø*.ð  =ó  �Hô & h´×4Ò4Ü,ØAÄ$ÀxÃ.ÐAQÐRóð ð ×$Ñ$ XÖ.à Ò(Ù Ø!×:Ñ:ØØØ%Ø&Ø&Ø*/ð ;ó �Fð %Ò,ñ à!×7Ñ7¸×8JÑ8JÈIÓVð $ð 06˜OÓ,ñK A÷N !Ø—{‘{ 9Ó-Ð-ä×0Ò0´°g·n±nÓ6FÓ1GÔHÜ.¨wÓ7ˆLØM˜nÐM¨mÐM¸nÐMˆMØ× Ñ  °×0HÑ0HÔIØ×$Ñ$ \×%=Ñ%=Ô>Ü�G‰G×#Ò# |×'CÑ'CÕCÕ#Ü�G‰G×&Ò&¨,×*IÑ*IÕIÕ&Ø×*Ñ*Ô,Ùà*×/Ñ/×BÑBØ×"Ñ"°-ð Có ‰NˆF÷ !Ø×4Ñ4ØØØ!Ø"Ø"Ø&*ð 5ó ð ð ×6Ñ6ØØØ!Ø"Ø"Ø&+ð 7ó �ð !R .Ð Q°-Ð QÀ.Ð Q�Ø×$Ñ$ ]°F×4FÑ4FÔGØ×"Ñ" 6×#5Ñ#5°}×7IÑ7IÔJä—‘×'Ò'¨6×+AÑ+AÕAÕ'Ü—‘×*Ò*¨f×.GÑ.GÕGÕ*Ø×.Ñ.Ô0Ùr]   c                	óˆ   € \         P                  P                  P                  \         P                  P                  4       R # r_   )r>   rý   r0  Úgenerate_debug_syncrƒ   s   &r[   Úcodegen_syncÚSIMDScheduling.codegen_syncÏ  s$   € Ü	�‰×Ñ×0Ñ0´·±×1EÑ1EÖFr]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   Ú	node_infor¦  rë  r�   rX   ztuple[str, TritonKernel]rY   )rZ   s   "r[   r\   r  Ò  s(   € ÷  ñ  àð ð  ð ð 
"ñ	 r]   c                	ó”  € / pV P                   P                  VP                  V4       V P                   ! VP                  3R VP                  RVP                  /VB pV P                  WAP                  V4       \        P                  ! V4      ;_uu_ 4        VP                  4       pRRR4       WT3#   + '       g   i     XT3# ; i)r­  r«  N)
r™  Ú apply_feature_required_overridesr­  rª  r«  rí  r¨  r>   r  rž  )r|   rc  rë  rˆ  rs   r  s   &&&   r[   Ú_codegen_standalone_kernelÚ)SIMDScheduling._codegen_standalone_kernelÒ  sÅ   € ð
 )+ˆØ×Ñ×9Ñ9Ø×Ñ ô	
ð ×!Ò!Ø×Ññ
à×'Ñ'ð
ð $×1Ñ1ð
ð ñ	
ˆð 	×Ñ˜F×$;Ñ$;Ð=NÔOÜ×!Ò! &×)Õ)Ø×,Ñ,Ó.ˆH÷ *àÐÐ÷ *Ö)à˜ÐÐús   ÂB4Â4C	c               ó    € V ^8„  d   QhRRRR/# )rU   rc  r¦  rX   ztuple[list[Any], Any]rY   )rZ   s   "r[   r\   r  æ  s   € ÷ >ñ >°Hð >ÐAVñ >r]   c                	ól  € ^RI HpHpHp ^RIHp \        RRR7      pV P                  P                  VP                  V4       V P                  ! VP                  3RVP                  RVP                  /VB p\        P                  ! RR7      ;_uu_ 4        \        P                  ! V4      ;_uu_ 4        V P!                  VP"                  V4       VP%                  4       pR	R	R	4       R	R	R	4       VP&                  P)                  4        U	U
u/ uFf  w  rš\+        V	4      '       d   VP,                  '       g   K*  V	\/        \1        \        P2                  P4                  P7                  V
4      4      4      bKh  	  pp	p
\9        R
/ VP;                  4       CVP=                  4       C4      pVP                  '       d>   V! VVP                  P?                  VP                  4      VP@                  VRR7      pW×3# VP,                  '       d>   V! VVP                  P?                  VP                  4      VP@                  VRR7      pW×3# VPB                  PE                  4       w   rŽpV! VVP@                  V! W¾4      VRR7      pW×3#   + '       g   i     EL­; i  + '       g   i     EL¹; iu up
p	i )rU   )rÔ  Ú	pointwiserŠ  )Úselect_tile_hintTF)Úis_combo_kernelr¸  r­  r«  rû  Nr)   )Úreduction_hintÚtriton_metaÚinductor_metaÚreturn_configs)rn  Ú	tile_hintro  rp  )#Úruntime.triton_heuristicsrÔ  rj  rŠ  Útriton_utilsrk  r©  r™  re  r­  rª  r«  r   r  r>   r  rÿ  r¨  rž  rÍ  r�   r7   rÐ  r,   rW   rý   rþ   r  r   Úinductor_meta_commonÚinductor_meta_per_kernelÚget_reduction_hintrn  r]  r\  )r|   rc  rÔ  rj  rŠ  rk  rˆ  rs   rx  rr   rp   Ú
size_hintsro  ÚconfigsÚ	signatures   &&             r[   Ú_probe_subkernel_heuristicÚ)SIMDScheduling._probe_subkernel_heuristicæ  sq  € ÷	
ñ 	
õ
 	3ä(,Ø Ø+0ô)
ˆð 	×Ñ×9Ñ9Ø×Ñ ô	
ð ×!Ò!Ø×Ññ
à×'Ñ'ð
ð $×1Ñ1ð
ð ñ	
ˆô �\Š\¨5×1Ö1´1×3GÒ3GÈ×3OÕ3OØ×2Ñ2°9×3JÑ3JÈFÔSØ×%Ñ%Ó'ˆA÷ 4P×1ð "(§¡×!4Ñ!4Ô!6ô
á!6‘�Ü& v×.Ò.°&×2IÕ2Iô TˆF”O¤C¬¯©×(8Ñ(8×(JÑ(JÈ5Ó(QÓ$RÓSÒSÙ!6ð 	ñ 
ô
 ØðØ×-Ñ-Ó/ðà×1Ñ1Ó3ðó
ˆð ×&×&Ð&Ù*ØØ%Ÿ™×AÑAÀ&×BVÑBVÓWØ"×.Ñ.Ø+Ø#ôˆGð0 ˆÐð# ×$×$Ð$ÙØØ%Ÿ™×AÑAÀ&×BVÑBVÓWØ"×.Ñ.Ø+Ø#ôˆGð  ˆÐð "(§¡×!;Ñ!;Ó!=ÑˆAˆq˜QÙØØ"×.Ñ.Ù*¨:ÓAØ+Ø#ôˆGð ˆÐ÷S 4P×3OÐ3Oú×1×1Ð1üó
s1   ÂJÂ7-J	Ã$JÄ'J0Ä=A J0ÊJÊ	JÊJ-	c               ó    € V ^8„  d   QhRRRR/# )rU   Úchosen_configsrõ  rX   zdict[str, int]rY   )rZ   s   "r[   r\   r  '  s   € ÷ 
ñ 
°9ð 
Àñ 
r]   c                óê   € \        V 4       UUUu/ uFS  w  rVP                   F>  pVP                  R4      '       g   K  V RV 2\        VP                  V,          4      bK@  	  KU  	  uppp# u upppi )z�Suffix each subkernel's BLOCK kwargs into one combo config dict
(XBLOCK -> XBLOCK_0, XBLOCK_1, ...; R0_BLOCK -> R0_BLOCK_0, ...).r¸   rx  )r*  ÚkwargsÚendswithrW   )r}  rú  Úcfgr'  s   &   r[   Ú_stitch_combo_block_configÚ)SIMDScheduling._stitch_combo_block_config&  sk   € ô $ NÔ3õ
á3‘�Ø—z”z�Ø�|‰|˜G×$ô /ˆsˆe�1�Q�CˆLœ#˜cŸj™j¨�oÓ.Ò.á!ñ Ù3ó
ð 	
ùô 
s
   �*A.¿*A.c               ó$   € V ^8„  d   QhRRRRRR/# )rU   r}  rõ  Úchosen_nodesrX   r   rY   )rZ   s   "r[   r\   r  2  s$   € ÷ &
ñ &
Ø!ð&
Ø1:ð&
à	ñ&
r]   c           	     	óÈ  aa€ ^ RI pR R lo\        \        \        V 4      4      VV3R lR7      pW,          p\        P                  V 4      pTP                  VP                  P                  4        UUu/ uF   w  rgVP                  R4      '       d   K  WgbK"  	  upp4       VP                  V\        VP                  4      \        VP                  4      R7      # u uppi )r   Nc               ó    € V ^8„  d   QhRRRR/# )rU   Úsnoder   rX   rW   rY   )rZ   s   "r[   r\   ÚBSIMDScheduling._stitch_no_bench_combo_config.<locals>.__annotate__7  s   € ÷ 	ñ 	 Cð 	¨Cñ 	r]   c                ó    € \        R  \        P                  ! V P                  P                  V P                  P
                  4       4       4      # )c              3  ód   "  € T F&  p\         P                  P                  VR R7      x € K(  	  R# 5i)T)Úcount_bytesN)r>   rý   Úget_dep_size_hint)rX  Úds   & r[   rY  ÚYSIMDScheduling._stitch_no_bench_combo_config.<locals>.node_total_bytes.<locals>.<genexpr>8  s2   é € ð ñ�Aô —‘×)Ñ)¨!¸Ð)×>Ð>óùs   ‚.0)r
  rÎ  rZ  rÓ  rÔ  rÞ  )rˆ  s   &r[   Únode_total_bytesÚFSIMDScheduling._stitch_no_bench_combo_config.<locals>.node_total_bytes7  sB   € Üñ ä"ŸšØ×%Ñ%×+Ñ+¨U×->Ñ->×-EÑ-Eôóó ð r]   c                ó"   <€ S! SV ,          4      # r_   rY   )rú  r…  r�  s   &€€r[   r$  Ú>SIMDScheduling._stitch_no_bench_combo_config.<locals>.<lambda>E  s   ø€ Ñ*¨<¸­?Ô;r]   r&  r¸   )Ú	num_warpsÚ
num_stages)rb   r  Úranger_  r˜  r‚  rÚ  r  r�   r€  ÚConfigrW   r”  r•  )	r}  r…  rb   Ú
winner_idxÚ
winner_cfgÚstitched_kwargsr'  r”  r�  s	   &f      @r[   Ú_stitch_no_bench_combo_configÚ,SIMDScheduling._stitch_no_bench_combo_config1  sÐ   ù€ ó 	õ	ô Ü”#�nÓ%Ó&Ý;ô
ˆ
ð $Õ/ˆ
ä*8×*SÑ*SØó+
ˆð 	×Ñð #-×"3Ñ"3×"9Ñ"9Ô";ôá";‘J�CØ—|‘| G×,ô �’
Ù";òô	
ð �}‰}ØÜ˜*×.Ñ.Ó/Ü˜:×0Ñ0Ó1ð ó 
ð 	
ùós   Á6C
ÂC
c               ó$   € V ^8„  d   QhRRRRRR/# )rU   r±  rõ  Únode_schedule_mapúdict[Any, NodeInfo]rX   zlist[Any | None]rY   )rZ   s   "r[   r\   r  Z  s)   € ÷ añ aØðaØ3Fðaà	ñar]   c           	     óî  a aaa a!a"€ ^ RI Hp ^RIHp ^RIHp V! 4       pVP                  4       p. o"/ p\        4       p	\        P                  p
V EF…  pW+,          p\        P                  ! RRRR/4      ;_uu_ 4        S P                  VP                  4       RVP                  P                  R7      pR	R	R	4       \         P"                  ! X4      pVP$                  pVP&                  p\)        VP*                  4      p\        P,                  '       * ;'       d    VR	J;'       d    \/        V4      ^8H  p\1        V4      pVV	9   pV	P3                  V4       VP5                  V4      pV'       gF   V'       g>   V'       g6   V'       d.   VP7                  R
V4      p\9        VV4      '       d   TMR	pVVV&   S"P;                  Y¾T;'       g    TVV34       EKˆ  	  V
\        n	        R	.\/        S"4      ,          o!\        4       oR V3R llo R V V V!V"3R llo/ p. p\=        S"4       FN  w  pEv pp\?        VRR	4      pVf   VP;                  V4       K-  VPA                  V. 4      P;                  V4       KP  	  R VV V"3R llp\        4       pV FR  pS"V,          R,          pVe   \1        V4      V9   d   R	pMVe   VP3                  \1        V4      4       V! VV.4       KT  	  V! V4       F  pV! VVV,          4       K  	  \B        ;QJ d(    V3R l\=        S!4       4       F  '       g   K   RM	  RM! V3R l\=        S!4       4       4      '       d   \E        R4      hS!#   + '       g   i     EL·; i)aÍ  Autotune each combo subkernel standalone at compile time and read back its winning
config (None for a subkernel whose compile or benchmark failed, which the caller
carves out to run standalone instead of failing the compile).

Triton precompiles are submitted to the async compile pool as each subkernel is codegened.
The subkernels are then benchmarked in compile-completion order through the standard
benchmark_codegened_module path, so device benchmarking overlaps the remaining precompiles
while retaining preserve_rng_state, argument cloning and the n_spills guard. Subkernels
with a single candidate config (unless coordinate descent tuning is on) or an autotune
cache hit are taken directly with no benchmark.
)Úas_completed)ÚAsyncCompile)ÚCodeCacheFutureztriton.cooperative_reductionsFz#triton.force_cooperative_reductionsT)rü  r†  Nrý  c               ó$   € V ^8„  d   QhRRRRRR/# )rU   r¢   rW   Úpnr   rX   rw   rY   )rZ   s   "r[   r\   ÚFSIMDScheduling._autotune_subkernels_compile_time.<locals>.__annotate__¤  s!   € ÷ 	ñ 	˜Cð 	 Sð 	¨Tñ 	r]   c                ó–   <€ \         R ,          R;;,          ^,          uu&   \        P                  RV4       SP                  V 4       R# )rø  Ú!combo_subkernel_autotune_fallbackz6combo compile-time autotune failed for %s; carving outN)r   rv  rw  r+  )r¢   r¥  Úfaileds   &&€r[   ÚfallbackÚBSIMDScheduling._autotune_subkernels_compile_time.<locals>.fallback¤  s5   ø€ Ü�ZÕ Ð!D×EÈÕJÓEÜ�K‰KÐPÐRTÔUØ�J‰J�uÖr]   c               ó    € V ^8„  d   QhRRRR/# )rU   r¢   rW   rX   rw   rY   )rZ   s   "r[   r\   r¦  ©  s   € ÷ *	$ñ *	$˜Sð *	$ Tñ *	$r]   c                óH  <€ SV ,          w  rr4pVP                   pVP                  pR pV'       gu   V'       gm    SP                  VRR7      w  rš\        P                  ! V	4      ;'       d!    \        VP                  4      ^8g  ;'       g    \        VP                  4      ^8„  pV'       d   S! W4       R# VP                  pVP                  ;'       g    / pT;'       g$    V'       * ;'       g    VP                  R4      R8H  p\        R	,          V'       d   R
MR;;,          ^,          uu&   V'       d   V^ ,          P                  SV &   R# V'       d   V^ ,          SV &   R# S! W4       R#   \         d'    \        P                  RRR7       \        R4      p	 ELQi ; i)FT)r  z combo subkernel benchmark raised)Úexc_infor—  NÚautotune_cache_stateÚhitrø  Úcombo_subkernel_autotune_cachedÚcombo_subkernel_autotune)rý  rx  r  Ú	Exceptionrv  rÐ  r˜  r™  Úisinfr_  Ú	launchersÚautotune_cache_infor.  r   r   )r¢   r¥  r  Úalready_tunedÚsinglerx  Ú	autotunerrx  Úbenchmark_failedr*  Ú	_mod_pathrµ  ÚinfoÚcachedrª  r|   ÚwinnersÚworks   &             €€€€r[   Ú	benchmarkÚCSIMDScheduling._autotune_subkernels_compile_time.<locals>.benchmark©  s„  ø€ Ø04°UµÑ-ˆB�]¨AØŸ™ˆIØ×'Ñ'ˆGØ$Ðß ¯ð&Ø$(×$CÑ$CØ¨Tð %Dó %‘M�Bô —J’J˜r“N×DÐD¤s¨9×+>Ñ+>Ó'?À1Ñ'D÷$2ð $2ä˜×,Ñ,Ó-°Ñ1ð !÷  Ù˜Ô#ÙØ!×+Ñ+ˆIØ×0Ñ0×6Ð6°BˆDð ÷ =ð =Ø ”=÷=ð =à—8‘8Ð2Ó3°uÑ<ð ô
 �ZÕ çñ 2à/÷ð õ	ó ÷
 Ø!*¨1¥×!4Ñ!4�˜“ßØ!(¨¥�˜“ñ ˜Ö#øô? !ô &Ü—I‘IÐ@È4�IÔPÜ˜u›“Bð&ús   ºE0 Å0-F!Æ F!Úfuturec               ó$   € V ^8„  d   QhRRRRRR/# )rU   rÂ  r   rC  rŸ  rX   rw   rY   )rZ   s   "r[   r\   r¦  Þ  s!   € ÷ 	!ñ 	!¨#ð 	!¸	ð 	!Àdñ 	!r]   c                ó¸   <€ V e    V P                  4        V F  pS! V4       K  	  R #   \         d%    T F  pS! TST,          ^ ,          4       K  	   R # i ; ir_   )Úresultr³  )rÂ  rC  r¢   rÀ  rª  r¿  s   && €€€r[   Úresolve_and_benchmarkÚOSIMDScheduling._autotune_subkernels_compile_time.<locals>.resolve_and_benchmarkÞ  s]   ø€ ØÒ!ð	ð
 —M‘M”Oó
 !�Ù˜%Ö ó !øô	 !ô Û!(˜Ù  ¨¨U­°A­Ö7ñ ")âðús   ‡* ª+AÁAc              3  óB   <"  € T F  w  rVS9  g   K  VR J x € K  	  R # 5ir_   rY   )rX  rú  Úwr©  s   &  €r[   rY  ÚCSIMDScheduling._autotune_subkernels_compile_time.<locals>.<genexpr>ù  s!   øé € ÐKÑ%7™T˜Q¸1ÀF¹?Œyˆq�D�yÓ%7ùs   ƒ“z-expected all combo subkernels to be autotunedr  )#Úconcurrent.futuresr¡  Úasync_compiler¢  Ú	codecacher£  Úuse_process_poolr   r   Úgenerated_kernel_countr   r  Úgenerate_kernel_code_from_nodesr²  r­  r†  r   r|  rý  rx  r�   rµ  r8  r_  r!  r+  r.  rb   r[  r  r*  ÚgetattrÚ
setdefaultrè  r­   )#r|   r±  rž  r¡  r¢  r£  rÌ  rÎ  Úfutures_by_autotunerÚseen_autotunersÚold_kernel_countr¥  rc  r  r  r¹  rx  r·  r¸  Úautotuner_idÚ	duplicaterÂ  Úmaybe_futureÚpendingÚreadyr¢   rx  Ú
raw_futurerÆ  Úresolved_futuresrÀ  r©  rª  r¾  r¿  s#   f&&                           @@@@@r[   Ú!_autotune_subkernels_compile_timeÚ0SIMDScheduling._autotune_subkernels_compile_timeZ  s  ý€ õ 	4å0Ý/á$›ˆØ(×9Ñ9Ó;ÐØ79ˆØ/1ÐÜ+5«<ˆô #×9Ñ9ÐÜˆBØ)Õ-ˆIô —’à3°UØ9¸5ð÷õ ð  ×?Ñ?Ø—L‘L“NØ%)Ø&/×&8Ñ&8×&JÑ&Jð @ó �÷ô ×"Ò" 8Ó,ˆCØŸ™ˆIØ×'Ñ'ˆGÜ  ×!4Ñ!4Ó5ˆMô
 ×4Ñ4Ô4÷ &ð &Ø 4Ð'÷&ð &ä˜“L AÑ%ð ô
 ˜i›=ˆLØ$¨Ñ7ˆIØ×Ñ Ô-Ø)×-Ñ-¨lÓ;ˆFß ¯¿6×FVð  -×3Ñ3°I¸xÓH�ä$.¨|¸_×$MÒ$M‘LÐSWð ð 6<Ð$ \Ñ2Ø�K‰K˜ -×"<Ð"<°9¸fÀfÐM×NñQ ðR *:ŒÔ&à%) F¬S°«YÕ$6ˆÜ",£,ˆ÷	ð 	÷
*	$ó *	$ðX )+ˆØˆÜ#,¨T¦?ÑˆE‘<�Q˜Ü  ¨°4Ó8ˆJØÒ!Ø—‘˜UÖ#à×"Ñ" :¨rÓ2×9Ñ9¸%Ö@ñ $3÷	!ò 	!ô -7«LÐÛˆEØ˜%•[ •_ˆFØÒ!¤b¨£jÐ4DÔ&DØ‘ØÒ#Ø ×$Ñ$¤R¨£ZÔ0Ù! &¨5¨'Ö2ñ ñ ' wÖ/ˆJÙ! *¨g°jÕ.AÖBñ 0÷ ‹3ÔK¤Y¨wÔ%7ÓK�3�3Š3ÔK¤Y¨wÔ%7ÓK×KÒKÜ Ð!PÓQÐQØˆ÷A÷ ð ús   Á>7M#Í#M4c               ó4   € V ^8„  d   QhRRRRRRRRRRR	RR
R/# )rU   Ú
fusion_pnsrõ  rž  rŸ  Úenable_autotuner�   Úmixed_sizesÚper_subkernel_blocksrë  rX   r   rY   )rZ   s   "r[   r\   r  ý  sP   € ÷ !ñ !àð!ð /ð!ð ð	!ð
 ð!ð #ð!ð  ð!ð 
ñ!r]   c           
     ó@  € ^RI Hp V! V P                  VVVR7      pV F{  p	W),          p
VP                  V
P                  V
P
                  V'       * V P                  V
P                  VR7      pV P                  VP                  V4      V
P                  V4       K}  	  V# )zGBuild a ComboKernel and its sub-kernels from a list of partition nodes.©ÚComboKernel)Útriton_kernel_clsrá  râ  rã  )r­  Úoptimize_maskrç  r«  rã  )
Útriton_combo_kernelræ  r™  Úcreate_triton_kernelrª  r­  r«  rí  Úcreate_sub_kernelr¨  )r|   rà  rž  rá  râ  rã  rë  ræ  rs   r¥  rc  Ú	subkernels   &&&&&&&     r[   Ú_build_combo_kernelÚ"SIMDScheduling._build_combo_kernelý  s©   € õ 	5áØ"×.Ñ.Ø+Ø#Ø!5ô	
ˆó ˆBØ)Õ-ˆIØ#×8Ñ8Ø× Ñ Ø"×+Ñ+Ø"-œoØ"&×"2Ñ"2Ø'×5Ñ5Ø%9ð 9ó ˆIð ×ÑØ×(Ñ(¨Ó3Ø×'Ñ'Ø!öñ ð ˆr]   c               ó    € V ^8„  d   QhRRRR/# )rU   r¾  rõ  rX   rY   )rZ   s   "r[   r\   r  !  s   € ÷ 
/ñ 
/¨)ð 
/¸	ñ 
/r]   c           	     ó  € ^RI Hp / pV  FÐ  pVP                  P                  4        UUu/ uF*  w  rEVP	                  R4      '       d   K  V\        V4      bK,  	  ppp\        \        VP                  4       4      4      \        VP                  4      \        VP                  4      3pV! V\        VP                  4      \        VP                  4      4      W'&   KÒ  	  \        V4       Uu. uF  qBV,          NK  	  up# u uppi u upi )z›Deduplicate the subkernels' winner launch configs (non-block kwargs + num_warps +
num_stages) into the candidate set for the combo's kernel-level autotune.)ÚComboLaunchConfigr¸   )
ré  rñ  r  r�   r€  rW   rØ   r"  r”  r•  )r¾  rñ  ÚseenrÉ  rG  rH  Únbr'  s   &       r[   Ú_combo_launch_candidatesÚ'SIMDScheduling._combo_launch_candidates   sÈ   € õ 	;à!ˆÛˆAØ()¯©¯©Ô(8ÔTÑ(8¡ ÀÇ
Á
È7×@S”)�!”S˜“V’)Ñ(8ˆBÑTÜœ §¡£
Ó+Ó,¬c°!·+±+Ó.>ÄÀAÇLÁLÓ@QÐRˆCÙ)¨"¬c°!·+±+Ó.>ÄÀAÇLÁLÓ@QÓRˆD‹Iñ ô "(¨¤Ó.¡˜A�Q—�¡Ñ.Ð.ùó Uùò /s   «D ÁD Ã,Dc               ó4   € V ^8„  d   QhRRRRRRRRRRRRR	R
/# )rU   Úsubkernel_nodesr  Úcustom_part_algorithmr�   rá  râ  rë  rã  rX   z!list[tuple[str | None, Any, Any]]rY   )rZ   s   "r[   r\   r  -  sY   € ÷ R ñ R à0ðR ð  $ðR ð ð	R ð
 ðR ð  ðR ð #ðR ð 
+ñR r]   c                ó  € ^RI Hp ^RIHp \	        V P
                  V4      '       g   \        RV P
                   24      hV U	u. uF  q™P                  4       NK  	  p
p	/ p\        W4       EF{  w  rÍ\        VR R7      P                  w  pw  ppV P                  WßV4      pRpV'       dÀ   \        P                  P                  P                   P                  '       dY   \!        V\"        P$                  \"        P&                  34      '       g   \        R\)        V4       24      hVP+                  4       pMRp\-        VVVVR7      pV P/                  VVVVP0                  4      w  ppM \-        VVV4      pV P3                  VVV4      pVP5                  4       ;'       d1    \6        P8                  P;                  VP=                  V4      R	R
7      p\?        VVVVVVVR7      W¼&   EK~  	  VPA                  VV VVR7      p\B        PE                  R\G        V4      V Uu. uF  p\G        V4      NK  	  up4       . pV EF  p\G        V4      ^ 8X  d   K  \G        V4      ^8X  dZ   VV^ ,          ,          pV'       d   VPI                  RRV34       KT  V PK                  VV4      w  ppVPI                  VVV34       K  V'       Ed†   V'       Ed}   \        PL                  '       Edf   V'       Eg]   \O        V4      pV PQ                  VV4      p\        VV4       UU u. uF  w  pp V f   K  VV 3NK  	  p!pp \        VV4       UU u. uF  w  pp V e   K  VNK  	  p"pp \G        V!4      ^8¼  d›   V! UUu. uF  w  rÎVNK	  	  p#ppV! UU u. uF  w  pp V NK
  	  p$pp V PS                  V#VRVRVR7      pRVn*        V PW                  V$4      Vn,        V P[                  V$4      Vn.        VP_                  4       pVPI                  VVV#34       M\O        V4      p"V" F3  pV PK                  W¼,          V4      w  p%p&VPI                  V%V&V.34       K5  	  EK  T;'       d    V'       * p'. p(. p). p*V''       Ed$   \`        Pb                  p+ V FÙ  pV Pe                  W¼,          4      w  p,p-\        Pf                  Ph                  '       g%   \        Pj                  Pm                  4       '       d   T,;'       d    V-Pn                  '       * p.MT,;'       d    \G        V,4      ^8*  p.V.'       d,   V(PI                  V4       V)PI                  V,^ ,          4       KÈ  V*PI                  V4       KÛ  	  V'       d   V+\`        n1        \G        T(4      ^8  d   . p(. p)\O        T4      p*M\O        V4      p(\G        V(4      ^8¼  dc   V PS                  V(VVVVVR7      pV''       d    V)'       d   V Pq                  V)V(4      Vn9        VP_                  4       pVPI                  VVV(34       V* F3  pV PK                  W¼,          V4      w  p/p&VPI                  V/V&V.34       K5  	  EK	  	  V# u up	i u upi u up pi u up pi u uppi u up pi   T'       d   T+\`        n1        i i ; i)a<  
Generate kernel code for combo kernel partitions.

Partitions subkernel_nodes using horizontal_partition(), then generates
kernel code for each partition. Single-node partitions are generated as
regular kernels, while multi-node partitions use ComboKernel.

Returns a list of (src_code, kernel, node_group) tuples.
rN   rå  z8expected self.kernel_type to subclass TritonKernel, got c                ó4   € \        V P                  4       4      # r_   rÌ  rÍ  s   &r[   r$  Ú;SIMDScheduling.generate_combo_kernel_code.<locals>.<lambda>L  ó   € ¼#¸a¿n¹nÓ>NÔ:Or]   r&  Nz;expected pn to be FusedSchedulerNode or SchedulerNode, got ©r†  F)rÒ  )r¨  rª  r«  rp   r¬  r­  r®  )r¬   Útriton_schedulingÚcustom_algorithmÚnode_info_mapz1ComboKernels: %d nodes partitioned into %s groupsT)rá  râ  rã  rë  ):rb   rO   ré  ræ  Ú
issubclassr™  r­   r²  rr  r  r±  rî  r`   ra   r   rm  r[  r   rÕ  rÚ  r\  rn  rJ   ro  r†  r¶  r„   r>   ÚchoicesrÓ  Úwith_tiling_scoresr¦  Úhorizontal_partitionrv  rÐ  r_  r  rf  Ú"combo_kernel_compile_time_autotuner§  rÝ  rí  Úcombo_compile_time_autotunerô  Úcombo_launch_candidatesr‚  Ústitched_block_configrž  r   rÏ  rz  ÚversionÚhipÚxpuÚis_availableÚautotune_hintsr›  Úno_bench_stitched_config)0r|   r÷  rø  rá  râ  rë  rã  rO   ræ  rô   Úfused_node_listsrž  r¥  r¬   rx  rp   r¬  r¨  r«  r†  r­  rª  r®  Ú
partitionsÚpÚkernel_code_listÚ
node_grouprc  r  rs   r±  ÚtunedrÉ  ÚfusableÚ	carve_outÚfusable_pnsr¾  Úco_srcÚ	co_kernelÚno_bench_moderà  Úfusion_configsÚcarve_out_pnsrÕ  rx  Úprobe_kernelÚfuse_okÚco_src_codes0   &&&&&&&                                         r[   Úgenerate_combo_kernel_codeÚ)SIMDScheduling.generate_combo_kernel_code-  sq  € õ$ 	)Ý4ô ˜$×*Ñ*¨L×9Ò9Ü ðØ×'Ñ'Ð(ð*óð ñ
 :IÓI¹°ŸN™NÖ,¹ÐÐIØ13ÐÜ˜_×?‰IˆBÜ!$ UÑ0OÔ!P×!VÑ!VÑˆA‰��vØ ×7Ñ7¸ÀfÓMˆMØ ˆMß#Ü—?‘?×)Ñ)×0Ñ0×I×IÐIÜ%ØœY×9Ñ9¼9×;RÑ;RÐS÷ò ô -ð2Ü26°r³(°ð=óð ð )+×(@Ñ(@Ó(BÑ%à(,Ð%Ü-Ø! 5¨&ÐDUô�ð )-×(BÑ(BØ!ØØØ×.Ñ.ó	)Ñ%�™ô .¨m¸UÀFÓK�Ø×+Ñ+¨M¸5À&ÓI�à×%Ñ%Ó'÷ ð Ü—I‘I×=Ñ=Ø×/Ñ/°Ó>Ø*/ð >ó ð $ô %-Ø+ØØ+ØØØ!Ø(?ô%ÐÔ!ñG @ðZ !×5Ñ5Ø!Ø"Ø2Ø+ð	 6ó 
ˆ
ô 	�	‰	Ø?Ü�Ó Ù'Ó(™Z˜ŒS�ŽV™ZÑ(ô	
ð
 ÐÜ$ˆJÜ�:‹ !Ô#Ùä�:‹ !Ô#à-¨j¸­mÕ<�	ß$à$×+Ñ+¨T°4¸Ð,DÖEà'+×'FÑ'FØ!Ð#4ó(Ñ$�H˜fð %×+Ñ+¨X°v¸zÐ,JÖK÷ $�Oß,Ð,Ü×A×AÑAß-Ð-ä  Ó,�EØ ×BÑBØÐ0ó�Eô
 58¸¸uÔ4EÔWÑ4E©5¨2¨qÈœw  A›wÑ4E�GÑWÜ14°U¸EÔ1BÔ PÑ1B©¨¨AÀa§ Ñ1B�IÑ PÜ˜7“| qÔ(Ù7>Ô&?±w©e¨b£r±w˜Ñ&?Ù18Ô"9±©¨¨A£1±˜Ñ"9Ø!%×!9Ñ!9Ø'Ø-Ø,0Ø(3Ø15Ø.?ð ":ó "˜ð >B˜Ô:Ø9=×9VÑ9VØ#ó:˜Ô6ð 8<×7VÑ7VØ#ó8˜Ô4ð $*×#8Ñ#8Ó#:˜à(×/Ñ/°¸6À;Ð0OÕPô %)¨£K˜	Û'˜Ø,0×,KÑ,KØ-Õ1Ð3Dó-Ñ)˜ 	ð )×/Ñ/°¸ÀRÀDÐ0IÖJñ (ò Ø 4× LÐ L¸_Ô9L�Ø(*�
Ø,.�Ø+-�ß �=Ü'.×'EÑ'EÐ$ðNÛ",˜BØ48×4SÑ4SØ 1Õ 5ó5Ñ1˜G \ô  %Ÿ}™}×0×0Ð0´E·I±I×4JÑ4J×4LÒ4LØ*1×*UÐ*U¸,×:UÑ:UÔ6U¡à*1×*GÐ*G´c¸'³lÀaÑ6G ß&Ø *× 1Ñ 1°"Ô 5Ø .× 5Ñ 5°g¸aµjÖ Aà -× 4Ñ 4°RÖ 8ñ #-÷ -Ø=MœGÔ:Ü˜:“¨Ô*Ø%'˜
Ø)+˜Ü(,¨ZÓ(8˜øä!% jÓ!1�Jä�z“? aÔ'Ø!×5Ñ5Ø"Ø)Ø(7Ø$/Ø-AØ*;ð 6ó �F÷ %¯à ×>Ñ>Ø .°
óð Ô7ð  &×4Ñ4Ó6�Hà$×+Ñ+¨X°v¸zÐ,JÔKã'�BØ-1×-LÑ-LØ)Õ-Ð/@ó.Ñ*�K ð %×+Ñ+¨[¸)ÀbÀTÐ,JÖKô (ñi %ðv  Ðùòm Jùòp )ùó@ XùÛ Pùã&?ùÛ"9ø÷j -Ø=MœGÕ:ð -úsV   ÁYÈYÌ	YÌ)YÍ	YÍYÍ/Y!ÎY'Ñ<A"Y-ÓY-Ó(Y-ÔY-ÔA Y-Ù-Zc                	óî  € VP                  4       pVP                  pVP                  p\        P                  ^8„  ;'       g    \        P                  ^8H  ;'       d    TpVP
                  pV P                  VVVVVR7      pV F\  w  r‰p
V P                  W�.V	4      pV P                  VP                  V4       \        P                  RV4       V	P                  V4       K^  	  V P                  4        R# )rT  )rã  z"ComboKernels: generated kernel %s.N)Úget_subkernel_nodesÚuse_custom_partition_algorá  r   Úcombo_kernel_allow_mixed_sizesrã  r   r?  rC  Úsnodesrv  rÐ  r:  rI  )r|   Úcombo_kernel_noder÷  rø  rá  râ  rã  r  r  rs   rx  r´  s   &&          r[   Úcodegen_combo_kernelÚ#SIMDScheduling.codegen_combo_kernel  sö   € Ø+×?Ñ?ÓAˆØ 1× KÑ KÐØ+×;Ñ;ˆÜ×;Ñ;¸aÑ?÷ 
ð 
Ü×1Ñ1°QÑ6×PÐPÐ;Pð 	ð  1×EÑEÐà×:Ñ:ØØ!ØØØ!5ð ;ó 
Ðó $4ÑˆH˜aØ×,Ñ,¨XÐ7JÈFÓSˆKØ× Ñ Ð!2×!9Ñ!9¸;ÔGÜ�I‰IÐ:¸KÔHØ×Ñ˜{Ö+ñ	 $4ð 	×&Ñ&Ö(r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   úlist[CandidateTiling]rY   )rZ   s   "r[   r\   r    s   € ÷ Cñ CÐ@Uñ Cr]   c           
     	óÔ  a aa
€ S^8H  pR V VV
3R llpVP                  4       w  po
\        V4      ^8:  d   \        S
4      ^8:  g   \        VS
,           4      '       d   . # VP                  4       w  po
T! TV'       d   TMS
VP                  V4      4      pV Uu. uF@  p\	        S P                  VP                  VS4      VP                  VP                  R7      NKB  	  p	pV	# u upi )rT  c               ó    € V ^8„  d   QhRRRR/# )rU   Úis_pointwiser�   rX   r+  rY   )rZ   s   "r[   r\   Ú6SIMDScheduling.candidate_tilings.<locals>.__annotate__   s   € ÷ b	ñ b	 dð b	Ð;Pñ b	r]   c                ó6  <€ \        VP                  4      \        V4      8w  d   \        RVP                  : RV: 24      hVP                  VP                  .p\
        ;QJ d9    R \        P                  P                  V4       4       F  '       d   K   RM.	  RM*! R \        P                  P                  V4       4       4      '       g   \        R4      h\        P                  P                  V4       Uu. uFH  pVP                  \        P                  P                  9  g   K.  \        V\        4      '       g   KF  VNKJ  	  pp\        VP                   Uu. uF  qDP                  NK  	  up4      pR R lp\!        SP#                  V! V4      .V 4      R	^ R
7      .pV EFQ  p\        P                  P$                  P'                  VP(                  VP                  4      p	\        V	4      \        V4      8w  d$   \        R\        V	4       R\        V4       24      h V	P)                  ^4      ^,           p
V
\        V4      8X  d   K©  \
        ;QJ d    R WšR  4       F  '       d   K   RM	  RM! R WšR  4       4      '       d   Ké   T! TRT
 4      T! YR 4      3p\        P                  P$                  P-                  \/        R \1        Y4       4       4      4      pTP                  T9   d
   T^,          p\         P3                  T^ ,          4      '       d
   T^,          p\         P3                  T^,          4      '       d
   T^,          p\        P                  P$                  P-                  T\/        \        P                  ! TS4      4      ,
          4      ^ 8¼  g   EK  TP5                  \!        SP#                  T! TRT
 4      T! YR 4      .S4      TTP                  R7      4       EKT  	  V# u upi u upi   \*         d     EKq  i ; i)z@
Compute tiling candidates by dividing up the iteration ranges.
zrw.range_vars=z ranges=c              3  óN   "  € T F  p\        V\        \        34      x € K  	  R # 5ir_   )r[  r   r   )rX  rg  s   & r[   rY  ÚHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>,  s&   é € ð áE�Cô ˜3¤¬GÐ 4×5Ð5ÛEùs   ‚#%FTz,expected all deps to be MemoryDep or StarDepc               ó    € V ^8„  d   QhRRRR/# )rU   r‡  rD  rX   rq   rY   )rZ   s   "r[   r\   ÚKSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.__annotate__9  s   € ÷ Hñ HÐ(<ð HÀñ Hr]   c                óf   € \         P                  P                  P                  \	        V 4      4      # r_   rÒ  )r‡  s   &r[   Úcollapse_rangesÚNSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.collapse_ranges9  s"   € Ü—w‘w×'Ñ'×0Ñ0´¸vÓ1FÓGÐGr]   Únone)rª  rj   Úscorez*expected len(strides) == len(ranges), got r†  c              3  ó*   "  € T F	  q^ 8H  x € K  	  R# 5ir�  rY   rW  s   & r[   rY  r2  S  s   é € Ð;©? a ž6«?ùs   ‚Nc              3  ó<   "  € T F  w  rV^ 8w  g   K  Vx € K  	  R# 5ir�  rY   )rX  r·  ru  s   &  r[   rY  r2  b  s   é € ð "Ù1E¡ ÈÐSTÉŸšÓ1Eùs   ‚’
©rª  r9  rj   )r_  Ú
range_varsr­   rÔ  rÞ  r`  rÎ  rZ  r[  rj   r>   rý   r>  r[  r   r   ÚCandidateTilingÚcreate_partial_tilingrþ   Ústride_hintsr¢   Ú
ValueErrorr  r9   rr  Úis_good_sizer  )r.  r‡  ÚrwÚdep_sourcesrg  ÚdepsÚwrite_namesr6  ÚtilingsrK  ÚsplitÚtiled_groupsr9  rÀ  r¾  r«  s   &&&          €€€r[   Útile_rangesÚ5SIMDScheduling.candidate_tilings.<locals>.tile_ranges   s‡  ø€ ô �2—=‘=Ó!¤S¨£[Ô0Ü$¨¨¯©Ñ'7°y¸¹	Ð%BÓCÐCð Ÿ8™8 R§Y¡YÐ/ˆKß“3ñ ä$Ÿ?™?×8Ñ8¸ÔEó—3—3’3ñ ä$Ÿ?™?×8Ñ8¸ÔEó÷ ò ô %Ð%SÓTÐTô %Ÿ?™?×8Ñ8¸ÔEóáE�CØ—8‘8¤1§7¡7×#:Ñ#:Ñ:ô ô ˜s¤I×.÷ �ÙEð ð ô %¸"¿)º)Ó%D¹)°3§h¤h¹)Ñ%DÓEˆKõHô
  Ø×4Ñ4Ù(¨Ó0Ð1°<óð  ØôðˆGô �ÜŸ'™'×*Ñ*×7Ñ7¸¿	¹	À2Ç=Á=ÓQ�Ü�w“<¤3 v£;Ô.Ü(ðÜ" 7›|˜n¨E´#°f³+°ð@óð ð
Ø#ŸM™M¨!Ó,¨qÕ0�EØ¤ F£Ô+Ù ß“sÑ;¨7°6©?Ó;—s—s’sÑ;¨7°6©?Ó;×;Ò;ñ !ð <ñ $ F¨6¨E NÓ3Ù# F¨6 NÓ3ð �ô Ÿ™×(Ñ(×:Ñ:Ü!ñ "Ü14°VÔ1Eó"ó ó�ð
 —8‘8˜{Ô*à˜Q•J�EÜ"×/Ñ/°¸Qµ×@Ò@Ø˜Q•J�EÜ"×/Ñ/°¸Qµ×@Ò@Ø˜Q•J�Eô —G‘G×$Ñ$×6Ñ6Ø¤¬i¯oªo¸fÐFVÓ.WÓ XÕXóð ÷ð
 —N‘NÜ'Ø#&×#<Ñ#<á$3°F¸6¸E°NÓ$CÙ$3°F¸6°NÓ$Cð!"ð !0ó$ð #(Ø!$§¡ô
÷ñY ðt ˆNùòcùò &EøôB "ô Ûðús<   Ã:+O>Ä*O>ÅO>ÅPÈ%(PÉ
PÉPÉ0PÐPÐPr<  )	r¥  r_  r   Ú"pointwise_or_reduction_read_writesr>  Úcomplete_partial_tilingrª  r9  rj   )rÀ  rô   rp   r¾  r.  rJ  Úpointwise_rangesÚpartial_tilingsrª  Úfull_tilingsr«  s   f&&f      @r[   Úcandidate_tilingsÚ SIMDScheduling.candidate_tilings  sþ   ú€ ð '¨!Ñ+ˆ÷b	ò b	ðH .2¯_©_Ó->Ñ*ÐÐ*äÐ Ó! QÔ&ÜÐ$Ó%¨Ô*Ü$Ð%5Ð8HÕ%H×IÒIàˆIð .2¯_©_Ó->Ñ*ÐÐ*Ù%Øß ,ÑÐ2BØ×3Ñ3°LÓAó
ˆñ *ó	
ñ *�ô Ø×2Ñ2Ø—M‘M 5¨/óð —l‘lØ—[‘[÷ñ *ð 	ð 	
ð Ðùò	
s   ÂAC%c               ó$   € V ^8„  d   QhRRRRRR/# )rU   Ú	pw_tilingrD  Úreduction_tilingrX   úimmutable_dict[str, sympy.Expr]rY   )rZ   s   "r[   r\   r  £  s$   € ÷ 
ñ 
Ø,ð
Ø@Tð
à	(ñ
r]   c           	     ó`  € RpRp\        V4      \        V4      8”  d$   \        R\        V4       R\        V4       24      h\        V4      \        V4      8”  d$   \        R\        V4       R\        V4       24      h\        . \        V\        V4      ) R VRR7      O\        WBRR7      O4      # )	z;
Create a tiling dict from pointwise and reduction splits.
z1expected len(pw_tiling) <= len(pw_prefixes), got r†  z?expected len(reduction_tiling) <= len(reduction_prefixes), got NF)Ústrict)r/  r2  r  )r0  r1  )r_  r­   r   rr  )rÀ  rT  rU  Úpw_prefixesÚreduction_prefixess   &&&  r[   Úcreate_tilingÚSIMDScheduling.create_tiling¢  s×   € ð &ˆØ+ÐÜˆy‹>œC Ó,Ô,Ü ðÜ˜9“~Ð& e¬C°Ó,<Ð+=ð?óð ô ÐÓ ¤3Ð'9Ó#:Ô:Ü ðÜÐ+Ó,Ð-¨U´3Ð7IÓ3JÐ2KðMóð ô
 ðÜ�[¤# i£. Ð!2Ð3°YÀuÔMðäÐ'À%ÔHðó
ð 	
r]   c               ó$   € V ^8„  d   QhRRRRRR/# )rU   rª  rD  r.  r�   rX   rV  rY   )rZ   s   "r[   r\   r  ¾  s(   € ÷ 
ñ 
à$ð
ð ð
ð 
)ñ	
r]   c                	óV   € T P                  V'       d   TM. V'       g   V4      # . 4      # r_   )r[  )rÀ  rª  r.  s   &&&r[   r?  Ú$SIMDScheduling.create_partial_tiling½  s0   € ð × Ñ ß"‰F¨ß&ˆFó
ð 	
à,.ó
ð 	
r]   c               ó(   € V ^8„  d   QhRRRRRRRR/# )rU   rª  r¶  rp   rq   r¾  rX   rV  rY   )rZ   s   "r[   r\   r  É  s2   € ÷ /ñ /à%ð/ð ð/ð $ð	/ð
 
)ñ/r]   c                ó°   € \        VP                  4       4      pRV9   pW#,          pV\        V4      ,          .pV'       d   WG3MWt3pV P                  ! V!  # )zR
Given a tiling for only pointwise or reduction dimensions, adds the missing one.
r  )r§  rò   r9   r[  )	rÀ  rª  rp   r¾  Úsplitsr.  Útotal_numelÚmissing_tilingÚtiling_argss	   &&&&     r[   rM  Ú&SIMDScheduling.complete_partial_tilingÈ  s^   € ô �f—m‘m“oÓ&ˆØ˜f‘}ˆàÕ-ˆØ%¬°fÓ(=Õ=Ð>ˆ÷ )5ˆVÑ$¸>Ð:Rð 	ð × Ò  +Ñ.Ð.r]   c               ó   € V ^8„  d   QhRR/# )rU   rX   z%list[immutable_dict[str, sympy.Expr]]rY   )rZ   s   "r[   r\   r  Þ  s   € ÷ Sñ Sð
 
/ñSr]   c                óâ  aaaa€ R VV3R lloR VV3R llpS^8H  p\         \        \        \        P                  3,          ,          ! 4       p\
        P                  ! V4       EFn  p\        V\        P                  4      '       g   K&  VP                  4       pV'       g   \        V^,          4      ^ 8X  d   KW  S! V^ ,          S4      p	S! V^,          S4      p
Wš3.pVP                  P                  4        Uu. uF9  p\        V\        4      '       g   K  \        VP                  4      ^ 8”  g   K7  VNK;  	  ppV EFI  o. SP                  P!                  4       Op\        P"                  P$                  p\&        P(                  P*                  p^ p\-        V4       F.  w  pw  ppVV,          pTpVP/                  VS4      '       g   K.   M	  VP1                  VS4      '       g   K­  V^,           pVRV pV'       d   RMVVR pV! VV^ ,          S4      pV'       d   \        P"                  P$                  3MV! VV^,          S4      p\        V4      '       g   EK#  \        V4      ^ 8”  g   EK6  VP3                  VV34       EKL  	  \4        P6                  ! \9        V!  !   F'  w  ppVP;                  V P=                  VV4      4       K)  	  EKq  	  \?        V\        RR7      pV# u upi )z°
Creates N-dimensional tiling candidates, attempting to simplify loads/stores
by tiling the kernel into higher dimensions.

Returns a list of tilings ranked by dimensionality.
c               ó$   € V ^8„  d   QhRRRRRR/# )rU   ÚdimsrD  Úfallback_numelrq   rX   útuple[sympy.Expr, ...]rY   )rZ   s   "r[   r\   Ú3SIMDScheduling.get_nd_tilings.<locals>.__annotate__ë  s)   € ÷ 	Oñ 	OØ&ð	OØ8Bð	Oà#ñ	Or]   c                ó˜  <€ V '       g   V3# \        ^4      p\        P                  P                  P	                  S^4      '       d=   \        P                  P                  P                  S^4      '       d   \        V^4      p\        ^ \        V 4      V,
          4      pV^,           p\        V RV 4      pV3\        WR 4      ,           # )z=
Collapse dimensions to the maximum allowed number of tiles.
N)rd   r>   rý   rþ   rÿ   r®  r   r  r_  r9   rØ   )rj  rk  rc   Únum_leading_dimsÚfirst_trailing_dimÚcollapsed_leading_dimÚpointwise_numelr¾  s   &&    €€r[   Úcollapse_dimsÚ4SIMDScheduling.get_nd_tilings.<locals>.collapse_dimsë  s¶   ø€ ÷ Ø&Ð(Ð(Ü% aÓ(ˆIÜ�w‰w×Ñ×7Ñ7Ø ÷ò ä—'‘'×"Ñ"×6Ñ6°È×JÒJô   	¨1Ó-�	Ü" 1¤c¨$£i°)Õ&;Ó<ÐØ!1°AÕ!5ÐÜ$1°$Ð7JÐ8JÐ2KÓ$LÐ!Ø)Ð+¬e°DÐ9LÐ4MÓ.NÕNÐNr]   c               ó   € V ^8„  d   QhRR/# )rU   rX   rl  rY   )rZ   s   "r[   r\   rm  ÿ  s   € ÷ 	6ñ 	6à#ñ	6r]   c                ó&  <€ . pV  F¡  w  rE\         P                  ! SP                  V4      p\        ^VP	                  \
        4      VP	                  \        4      ,           \        V4      4      p\         P                  ! WdWW4      pVe
   V^ ,          MV.p	VP                  V	4       K£  	  V U
u. uFO  p
\        P                  P                  P                  V
\        P                  P                   4      '       d   KM  V
NKQ  	  pp
S! W24      # u up
i ©rU   )r?   Úget_subexpr_involving_symbolr¢   r  rÏ  r   r   r_  Úmatch_mod_div_block_exprr@  r>   rý   rþ   rÿ   r™   rš   r›   )rn   Úranges_to_tilerc  rª  Úvarrp   r¢   Únum_dimsÚmatch_resultrj  Údimrs  rg  s   &&&        €€r[   Útile_var_rangesÚ6SIMDScheduling.get_nd_tilings.<locals>.tile_var_rangesÿ  sí   ø€ ð ˆFÛ(‘
�Ü+×HÒHÈÏÉÐTWÓX�ô ØØ—K‘K¤Ó)¨E¯K©K¼Ó,HÕHÜ˜Ó'ó�ô  3×KÒKØ ó �ð +7Ò*B�| A–ÈÈ�Ø—‘˜dÖ#ñ! )ñ* "óá!�CÜ—w‘w×'Ñ'×?Ñ?ÀÄUÇWÁWÇ[Á[×Q÷ �Ù!ð ð ñ ! Ó5Ð5ùòs   Â/A
DÃ>DNT)r'  Úreverse) r   r   rk   r™   rB  rG   rV  r[  r   rÚ  r¥  r_  rÓ  Úreads_and_writesr   r‡  r�   rš   r›   r>   rý   rþ   r*  Ústatically_known_geqrÿ   r  rÎ  Úproductrr  r+  r[  r"  )rÀ  r¨  rr  r¾  r  r.  rG  rô   Únode_rangesÚdefault_pointwise_tilingÚdefault_reduction_tilingÚnode_tilingsrg  Úmemory_depsÚall_var_rangesÚpointwise_vars_numelrþ   Úpointwise_end_idxr,  Ú_varrp   Úreduction_start_idxÚpointwise_var_rangesÚreduction_var_rangesÚpointwise_tilingrU  Úranked_tilingsrs  s   &&ff        `              @r[   Úget_nd_tilingsÚSIMDScheduling.get_nd_tilingsÝ  sÎ  û€ ÷	Oñ 	O÷(	6ñ 	6ð@ '¨!Ñ+ˆÜœ^¬C´·±¨OÕ<Ö=Ó?ˆÜ#×*Ò*¨=×9ˆDÜ˜d¤I×$;Ñ$;×<Ò<Ùð Ÿ/™/Ó+ˆKß¤C¨°A­Ó$7¸1Ô$<Ùñ (5°[Àµ^À_Ó'UÐ$Ù'4°[Àµ^À_Ó'UÐ$Ø5ÐPÐQˆLð  ×+Ñ+×<Ñ<Ô>óá>�CÜ˜c¤9×-ô ä25°c·j±j³/ÀAÑ2E÷ �Ù>ð ð ô
 #�ð "7 3§:¡:×#3Ñ#3Ó#5Ð!6�Ü',§w¡w§{¡{Ð$ÜŸ7™7×+Ñ+�Ø$%Ð!Ü*3°NÖ*CÑ&�C™˜$ Ø(¨EÕ1Ð(Ø(+Ð%Ø×4Ñ4Ø,¨o÷ô ñ ñ +Dð  ×7Ñ7Ø(¨/÷ò ñ ð '8¸!Õ&;Ð#Ø'5Ð6JÐ7JÐ'KÐ$ç(‘D¨nÐ=PÐ=QÐ.Rð %ñ $3Ø(¨+°a­.¸/ó$Ð ÷
 $ô —W‘W—[‘[‘Ná(Ø,¨k¸!­n¸oóð !ô Ð'×(Õ(¬SÐ1AÓ-BÀQ×-FØ ×'Ñ'Ð)9Ð;KÐ(L×MñQ #ôX 7@×6GÒ6GÜ�lÑ#ô7Ñ2Ð Ð"2ð —‘˜C×-Ñ-Ð.>Ð@PÓQÖRô7ñG :ôT  ØÜØô
ˆð Ðùò}s   Ã<K,ÄK,Ä5K,c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
rU   r¨  rê  rr  rq   r¾  r†  rP   rX   Ú_TilingSelectionrY   )rZ   s   "r[   r\   r  t  sC   € ÷ p<ñ p<à.ðp<ð $ðp<ð $ð	p<ð
 /ðp<ð 
ñp<r]   c                ó(	  aaaaaaaaaaaaaa € SP                   '       g   RMSP                   P                  oSP                  P                  oSP                  P                  oSP                  P
                  pS Uu. uF  qeV,          NK  	  upoS Uu. uF  qeV,          NK  	  upo\        P                  P                  P                  p\        P                  ! V! \        S4      4      V! S4      8H  VVV3R l4       \        P                  ! V! \        S4      4      V! S4      8H  VVV3R l4       / o. pRR VVVVVVVVV3	R lllp	VP                  V	! RR7      V	! RR7      34       S'       d%   VP                  V	! S3RRR	7      V	! RR7      34       SSP                  P                  4       ,          p
V
 F&  pVP                  V	! V3RR7      V	! RR7      34       K(  	  \!        ^R
7      ^8X  dH   S^8X  dA   \"        P$                  ! V
^4       F%  pVP                  V	! VRR7      V	! RR7      34       K'  	  . pV F¸  w  rÞ\'        V P)                  VP*                  VP*                  4      \-        VP.                  4      \-        VP.                  4      ,           R7      pV P)                  VP.                  VP.                  4      pVP                  VVVP0                  VP0                  ,           34       Kº  	  V P)                  S.S.4      pRoRo\-        SP2                  P5                  4       4      o \-        SP                  P5                  4       4      S ,           oVVV 3R lpR V3R llp\7        VVR7       Fô  w  pppV P9                  SSSVP:                  4      '       g   VP:                  V8X  d’   \=        VP:                  4      S^8X  d   ^M^ ,
          pV\!        ^R
7      8”  dF   \>        PA                  RV\        PB                  PD                  PF                  PH                  4       K·  V! VP:                  VV4      u # VP:                  V8X  g   Kà  V! VP:                  VV4      u # 	  \K        VRR4      # u upi u upi )zb
Generates a tiling, and a score of each tile according to each tile's coalesced memory accesses.
Nc                 ó   <€ S R S R S  2# ©r²   rY   )r¨  rr  Ú	pw_rangess   €€€r[   r$  Ú8SIMDScheduling.compute_tiling_strategy.<locals>.<lambda>�  s   ø€ �y�k  OÐ#4°B°}°oÑFr]   c                 ó   <€ S R S R S  2# r™  rY   )r¨  Ú
red_rangesr¾  s   €€€r[   r$  r›  ”  s   ø€ �z�l " _Ð$5°R¸°ÑGr]   Fc               ó(   € V ^8„  d   QhRRRRRRRR/# )rU   Úvars_to_userl  Úuse_split_varr�   r.  rX   Ú	_SubSplitrY   )rZ   s   "r[   r\   Ú<SIMDScheduling.compute_tiling_strategy.<locals>.__annotate__œ  s6   € ÷ O	ñ O	Ø/ðO	àðO	ð ðO	ð ñ	O	r]   c                óH  <	€ V'       d   SMSpV'       d   SMSpV'       g$   V'       d   \        V.. ^ 4      # \        . . ^ 4      # \        V 4      W3pSP                  V4      ;p'       d   V# V'       d   SMSp. p. p	^p
^ p\        Ws4       EF8  w  rÍWÀ9  d'   W­,          p
SP                  P                  V^ 4      pK2  V'       d¹   VS8X  d²   SP
                  pVf   \        R4      hVP                  p\        WÞP                  4      pVP                  V
V,          4       V	P                  VP                  4       VP                  V4       V	P                  SP                  P                  V^ 4      4       ^p
^ pKò  W­,          p
VP                  V
4       V	P                  SP                  P                  V^ 4      4       ^p
EK;  	  V
^8w  g   V'       d3   \        V4      ^ 8X  d#   VP                  V
4       V	P                  V4       \        V	4      p\        \        V4      4       Fd  p\        P                  P                   P#                  VV,          ^ R7      p\%        V^4      p\'        V	V,          V,          ^,          4      V	V&   Kf  	  \        W‰V4      pVSV&   V# )zE
Generate a tiling, and a tiling score, given vars to use as splits.
z"expected var_tiling to not be None©rª  )r¡  r›  r.  rr  Úcoalesced_by_varÚsuggested_splitr­   Útiling_factorr   r  r9  r_  r
  r–  r>   rý   rþ   r  r   rW   )rŸ  r   r.  r‡  Útarget_numelr'  r:  Úsplitting_varsrb  Úsplit_scoresÚprodÚprev_var_coalesced_scorerH  Úv_rangeÚ
var_tilingÚtileÚ	remainderÚcoalesced_memoryrú  r0  Ú	sub_splitÚall_iter_varsÚall_red_varsr†  rr  rš  r�  r¾  Úscored_sub_splitÚ
tiling_vars   &&&                  €€€€€€€€€r[   Úprocess_node_varsÚASIMDScheduling.compute_tiling_strategy.<locals>.process_node_varsœ  s_  ø€ ÷ #/‘Y°JˆFß.:™?ÀˆLçßÜ$ l ^°R¸Ó;Ð;ä$ R¨¨QÓ/Ð/ä˜Ó$ mÐBˆCØ&×*Ñ*¨3Ó/Ð/ˆsÖ/Ø�
ç.:™]ÀˆNàˆFØˆLØˆDØ'(Ð$ô " .×9‘
�ØÔ'Ø•O�DØ/@×/QÑ/Q×/UÑ/UØ˜1ó0Ð,ñ ç  Q¨*¤_Ø!2×!BÑ!B�JØ!Ò)Ü,Ð-QÓRÐRà%×3Ñ3�DÜ (¨×2JÑ2JÓ K�Ià—M‘M $¨Õ"2Ô3Ø ×'Ñ'¨
×(8Ñ(8Ô9à—M‘M $Ô'Ø ×'Ñ'Ð(9×(JÑ(J×(NÑ(NÈqÐRSÓ(TÔUà�DØ/0Ð,áà•�Ø—‘˜dÔ#Ø×#Ñ#Ð$5×$FÑ$F×$JÑ$JÈ1ÈaÓ$PÔQØ“ñ= :ð@ �qŒyŸ\¬c°&«k¸QÔ.>Ø—‘˜dÔ#Ø×#Ñ#Ð$<Ô=ä" <Ó0Ðô œ3˜v›;Ö'�Ü—G‘G×$Ñ$×6Ñ6°v¸aµyÈ2Ð6ÓN�Ü˜˜1“I�Ü"% l°1¥o¸Õ&9¸AÕ&=Ó">�˜Q“ñ (ô
 " &Ð8HÓIˆIØ$-Ð˜SÑ!ØÐr]   T)r.  )r   r.  ©rV   )r9  gffffffð?g®Gázð?c                ó  <€ R pV ^ ,          P                   P                  4        F2  p\        P                  V4      '       g   VS,          pK)  VS,          pK4  	  SR,          pV ^ ,          P                  V,           ) V,          # )g      ð?gš™™™™™©?)rª  rò   r>  rB  r9  )r  Úscore_factorÚ	tile_sizeÚuncoalesced_penaltyÚ"bad_size_additional_tiling_penaltyÚgood_size_tiling_penaltyÚtotal_uncoalesceds   &   €€€r[   Ú	score_modÚ9SIMDScheduling.compute_tiling_strategy.<locals>.score_mod.  ss   ø€ ØˆLØ˜q�TŸ[™[×/Ñ/Ö1�	Ü&×3Ñ3°I×>Ò>Ø#/Ð2TÕ#T’Là#/Ð2JÕ#J’Lñ	 2ð #4°dÕ":Ðà�q•T—Z‘ZÐ"5Õ5Ð6¸ÕEÐEr]   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rU   rª  r¶  r‡  rV  r±  rW   rX   r–  rY   )rZ   s   "r[   r\   r¢  <  s0   € ÷ 	ñ 	Ø)ð	à9ð	ð "ð	ð ñ		r]   c                óV   <€ \        V V\        V\        SV,
          ^ 4      R7      4      # )r   )Ú	coalescedÚuncoalesced)r–  ÚMemoryCoalescingr  )rª  r‡  r±  Útotal_memorys   &&&€r[   Úmake_selectionÚ>SIMDScheduling.compute_tiling_strategy.<locals>.make_selection<  s3   ø€ ô
 $ØØÜ Ø.Ü # LÐ3CÕ$CÀQÓ Gôóð r]   r&  zmFound optimal tiling with %s tiles but torch._inductor.config.triton.max_tiles set to %s. Consider increasing)rY   FF)&r¦  r{  Únorm_read_writesr)  rÀ  rn   r>   rý   rþ   r  r`   Ú_checkr9   r  r¥  r  rd   rÎ  Úcombinationsr>  r[  rb  r
  rª  r±  Úuncoalesced_addrsrò   r"  Útiling_is_compatiblerª  r_  Úperf_hint_logr¼  ra   r   rb   rc   r–  )!rÀ  r¨  rr  r¾  r†  r‡  rH  Úget_hintÚscore_splitr·  Úoverlapping_iter_varsrŸ  rG  ÚpwÚredÚ	candidater‡  Údefault_tilingrÁ  rÉ  Úcandr±  Ú
tiling_lenr³  r´  r¾  r¿  rš  r�  rµ  r¶  rÈ  rÀ  s!   &ffff                  @@@@@@@@@@r[   Úcompute_tiling_strategyÚ&SIMDScheduling.compute_tiling_strategys  s  ÿý€ ð %×4×4Ð4ñ à"×2Ñ2×6Ñ6ð 	ð *×:Ñ:×EÑEˆØ(×9Ñ9×EÑEˆØ"×3Ñ3×>Ñ>ˆá(5Ó6© 1˜A—Y�Y©Ñ6ˆ	Ù)5Ó6© A˜Q—i�i©Ñ6ˆ
ô —7‘7×#Ñ#×5Ñ5ˆÜ�ŠÙ”] 9Ó-Ó.±(¸?Ó2KÑKÞFô	
ô
 	�ŠÙ”] :Ó.Ó/±8¸OÓ3LÑLÞGô	
ð 24Ðà9;ˆ÷O	÷ O	ò O	ðd 	×Ñá!¨tÔ4Ù!¨uÔ5ðô	
÷ Ø×Ñá%Ø#˜°TÈôñ &°5Ô9ð	ôð Ð-×>Ñ>×CÑCÓEÕEð 	ó 'ˆAØ×Ñá% q d¸Ô>Ù%°5Ô9ðöñ 'ô  Ô# qÔ(¨_ÀÔ-AÜ(×5Ò5Ð6KÈQÖO�Ø×"Ñ"á)¨+ÀDÔIÙ)°uÔ=ðöñ  Pð WYˆÛ"‰GˆBÜ'Ø×!Ñ! "§)¡)¨S¯Z©ZÓ8Ü˜"Ÿ/™/Ó*¬S°×1AÑ1AÓ-BÕBôˆIð ×,Ñ,¨R¯_©_¸c×>NÑ>NÓOˆLØ�N‰NØ˜L¨"×*=Ñ*=À×@TÑ@TÕ*TÐUöñ #ð ×*Ñ*¨OÐ+<¸Ð>OÓPˆð .3Ð*Ø#(Ð äÐ 1× CÑ C× JÑ JÓ LÓMÐô Ð!×2Ñ2×9Ñ9Ó;Ó<Ð?PÕPð 	÷	F÷	ð 	ô 5;¸7È	×4RÑ0ˆD�,Ð 0à×(Ñ(Ø! ?°OÀTÇ[Á[÷ò ð —;‘; .Ô0ô ! §¡Ó-°oÈÔ6J±ÐPQÕR�
Ø¤°aÔ 8Ô8Ü!×&Ñ&ð9à"ÜŸ™×.Ñ.×5Ñ5×?Ñ?ô	ñ á% d§k¡k°<ÐAQÓRÒRð �{‰{˜nÖ,Ù% d§k¡k°<ÐAQÓRÒRñ/ 5Sô2   °°dÓ;Ð;ùòy 7ùÚ6s   Á?R
ÂRc               ó(   € V ^8„  d   QhRRRRRRRR/# )rU   r¨  rê  rp   rq   r¾  rª  r¶  rY   )rZ   s   "r[   r\   r  g  s2   € ÷ 
ñ 
à.ð
ð ð
ð $ð	
ð
 &ñ
r]   c                	óØ   aa€ \        S\        4      '       g   \        R \        S4       24      h\        ;QJ d     VV3R lV 4       F  '       d   K   R# 	  R# ! VV3R lV 4       4      # )z"expected tiling to be a dict, got c              3  óÐ   <"  € T F[  p\        V\        P                  4      '       g   K%  \        P	                  SP                  4       VP                  4       SR 7      x € K]  	  R# 5ir¤  )r[  r   rÚ  rt   rÆ  rò   r¥  )rX  rô   r¾  rª  s   & €€r[   rY  Ú6SIMDScheduling.tiling_is_compatible.<locals>.<genexpr>p  sW   øé € ð 
ñ &�Ü˜$¤	× 7Ñ 7×8ô	ŒJ×$Ñ$Ø—‘“ §¡Ó!2ÀOð %÷ ð ó &ùs
   ƒ"A&ª<A&FT)r[  r©  r­   r\  r`  )rÀ  r¨  rp   r¾  rª  s   &&&ffr[   rÏ  Ú#SIMDScheduling.tiling_is_compatiblef  sh   ù€ ô ˜&¤$×'Ò'Ü Ð#EÄdÈ6ÃlÀ^Ð!TÓUÐUß‹sõ 
ñ &ó	
�sŒsð 	
Šsð 	
ˆsõ 
ñ &ó	
ó 
ð 	
r]   c               ó(   € V ^8„  d   QhRRRRRRRR/# )rU   r¨  rê  rp   rq   r¾  r’  zlist[dict[str, sympy.Expr]]rY   )rZ   s   "r[   r\   r  y  s2   € ÷ ñ à.ðð ðð $ð	ð
 4ñr]   c                	óP   € V F  pV P                  WW54      '       g   K  Vu # 	  R # r_   )rÏ  )rÀ  r¨  rp   r¾  r’  rª  s   &&&&& r[   Úget_first_compatible_tilingÚ*SIMDScheduling.get_first_compatible_tilingx  s+   € ó %ˆFØ×'Ñ'¨¸o×VÔVØ’ñ %ñ r]   c               ó    € V ^8„  d   QhRRRR/# )rU   r†  rÉ  rX   r¶  rY   )rZ   s   "r[   r\   r  ‡  s    € ÷ 	ñ 	ð
 6ð	ð 
ñ	r]   c                	ó4   € V P                  WW44      ^ ,          # r`  )ro  )rÀ  r¨  rp   r¾  r†  s   &&&&&r[   r¶  ÚSIMDScheduling.select_tiling†  s$   € ð ×(Ñ(Ø /ó
à
õð 	r]   c               ó    € V ^8„  d   QhRRRR/# )rU   r†  rÉ  rX   z:tuple[dict[str, sympy.Expr], dict[str, sympy.Expr] | None]rY   )rZ   s   "r[   r\   r  “  s!   € ÷ 9ñ 9ð
 6ð9ð 
Dñ9r]   c                óV   € V P                  WW44      pVP                  VP                  3# )zª
Heuristics to decide how to tile kernels.
Currently, we tile based on stride-1 dimensions.

Returns:
    `(tile1, tile2, reduction_numel)` s.t. `tile1 * tile2 == numel`
)Úselect_tiling_with_memoryrª  r«  )rÀ  r¨  rp   r¾  r†  Ú	selections   &&&&& r[   ro  Ú$SIMDScheduling.get_tiling_and_scores’  s2   € ð ×1Ñ1Ø /ó
ˆ	ð ×Ñ ×!8Ñ!8Ð8Ð8r]   c               ó    € V ^8„  d   QhRRRR/# )rU   r†  rÉ  rX   r–  rY   )rZ   s   "r[   r\   r  §  s$   € ÷ U<ñ U<ð
 6ðU<ð 
ñU<r]   c                ó>  € V^8H  pV P                  V.V.4      p\        P                  ! V4       F²  p\        VP                  \
        P                  4      '       g   K/  VP                  P                  4       R8X  g   KP  \        P                  P                  '       g   Kr  VP                  4       pV^ ,          p	V^,          p
V P                  Wš4      p\        VRR4      u # 	  \        P                  P                  P                  P                  '       d;   V'       d3   \        P                  P                   '       g   V P#                  WW44      # V'       g!   \        P                  P$                  '       d   \'        ^R7      ^8:  d»   \(        P*                  \,        P.                  8:  d‹   \        P                  ! V4       Fp  p\        P                  P$                  '       d   K%  \1        V P3                  WrV4      4      ^ 8”  g   KG  \(        P5                  \6        P8                  ! R4      4        M	  \        VRR4      # \;        4       p\<        P>                  ! 4       p\        P                  ! V4       Fv  pV P3                  WrV4       F]  pVP@                  V9   d   K  VP@                  e   VPC                  VP@                  4       WÞ;;,          VPD                  ,          uu&   K_  	  Kx  	  VPG                  4        UUu. uF  w  rïVPH                  NK  	  ppp\'        ^R7      ^8¼  dR   V'       dJ   R R lp\K        ^\1        V4      4       F*  pV! V^ ,          VV,          4      pVf   K   V.V,           p M	  \1        V4      ^8”  d   \(        P5                  RV4       \        P                  P                   '       d   V PM                  WV4      V,           pV PO                  WVV4      ;p'       d   \        VRR4      # \        VRR4      # u uppi )zæ
Heuristics to decide how to tile kernels.
Currently, we tile based on stride-1 dimensions.

Same selection as get_tiling_and_scores(), additionally reporting the
coalesced/uncoalesced memory under the selected tiling when known.
r½  Nr¹  z½
                                Reduction over non-contiguous dims.
                                Consider setting config.triton.tile_reductions to True.
                                c               ó$   € V ^8„  d   QhRRRRRR/# )rU   Útiling0r¶  rÁ  rX   r¹  rY   )rZ   s   "r[   r\   Ú>SIMDScheduling.select_tiling_with_memory.<locals>.__annotate__  s$   € ÷ !"ñ !"Ø.ð!"Ø9Nð!"à-ñ!"r]   c           	     ó  € V R ,          V P                  R^4      r2VR ,          VP                  R^4      rT\        P                  P                  P                  pV! W5,
          4      ^ 8X  d   R# V! W5,
          4      ^ 8  d   WE3W#3uw  r#w  rEV! W5,
          4      ^ 8”  g   \        RV! W5,
          4       24      h\        P                  P                  P                  W54      '       g   R# RVR\        W54      R VRV R,          /pV# )r  r2  Nz expected hint(a1 - b1) > 0, got r/  r0  )r.  r>   rý   rþ   r­  r­   r•  r   )rð  rÁ  Úa0Úa1Úb0Úb1rI  Ú
new_tilings   &&      r[   Úconvert_tiling_to_3dÚFSIMDScheduling.select_tiling_with_memory.<locals>.convert_tiling_to_3d  só   € ð ! � w§{¡{°3¸Ó':�BØ  � w§{¡{°3¸Ó':�Bô —w‘w×'Ñ'×>Ñ>�Ù˜�“= AÔ%ÙÙ˜�“= 1Ô$à*,¨°B°8Ð&‘H�R™h˜rá˜R�W›¨Ô)Ü(Ø:¹4ÀÅ»=¸/ÐJóð ô —w‘w×'Ñ'×DÑDÀR×LÒLÙð ˜Øœ "Ó)Ø˜Ø˜7 5�>ð	�
ð "Ð!r]   zpossibly bad tiling: %s)(r[  rG   rV  r[  rô   r   rÛ  rÜ  r   rb   rÙ  r¥  r–  r`   ra   rm  Úprefer_nd_tilingrÚ  Útile_reductionsrd   rÐ  ÚlevelÚloggingÚWARNINGr_  rQ  r¼  ÚtextwrapÚdedentr   Úcollectionsr   rj   r+  r9  Úmost_commonrª  r–  r“  rã  )rÀ  r¨  rp   r¾  r†  r.  r×  rô   r…  Ú	range_y_xÚrange_rrª  Ú
seen_namesÚcandidate_tilesÚcandidate_tilingr9  r’  rø  rú  Únew_3d_tilings   &&&&&               r[   rê  Ú(SIMDScheduling.select_tiling_with_memory¦  sh  € ð  '¨!Ñ+ˆð ×*Ñ*¨E¨7°_Ð4EÓFˆô $×*Ò*¨=Ö9ˆDÜ˜$Ÿ)™)¤R×%6Ñ%6×7Ô7à—I‘I×0Ñ0Ó2°eÖ;ÜŸ™×3×3Ò3ð #'§/¡/Ó"3�KØ +¨A¥�IØ)¨!�n�GØ ×.Ñ.¨yÓB�FÜ+¨F°D¸$Ó?Ò?ñ :ô  �O‰O×"Ñ"×)Ñ)×B×BÐBß!Ü—M‘M×2×2Ð2à×.Ñ.Ø oóð ÷ ¤V§]¡]×%B×%BÐ%BÄ}ØôH
àôHô ×"Ñ"¤g§o¡oÔ5Ü+×2Ò2°=ÖA�Dä"ŸM™M×9×9Ò9Ü × 5Ñ 5°dÀ?Ó SÓTÐWXÖXä%×*Ñ*Ü$ŸOšOð!$óôñ ñ Bô $ N°D¸$Ó?Ð?ä&0£lˆ
Ü4?×4GÒ4GÓ4IˆÜ#×*Ò*¨=Ö9ˆDØ$'×$9Ñ$9¸$ÀÖ$WÐ Ø#×(Ñ(¨JÔ6ÙØ%×*Ñ*Ò6Ø—N‘NÐ#3×#8Ñ#8Ô9Ø×1Ð5E×5KÑ5KÕKÕ1ó %Xñ :ð ,;×+FÑ+FÔ+Hô7
á+HÑ'Ð ð ×#Ô#Ù+Hð 	ñ 7
ô
  Ô# qÔ(¯\õ!"ôF ˜1œc .Ó1Ö2�Ù 4Ø" 1Õ% ~°aÕ'8ó!�ð !Ô,Ø&3 _°~Õ%E�NÙñ 3ô ˆ~Ó Ô"Ü×ÑÐ8¸.ÔIô �=‰=×)×)Ð)à×"Ñ" =¸ÓIØ õ!ð ð
 ×4Ñ4Ø /°>ó
ð 
ˆ6ö 
ô $ F¨D°$Ó7Ð7ä °°dÓ;Ð;ùóO7
s   Ì	Pc                	ó   € R # r_   rY   rƒ   s   &r[   ÚflushÚSIMDScheduling.flush>  r¢  r]   c               ó   € V ^8„  d   QhRR/# r€   rY   )rZ   s   "r[   r\   r  A  s   € ÷ ñ  ñ r]   c                	ó   € R # rä   rY   rƒ   s   &r[   Úready_to_flushÚSIMDScheduling.ready_to_flushA  rç   r]   c               ó    € V ^8„  d   QhRRRR/# )rU   rQ  r§   r†  rÉ  rY   )rZ   s   "r[   r\   r  D  s    € ÷ <ñ <ð "ð	<ð
 6ñ<r]   c           
     	óª  € \         ;QJ d    R  V 4       F  '       g   K   RM	  RM! R  V 4       4      '       Eg\   \        VR R7      P                  w  pw  rgV P                  WV4      pVfC   \        P
                  P                  P                  P                  '       d   ^ RI	H
p	 V	! V4      p\        W†WtR7      p
V P                  VVVV
P                  4      w  r¼/ pV P                  P                  W­4       V P                  ! V3RV
R	V/VB pV P!                  WŽ4       V P#                  V4      pW/R
&   \        P$                  ! R/ VB ;_uu_ 4        \&        P(                  ! V4      ;_uu_ 4        VP+                  4       pRRR4       RRR4       MZV^ ,          P-                  V4      w  ppp\        P$                  ! R
V4      ;_uu_ 4        V P/                  VVVRVR7      pRRR4       XP1                  \3        \4        P6                  4      R4      pV#   + '       g   i     Lž; i  + '       g   i     LN; i  + '       g   i     La; i)c              3  ó@   "  € T F  qP                  4       x € K  	  R # 5ir_   )r³  )rX  r1  s   & r[   rY  ÚASIMDScheduling.generate_kernel_code_from_nodes.<locals>.<genexpr>K  s   é € Ð2©E q—=‘=—?�?«Eùs   ‚TFc                ó4   € \        V P                  4       4      # r_   rÌ  rÍ  s   &r[   r$  Ú@SIMDScheduling.generate_kernel_code_from_nodes.<locals>.<lambda>L  rü  r]   r&  N)Ú#analyze_memory_coalescing_for_nodesrý  r­  r«  rü  )rë  rQ  rý  rY   )rè  r  r±  rî  r`   ra   r   rb   rm  Útorch._inductor.tiling_utilsr  rJ   ro  r†  r™  re  rÿ  r�  r  r>   r  rž  Úget_prologue_template_epiloguer\  r  rk   r6   r  )r|   r¬   rü  rQ  r†  rx  rp   r¬  r¨  r  r­  rª  r«  rˆ  rs   r“  r  r$  ÚtemplateÚepilogues   &&&&&               r[   rÐ  Ú.SIMDScheduling.generate_kernel_code_from_nodesD  s  € ÷ ‹sÑ2©EÓ2�s�sŠsÑ2©EÓ2×2Ó2Ü!$ UÑ0OÔ!P×!VÑ!VÑˆA‰�Ø ×7Ñ7¸ÀfÓMˆMà!Ò)Ü—O‘O×*Ñ*×1Ñ1×J×JÐJõñ %HÈÓ$NÐ!Ü)Ø fôˆHð %)×$>Ñ$>ØØØØ×*Ñ*ó	%Ñ!ˆFð -/ˆMØ×Ñ×=Ñ=¸hÔVØ×%Ò%Øñà!ðð ,ðð  ñ	ˆFð ×2Ñ2°=ÔIà!×9Ñ9¸-ÓHˆNØ1AÐ-Ñ.ä—’Ñ.˜~×.Ó.Ü×$Ò$ V×,Õ,à!×0Ñ0Ó2�÷ -÷ /Ð.ð
 ,1°­8×+RÑ+RØó,Ñ(ˆH�h ô —’Ð0Ð2B×CÕCØ×0Ñ0ØØØØ&*Ø"/ð 1ó �÷ Dð ×#Ñ#¤C¬×(?Ñ(?Ó$@À)ÓLˆØˆ÷% -×,ú÷ /×.ú÷ D×Cús0   ÅH/Å5H	ÆH/ÇIÈH,È'H/È/H?	ÉI	c                	ó   € \         hr_   ré  )r|   r  r¨  rs   s   &&&&r[   r?  ÚSIMDScheduling.define_kernel‚  r  r]   rY   )é   NFr_   rä   )FF)FNN)Jr”   r•   r–   r—   r˜   rt   r™  r¤  rž  r­  Úcan_fuse_verticalr¹  rî  ró  rø  r  r  rò  rp  rt  rs  rœ  r¨  r£  r™  r©  r¼  rq  rr  rH  rÖ  rç  rí  r�  rÑ  r‘  rþ  rÿ  r)  r;  rF  rO  r\  r`  rf  rz  r‚  r›  rÝ  rí  rô  r   r(  r9  r:  rQ  r[  r?  rM  r“  rÚ  rÏ  rã  r™   rš   r›   r¶  ro  rê  r  r  rÐ  r?  r�   rY   r]   r[   r˜  r˜    sÝ  ‡ ñð
 (€K�Ó'òQòt6ðl !ÐØ"ÐòdòL
8ò
%ò5)÷p"òj)òXm
÷^Fõ8Põd
ð& ô
ó ð
õð2 ô
ó ð
õ4Kð(ð ;?÷(÷B
õ*=ð6 ô"ó ð"÷H@õ"õJ)ðX-ð
 "÷-õ<
ò!-ðFMð  ôMõ^õ,
õ
ð"rð  ðrð %)÷ròhGõ õ(>ð@ ô
ó ð
ð ô&
ó ð&
õPaõF!ðF ô
/ó ð
/÷R òh)ð4 Ø×Ò˜ÓôCó ó ðCðJ ô
ó ð
ð4 ô
ó ð
ð ô/ó ð/ð( ôSó ðSðj ôp<ó ðp<ðd ô
ó ð
ð" ôó ðð ð
 Ÿ™Ÿ™Ø8<ö	ó ð	ð ð
 Ÿ™Ÿ™Ø8<ö9ó ð9ð& ð
 Ÿ™Ÿ™Ø8<öU<ó ðU<ònõ÷<ö|"r]   r˜  c                  ó:   € ] tR tRt$ RtR]R&   R]R&   R]R&   R	tR
# )r¡  i†  z@A candidate pointwise or reduction split, and what it coalesces.r  rb  rŸ  rª  rW   r±  rY   Nr¯  rY   r]   r[   r¡  r¡  †  s   ‡ áJàÓàÓà×r]   r¡  c                  ó<   € ] tR tRt$ RtR]R&   R]R&   R R ltRtR	# )
rÇ  i•  a
  Memory accessed under a tiling, split by whether it coalesces.

These are absolute memory scores (writes weighted x2, as in
CoalesceVarAnalysis), so unlike tiling scores they are comparable across
different kernels - e.g. one fused kernel against two separate ones.
rW   rÅ  rÆ  c               ó   € V ^8„  d   QhRR/# rf  rY   )rZ   s   "r[   r\   ÚMemoryCoalescing.__annotate__¡  s   € ÷ Sñ S˜sñ Sr]   c                óR   € V P                   \        V P                  ,          ,           # )z<Total cost, weighting uncoalesced traffic as more expensive.)rÅ  Ú_UNCOALESCED_MEMORY_COST_WEIGHTrÆ  rƒ   s   &r[   Úweighted_costÚMemoryCoalescing.weighted_cost¡  s   € à�~‰~Ô ?À$×BRÑBRÕ RÕRÐRr]   rY   N)r”   r•   r–   r—   r˜   r¤  r'  r�   rY   r]   r[   rÇ  rÇ  •  s    ‡ ñð ƒNØÓ÷Sñ Sr]   rÇ  c                  ó6   € ] tR tRt$ R]R&   R]R&   R]R&   RtR	# )
r–  i¦  r¶  rª  r¹  r«  zMemoryCoalescing | NoneÚmemoryrY   Nr£  rY   r]   r[   r–  r–  ¦  s   ‡ à!Ó!Ø/Ó/à#×#r]   r–  c                  óJ   € ] tR tRt$ R]R&   R]R&   RtR]R&   ]R	 4       tR
tR# )r>  i®  r¶  rª  rW   r9  NrS  rj   c                óŠ   € \         P                  P                  P                  V RR7      p V ^ 8¬  ;'       d    V ^ ,          ^ 8H  # )z@Somewhat arbitrary heuristic used to boost scores for some sizesi    r¤  )r>   rý   rþ   r  )r0  s   &r[   rB  ÚCandidateTiling.is_good_size´  s=   € ô �G‰G×Ñ×.Ñ.¨q¸4Ð.Ó@ˆØ�B‰w×(Ð(˜A �F a™KÐ(r]   rY   )	r”   r•   r–   r—   r¤  rj   r¨  rB  r�   rY   r]   r[   r>  r>  ®  s)   ‡ à!Ó!ØƒJØ€Dˆ*Óàñ)ó ô)r]   r>  c                  ó.   a € ] tR tRtV 3R ltR tRtV ;t# )r–  i»  c                	ó<   <€ \         SV `  4        Wn        W n        R # r_   )rz   r{   r  r˜  )r|   r  r˜  r}   s   &&&€r[   r{   ÚCantSplit.__init__¼  s   ø€ Ü‰ÑÔØŒ	Ø"Žr]   c                	ó8   € V P                    R V P                   2# )z not divisible by ©r  r˜  rƒ   s   &r[   Ú__str__ÚCantSplit.__str__Á  s   € Ø—)‘)�Ð.¨t¯~©~Ð.>Ð?Ð?r]   r2  )r”   r•   r–   r—   r{   r3  r�   rž   rŸ   s   @r[   r–  r–  »  s   ø† õ#÷
@ð @r]   r–  )r/  r2  r  r0  r1  rw  )›Ú
__future__r   r  r€  rÏ  r9  rÎ  rý  r™  r¯  rÿ  r   Útypingr   r   r   r   r	   Útyping_extensionsr
   r™   r`   Útorch._loggingÚtorch._inductorr   Útorch._inductor.irr   Ú%torch.fx.experimental.symbolic_shapesr   Útorch.fx.immutable_collectionsr   Útorch.utils._ordered_setr   Útorch.utils._sympy.functionsr   r   r   Útorch.utils._sympy.symbolr   r   r   r   Ú_dynamo.utilsr   Ú r   r   r   Úanalyze_preserves_zero_maskr   rÍ  r   r   Údependenciesr   r   r    Úcollections.abcr!   r#   Úops_handlerr$   Úoptimize_indexingr%   r&   Ú runtime.coordinate_descent_tunerr'   Úruntime.hintsr(   r)   Úruntime.runtime_utilsr*   r+   r,   r-   r.   r/   r0   Úutilsr1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   Úvirtualizedr<   r=   r>   Úblock_analysisr?   Úcommonr@   rA   rB   rC   r[  rD   rE   Úsimd_kernel_featuresrF   rG   rH   rI   rJ   r  rK   rL   rM   Útorch._inductor.codegen.tritonrO   r  rP   Ú	getLoggerr”   rv  Ú_loggingÚgetArtifactLoggerrÐ  rÏ  Ú
fusion_logÚdoprintr§  r3  rd   Ú	dataclassrf   rv   rû   rr  rœ  r�  r   r¦  rt   r¬  r®  rÒ  rÚ  rc  rj  r‹  r˜  r¡  r&  rÇ  r–  r>  r³  r–  rY   r]   r[   Ú<module>rV     s‹  ðå "ã Û Û Û Û Û Û Û Û Ý ß @Õ @Ý %ã ã Û Ý #Ý 2Ý GÝ 9Ý /ß LÑ L÷ó õ &ß $Ñ $Ý Fß .ß 6Ñ 6÷ Ý(åå (÷õ =ß :÷ó ÷ EÑ D÷÷ ÷ ñ ÷ -Ñ ,Ý /ß PÓ Pß :÷õ ð +-Ð '÷ ß<Ñ<å;Ý@ð ×Ò˜Ó!€Ø—‘×0Ñ0°¸<ÓH€Ø�~‰~×/Ñ/°¸*ÓE€Ø�^‰^×-Ñ-¨h¸ÓA€
ñ 	‹×Ñ€áÒ7Ó8€÷;ð
 ×Ò÷3+ð 3+ó ð3+ôls;˜/ô s;ôl='˜?ô ='ô@CÐ!4ô CõLñ Ð+°;ÈÔT€ð ×Ò÷ð ó ðô"ˆzô "ôO�˜Õ(¨'°/Õ*Bô Oðd ×Ó˜dÔ#÷!ð !ó $ð!ð ×Ò÷;ð ;ó ð;ð| ×Ó˜dÔ#÷$ð $ó $ð$ð& ×Ó˜dÔ#÷Oð Oó $ðOðd
 ×Ó˜dÔ#÷

ð 

ó $ð

ô:; .ô :;ôz(F˜^ô (FôV@."�^ô @."ðF\ ×Ó˜dÔ#÷ð ó $ðð #%Ð ð ×Ó˜dÔ#÷Sð Só $ðSð  ×Ó˜dÔ#÷$ð $ó $ð$ð ×Ó˜dÔ#÷	)ð 	)ó $ð	)ô@�	ö @r]   