+
    –°³jN ã                   ó^  a € Ru t§0 t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	t	^ RI
t
^ RIHtHt ^ RIHt ^ RIHtHtHtHtHt ^ RIt^ RIt^ RIt^ RIHt ^ RIHtHt ^ RIHt ^ RI H!t!H"t"H#t# ^ R	I$H%t%H&t&H't' ^R
I(H)t) ^RI*H+t+H,t,H-t-H.t.H/t/ ^RI0H1t1 ^RI2H3t3 ^RI4H5t5H6t6H7t7H8t8H9t9H:t:H;t; ^RI<H=t= ^RI>H?t?H@t@HAtAHBtBHCtCHDtDHEtEHFtFHGtGHHtHHItIHJtJ ^RIKHLtLHMtMHNtNHOtO ^RIPHQtQHRtRHStSHTtTHUtUHVtVHWtWHXtXHYtYHZtZH[t[H\t\ ^RI]H^t^H_t_H`t`HataHbtbHctcHdtdHeteHftfHgtgHhthHitiHjtjHktkHltl ]	PÚ                  R8H  tn]PÞ                  R 4       tp]Pâ                  På                  ]sR4      tt]! . RvO4      tuRRRRRRRRRRR R R!R!R"RR#R$R%R$/
tv]! . RwO4      twR&R'R(R)R*R+R,R,R-R.R/R0R1R2R3R4R5R6R7R8/
txR9R:R;R</ty]Pô                  ]Pö                  .t|]Pú                  ]Pü                  ]Pô                  ]Pö                  ]Pþ                  ]EP                   ]EP                  ]EP                  ]EP                  ]EP                  ]EP
                  .t†] ^ k R= t‡R> tˆRxR? R@ llt‰RA tŠRB RC lt‹RD RE ltŒRF RG lt�RH RI ltŽ]EP                   ! RJ RK4      4       t� ! RL RM]94      t‘ ! RN RO4      t’RP t“ ! RQ RR][4      t”]”EP+                  RS4        ! RT RU]”4      t–]–EP+                  RV4       ]–EP/                  4         ! RW RX]–4      t˜ ! RY RZ]Y4      t™ ! R[ R\]™4      tš ! R] R^]š4      t›R_ R` ltœ]EP                   ! Ra Rb4      4       t� ! Rc Rd4      tž ! Re Rf]™4      tŸ ! Rg Rh]™4      t  ! Ri Rj]4      t¡ ! Rk Rl]64      t¢ ! Rm Rn4      t£ ! Ro Rp4      t¤]EP                   ! Rq Rr4      4       t¥]EP                   ! Rs Rt4      4       t¦R# )yé    N)ÚCallableÚSequence)ÚEnum)ÚAnyÚcastÚClassVarÚLiteralÚOptional)Údependencies)Úis_float_dtypeÚis_integer_dtype)Ú
OrderedSet)ÚCeilDivÚFloorDivÚModularIndexing)Úfree_symbol_is_typeÚsymbol_is_typeÚSymT)Úcounters)ÚconfigÚcpp_builderÚcpu_vec_isaÚirÚmetrics)Ú'set_kernel_post_grad_provenance_tracing©ÚLoopBody)ÚBaseSchedulerNodeÚBaseSchedulingÚExternKernelSchedulerNodeÚForeachKernelSchedulerNodeÚFusedSchedulerNodeÚ	SchedulerÚSchedulerNode)Ústride_at_vec_range)Úcache_on_selfÚget_bounds_index_exprÚget_fused_kernel_nameÚhas_free_symbolsÚis_multi_outputs_templateÚis_welford_reductionÚparallel_num_threadsÚPlaceholderÚsympy_index_symbolÚsympy_index_symbol_with_prefixÚsympy_productÚ
sympy_subs)ÚNullKernelHandlerÚopsÚOpsValueÚV)ÚBackendFeatureÚBracesBufferÚCSEÚCSEVariableÚDataTypePropagationÚDeferredLineÚDTYPE_TO_COMPUTATION_DTYPEÚIndentedBufferÚKernelÚ
KernelArgsÚOpOverridesÚOptimizationContext)Ú_get_dtype_from_loopbodiesÚ_get_loop_bodyÚcexprÚcexpr_indexÚcodegen_randÚCppCSEVariableÚDTYPE_TO_CPPÚget_promote_dtypeÚ
INDEX_TYPEÚLocalBufferContextÚmay_unify_binary_op_mask_typeÚpromote_argsÚ(template_fusion_with_epilogues_supportedÚunify_mask_base_typeÚvalue_to_cppÚwin32c                  ó"   € \         '       d   R # R# )z__declspec(dllexport)Ú ©Ú_IS_WINDOWS© ó    Ú\/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/_inductor/codegen/cpp.pyÚget_export_declarationrY   [   s   € ç&1£kÐ"Ð9°rÐ9rW   ÚscheduleÚ+Ú*Ú^ú||ÚminÚmaxÚsumÚprodÚxor_sumÚargminÚargmaxÚanyÚwelford_reduceÚwelfordÚwelford_combineÚTensorz
at::TensorÚintÚlongÚfloatÚdoubleÚboolÚstrzstd::stringÚ
ScalarTypezc10::ScalarTypeÚMemoryFormatzat::MemoryFormatÚLayoutz
at::LayoutÚDevicez
at::DeviceÚnumberz
at::ScalarÚListzstd::vectorr
   zstd::optionalc                 óú  € V\         9   d   \        P                  pV R9   d   ^ # V R8X  d   ^# V R9   dœ   \        V,          pV\        P                  8X  d#   V R9   d   \        \        P
                  ,          p\        V4      '       d   RV R2MRV R2p\        V4      '       d   RV R2MRV R2pV R9   d   TMTpV R9   d   V# RV RV R	2# \        V 4      '       d   R
\        V,           R2# \        V 4      h)rc   rb   ú-std::numeric_limits<ú>::infinity()ústd::numeric_limits<ú>::min()ú>::max()úIndexValue<z>{0, Ú}úWelford<ú>())rc   ra   rf   )r`   re   r_   rd   ©rd   re   )r`   re   )r`   r_   )	ÚDTYPE_LOWP_FPÚtorchÚfloat32rH   ro   rm   r   r+   ÚAssertionError)Úreduction_typeÚdtypeÚcdtypeÚmin_varÚmax_varÚinit_vars   &&    rX   Úreduction_initrŒ   ¤   s$  € Ø”Ôô —‘ˆØÐ2Ô2ÙØ˜ÔÙØÐ;Ô;Ü˜eÕ$ˆØ”E—J‘JÔ >Ð5IÔ#IÜ!¤%§+¡+Õ.ˆFô ˜e×$Ò$ð $ F 8¨=Ñ9à'¨ x¨xÐ8ð 	ô ˜e×$Ò$ð # 6 (¨-Ñ8à'¨ x¨xÐ8ð 	ð
 -Ð0AÔA‘7Àwˆð  Ô/ð ð	
ð ˜v˜h f¨X¨J°bÐ9ð	
ô
 ˜N×+Ò+Øœ, uÕ-Ð.¨cÐ2Ð2Ü
˜Ó
(Ð(rW   c                 óÞ   € \         \        V,          ,          p\        V 4      '       d   R V R2# V R9   d7   V\        P                  8X  d   \         \        P
                  ,          pRV R2# V# )r   Ú>r}   r�   )rH   r<   r+   rƒ   ro   rm   )r†   r‡   Úscalar_types   && rX   Úreduction_acc_typer�   Æ   sd   € ÜÔ9¸%Õ@ÕA€KÜ˜N×+Ò+Ø˜+˜ aÐ(Ð(ØÐ-Ô-Ø”E—J‘JÔÜ&¤u§{¡{Õ3ˆKØ˜[˜M¨Ð+Ð+ØÐrW   c                ó\   € V ^8„  d   QhR\         P                  \        ,          R,          /# )é   ÚindexN)ÚsympyÚExprr9   )Úformats   "rX   Ú__annotate__r—   Ñ   s$   € ÷ 2)ñ 2)ô
 �:‰:œÕ# dÕ*ñ2)rW   c           	      óâ  € V\         P                  8H  pV R 8X  d(   V'       d
   RV RV R2# V'       d   RMRpV RV RV 2# V R8X  d   V RV 2# V R	8X  d   V R
V 2# V R8X  d   V RV 2# V R9   d   V  RV RV R2# V R8X  d   V'       d   RV RV RV R2# RV RV R2# V R8X  d:   \        V\        4      '       d   Vw  r‰p
M\	        W4      w  r‰p
RV RV RV	 RV
 R2	# V R9   d~   \        VR4      '       dO   VP                  \         P                  8X  d0   VP                  '       g   Ve   V  RV RV RV R2# V  RV RV R2# Ve   V  RV RV RV R2# V  RV RV R2# \        V 4      h)ra   úcascade_sum_combine(ú, &Ú)Ú|r[   Ú rb   ú * rc   ú ^ rf   ú || z_propagate_nan(ú, rg   úwelford_combine(ri   ú, {ú})r‡   z	_combine(z, static_cast<float>(ú), ú)))r_   r`   r�   )	rƒ   ro   Ú
isinstanceÚtupleÚreduction_projectÚhasattrr‡   Úis_vecr…   )r†   ÚvarÚ
next_valueÚ
helper_valr“   Ú	src_dtypeÚis_boolÚconjunctionÚmeanÚm2Úweights   &&&&&&     rX   Úreduction_combinerµ   Ñ   s  € ð œ5Ÿ:™:Ñ%€GØ˜ÔßØ)¨*¨°S¸¸ÀAÐFÐFç!(™#¨cˆKØ�U˜!˜K˜=¨¨*¨Ð6Ð6Ø˜ÔØ��c˜*˜Ð&Ð&Ø˜Ô"Ø��c˜*˜Ð&Ð&Ø˜ÔØ��d˜:˜,Ð'Ð'Ø˜Ô'Ø Ð! °°°R¸
°|À1ÐEÐEØÐ)Ô)ßØ% c U¨"¨Z¨L¸¸J¸<ÀqÐIÐIà% c U¨"¨Z¨L¸Ð:Ð:ØÐ*Ô*Ü�j¤%×(Ò(Ø)ÑˆD‘fä0°ÓLÑˆD�fØ! #  d¨4¨&°°2°$°b¸¸ÀÐDÐDØÐ-Ô-ä�J ×(Ò(Ø× Ñ ¤E§J¡JÔ.Ø×%×%Ð%àÒ Ø(Ð)¨°3°%Ð7LÈZÈLÐX[Ð\aÐ[bÐbcÐdÐdð &Ð& i°¨uÐ4IÈ*ÈÐUWÐXðð ÒØ$Ð% Y¨s¨e°2°j°\ÀÀEÀ7È!ÐLÐLà$Ð% Y¨s¨e°2°j°\ÀÐCÐCÜ
˜Ó
(Ð(rW   c                 ó\   € \        V 4      '       d   V R 2V R2V R23# V R9   d   V R2# V# )z.meanz.m2z.weightz.indexr�   )r+   )r†   Úaccs   &&rX   r©   r©     sF   € Ü˜N×+Ò+Ø��eˆ}   S˜k¨c¨U°'¨?Ð:Ð:Ø	Ð/Ô	/Ø��fˆ~ÐØ€JrW   c                óœ   € V ^8„  d   QhR\         R\        P                  R\        R\        P                  R\        P                  R\        /# )r’   ÚcodeÚiter_varÚnew_iter_varÚ
loop_startÚloop_endÚreturn)r=   r”   r•   rp   r7   )r–   s   "rX   r—   r—     sP   € ÷ 2ñ 2Ü
ð2ä�j‰jð2ô ð2ô —
‘
ð	2ô
 �j‰jð2ô ñ2rW   c                óÐ  € \        4       p\        P                  ! 4       ;_uu_ 4       pVP                  R\         RV R\        V4       R2V R\        V4       RV R2,           4       VP                  VP                  4       4       \        V P                  4       F¨  w  rx\        V\        \        34      '       g   \        R4      hR	p	\        V\        4      '       d   VP                  p	VP                  p\         P"                  ! R
V ,           R
,           V V4      p
V	'       d   \        Wš4      p
VP                  V
4       Kª  	  R	R	R	4       V#   + '       g   i     T# ; i)aÂ  
f(iter_var) is transformed to f(new_iter_var) under the inner loop
  \/
for (new_iter_var = loop_start; new_iter_var < loop_end; new_iter_var++) {
    f(new_iter_var)
}
Please be careful while using this function,
as the variable defined in f(iter_var) will be invalid outside the for loop.
For example:
auto tmp0 = in_ptr[x0]; ->
for (new_x0 = start; new_x0 < end; new_x0++){
    auto tmp0 = in_ptr[new_x0];
}
The tmp0 is invalid outside the loop.
zfor (r�   ú = Ú;ú < ú; ú++)ú4expected isinstance( line, ( str, DeferredLine, ), )Nú\b)r7   Ú
contextlibÚ	ExitStackÚ	writelinerJ   rE   Úenter_contextÚindentÚ	enumerateÚ_linesr§   rp   r;   r…   ÚnameÚlineÚreÚsub)r¹   rº   r»   r¼   r½   Útransformed_codeÚstackÚ_rÏ   Údeferred_nameÚnew_lines   &&&&&      rX   Úmove_code_under_inner_loopr×     sG  € ô, $“~ÐÜ	×	Ò	×	Ô	 5Ø×"Ñ"Ø”J�<˜q  ¨c´+¸jÓ2IÐ1JÈ!ÐLØ�˜c¤+¨hÓ"7Ð!8¸¸<¸.ÈÐLõMô	
ð 	×ÑÐ,×3Ñ3Ó5Ô6Ü  §¡Ö-‰GˆAäØäÜ$ð÷ò ô %ØJóð ð !ˆMÜ˜$¤×-Ò-Ø $§	¡	�Ø—y‘y�Ü—v’v˜e¨¨
Õ3°eÕ;À¸~ÐPTÓUˆHßÜ'¨Ó@�Ø×&Ñ& xÖ0ñ) .÷ 
 ð6 Ð÷7 
 Ö	ð6 Ðús   ¨DEÄ* EÅE%	c                ó”   € V ^8„  d   QhR\         \        ,          R\         R\         R\        P                  R\         \        ,          /# )r’   Úacc_varÚacc_typer†   r‡   Úlen)rp   r9   rƒ   r‡   rk   )r–   s   "rX   r—   r—   C  sG   € ÷ ñ Ü”;Õðäðô ðô �;‰;ð	ô
 
Œs�ñrW   c                óî   € \        4       p\        P                  ! 4       '       d   RV  RV RV R2M
V RV  RV R2pVP                  V 4       VP	                  RV R	2R
RV  RV! W#4       R2R.4       V# )a€  
MSVC don't support dynamic array(VLA). So we use std::unique_ptr here.
Ref: https://stackoverflow.com/questions/56555406/creating-dynamic-sized-array-using-msvc-c-compiler
MSVC is the only one compiler without VLA. support. Since MSVC can't get good performance here.
We just use unique_ptr make it works on MSVC.
For other compilers, we continue to use VLA to get best performance.
úauto z_arr = std::make_unique<z[]>(ú);r�   ú_arr[ú];úfor (int i = 0; i < ú; i++)Ú{ú    z
_arr[i] = rÁ   r~   )r=   r   Ú
is_msvc_clrÉ   Ú
writelines)rÙ   rÚ   r†   r‡   rÛ   Úinit_fnÚcode_bufferÚacc_decls   &&&&&&  rX   Úreduction_prefix_arrayrê   C  sª   € ô !Ó"€Kô ×!Ò!×#Ò#ð �ˆyÐ0°°
¸$¸s¸eÀ2ÑFàˆz˜˜7˜) 5¨¨¨RÐ0ð ð
 ×Ñ˜X˜JÔ(Ø×Ñà" 3 % vÐ.ØØ�7�)˜:¡g¨nÓ&DÐ%EÀQÐGØð		
ôð ÐrW   c                ó<   € V ^8„  d   QhR\         R\        R\        /# )r’   ÚbufferrÎ   Únew_name)r=   rp   )r–   s   "rX   r—   r—   d  s&   € ÷ Vñ Vœ^ð V´3ð VÄ#ñ VrW   c                 ó˜  € \        V P                  4       F°  w  r4\        V\        \        34      '       g   \        R 4      h\        V\        4      '       d:   \        P                  ! RV ,           R,           V VP                  4      Vn        K{  \        P                  ! RV ,           R,           V V4      V P                  V&   K²  	  R# )rÅ   rÆ   N)	rÌ   rÍ   r§   rp   r;   r…   rÐ   rÑ   rÏ   )rì   rÎ   rí   ÚirÏ   s   &&&  rX   Úreplace_acc_namerð   d  s�   € Ü˜VŸ]™]Ö+‰ˆäØäÜ ð÷ò ô !Ð!WÓXÐXÜ�dœL×)Ò)ÜŸš˜u¨$¨Õ0°5Õ8¸X¸JÈÏÉÓSˆDŽIä!Ÿvšv e°°Õ&7¸%Õ&?ÀHÀ:ÐPTÓUˆF�M‰M˜!Óó ,rW   c                ó$   € V ^8„  d   QhR\         /# )r’   rì   )r=   )r–   s   "rX   r—   r—   v  s   € ÷ /ñ /¬ñ /rW   c           
     óØ  € Rp\        V P                  4       FÎ  w  r#\        V\        \        34      '       g   \        R4      h\        V\        4      '       d   VP                  MTp\        P                  ! W4      pV'       g   Kp  VP                  4       w  rg\        P                  ! W RV RV R2V4      p\        V\        4      '       d	   Wƒn        KÀ  W€P                  V&   KÐ  	  R# )zL
Replaces `acc = cascade_sum_combine(value, ...)` with `acc = acc + value;`
z/(.*?)\s*=\s*cascade_sum_combine\(([^,]+),.*?\);rÅ   rÀ   ú + rÁ   N)rÌ   rÍ   r§   rp   r;   r…   rÏ   rÐ   ÚsearchÚgroupsrÑ   )	rì   Úpatternrï   rÏ   ÚcontentÚmatchr·   ÚvalueÚnew_contents	   &        rX   Úreplace_cascade_sum_with_addrû   v  sÀ   € ð
 A€GÜ˜VŸ]™]Ö+‰ˆäØäÜ ð÷ò ô !Ð!WÓXÐXÜ)¨$´×=Ò=�$—)’)À4ˆÜ—	’	˜'Ó+ˆß‰5ØŸ™›‰JˆCÜŸ&š& ¨E°°S°E¸¸U¸GÀ1Ð*EÀwÓOˆKÜ˜$¤×-Ò-Ø'–	à#.—‘˜aÓ ó' ,rW   c                   ó0   a € ] tR tRt o RtV 3R ltRtV tR# )ÚParallelDepthi’  zo
A class representing parallel depth.
Includes the starting depth of parallelism and the depth of parallelism.
c                ó2   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   # )r’   Úparallel_depthÚstart_depth©rk   )r–   Ú__classdict__s   "€rX   r—   ÚParallelDepth.__annotate__’  s   ø‡ ‚ ñ Ññ ñ Ñò rW   rV   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r  s   @rX   rý   rý   ’  s   ø‡ € ñ÷ ƒ rW   rý   c                   ól   a a€ ] tR tRt o]V3R lR l4       tV3R lV 3R lltR tR tR t	R	t
VtV ;t# )
ÚOuterLoopFusedSchedulerNodei�  c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r’   Únode1Únode2©r   )r–   r  s   "€rX   r—   Ú(OuterLoopFusedSchedulerNode.__annotate__Ÿ  s!   ø€ ÷ )Qñ )QÙ%ð)QÙ.?ñ)QrW   c                ó|  € VP                   VP                   Jd   \        R 4      h\        ;QJ d    R W3 4       F  '       d   K   RM	  RM! R W3 4       4      '       g   \        R4      h\        ;QJ d    R W3 4       F  '       g   K   RM	  RM! R W3 4       4      '       d�   T ! VP                   \	        V4      \
        J d   \        VP                  4       4      MV.\	        V4      \
        J d%   \        VP                  4       4      ,           V4      # V.,           V4      # V ! VP                   W.V4      # )z+expected node1.scheduler is node2.schedulerc              3   ó\   "  € T F"  p\        V4      \        \        \        39   x € K$  	  R # 5i©N)Útyper  r$   r"   ©Ú.0Únodes   & rX   Ú	<genexpr>Ú3OuterLoopFusedSchedulerNode.fuse.<locals>.<genexpr>¥  s2   é € ð ñ +�Dô �T“
ä/Ü!Ü&ðöó +ùó   ‚*,FTzZexpected all nodes to be OuterLoopFusedSchedulerNode, SchedulerNode, or FusedSchedulerNodec              3   óD   "  € T F  p\        V4      \        J x € K  	  R # 5ir  ©r  r  r  s   & rX   r  r  ³  ó   é € ÐTÁ^¸TŒt�D‹zÔ8Õ8Ã^ùó   ‚ )Ú	schedulerr…   Úallrf   r  r  ÚlistÚget_outer_nodes)Úclsr  r  Úouter_loop_fusion_depths   &&&&rX   ÚfuseÚ OuterLoopFusedSchedulerNode.fusež  s4  € ð �?‰? %§/¡/Ó1Ü Ð!NÓOÐOç‹Cñ ð #™Nó�C�CŠCñ ð #™Nó÷ ò ô !ð7óð ÷ ‹3ÑTÀeÁ^ÓT�3�3Š3ÑTÀeÁ^ÓT×TÒTÙØ—‘ô ˜E“{Ô&AÓAô ˜×.Ñ.Ó0Ô1ð ðô ˜E“{Ô&AÓAô ˜×.Ñ.Ó0Ó1õð (ó#ð ð ðõð (ó#ð ñ( �u—‘¨¨Ð8OÓPÐPrW   c                óD   <€ V ^8„  d   QhRRRS[ S[S[,          ,          /# )r’   r"  r#   Úouter_fused_nodes)r$  r"   r$   )r–   r  s   "€rX   r—   r  Ê  s*   ø€ ÷ 4ñ 4àð4ñ  Ñ 2±]Õ BÕCñ4rW   c                ó  <€ VV n         W0n        . pV P                    FR  p\        V\        \        34      '       g   \        R 4      hVP                  \        VP                  4       4      4       KT  	  \        SV `)  W4       R# )z?expected isinstance(_node, (SchedulerNode, FusedSchedulerNode))N)r+  r'  r§   r$   r"   r…   Úextendr$  Ú	get_nodesÚsuperÚ__init__)Úselfr"  r+  r'  Úflatten_snodesÚ_nodeÚ	__class__s   &&&&  €rX   r0  Ú$OuterLoopFusedSchedulerNode.__init__Ê  s{   ø€ ð ð 	Ôð (?Ô$ØˆØ×+Ô+ˆEÜ˜e¤mÔ5GÐ%H×IÒIÜ$ØUóð ð ×!Ñ!¤$ u§¡Ó'8Ó"9Ö:ñ ,ô 	‰Ñ˜Ö3rW   c                ó   € V P                   # r  )r+  ©r1  s   &rX   r%  Ú+OuterLoopFusedSchedulerNode.get_outer_nodesÝ  s   € Ø×%Ñ%Ð%rW   c                óx  a€ R  V3R llo\        \        V4      ^,
          4       FB  pW,          P                  pW^,           ,          P                  pS! VVV^ 4      '       d   KA   R# 	  V FÈ  p\        P                  ! \
        P                  VP                  RV 4      p\        VP                  4      V8”  g   KQ  \        V\        P                  4      '       g   Ks  \        VP                  V,          \        P                  4      '       g   K¦  VR,          VP                  V,          8  g   KÇ   R# 	  R# )c          
      óT   € V ^8„  d   QhR\         R\         R\        R\        R\        /# )r’   Úleft_loop_nestÚright_loop_nestÚloop_fusion_depthÚcurrent_checking_depthr¾   ©ÚLoopNestrk   ro   )r–   s   "rX   r—   ÚTOuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr.<locals>.__annotate__é  s:   € ÷ 1	ñ 1	Ü$ð1	ä%ð1	ô  #ð1	ô %(ð	1	ô
 ñ1	rW   c                 ól  <aa€ V P                   '       g   \        R 4      hVP                   '       g   \        R4      hV P                   V,          oVP                   V,          o. ROp\        ;QJ d     VV3R lV 4       F  '       d   K   RM	  RM! VV3R lV 4       4      '       g   R# V^8  d   \        R4      hV^,
          ;p^ 8”  dg   V^,           pV\        V P                   4      8¼  d   \        R4      hV\        VP                   4      8¼  d   \        R4      hS! V VVV4      '       g   R# R# )	zexpected left_loop_nest.loopszexpected right_loop_nest.loopsc              3   óV   <"  € T F  p\        SV4      \        SV4      8H  x € K   	  R # 5ir  )Úgetattr)r  Úattr_compareÚleft_loop_levelÚright_loop_levels   & €€rX   r  ÚaOuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr.<locals>._inner.<locals>.<genexpr>ý  s2   øé € ð ñ )F˜ô ˜O¨\Ó:ÜÐ/°Ó>ö?ã(Eùs   ƒ&)FTzexpected loop_fusion_depth >= 1z;expected current_checking_depth < len(left_loop_nest.loops)z<expected current_checking_depth < len(right_loop_nest.loops))r¬   ÚsizeÚoffsetÚsteps)Úloopsr…   r#  rÛ   )r;  r<  r=  r>  Úouter_loops_attr_compare_listrF  rG  Ú_inners   &&&& @@€rX   rN  ÚNOuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr.<locals>._inneré  s9  ú€ ð "×'×'Ð'Ü$Ð%DÓEÐEØ"×(×(Ð(Ü$Ð%EÓFÐFØ,×2Ñ2Ð3IÕJˆOØ.×4Ñ4Ð5KÕLÐò-Ð)÷ “õ ñ )Fó——’õ ñ )Fó÷ ò ñ à  1Ô$Ü$Ð%FÓGÐGØ%6¸Õ%:Ð:Ð!¸aÔ?à)?À!Õ)CÐ&Ø)¬S°×1EÑ1EÓ-FÔFÜ(ØUóð ð *¬S°×1FÑ1FÓ-GÔGÜ(ØVóð ñ Ø"Ø#Ø%Ø*÷	ò ñ !árW   FNé,  T)ÚrangerÛ   Ú	loop_nestÚ	functoolsÚreduceÚoperatorÚmulÚrangesr§   r”   ÚInteger)	r1  Úcpp_kernel_proxy_listr'  Úidxr;  r<  Úcpp_kernel_proxyÚouter_rangesrN  s	   &&&     @rX   Ú"check_outer_fusion_loop_level_attrÚ>OuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attrà  s  ø€ ÷1	ð 1	ôf œÐ2Ó3°aÕ7Ö8ˆCØ2Õ7×AÑAˆNØ3¸!µGÕ<×FÑFˆOÙØØØ'Ø÷	ô ò ñ 9ó !6ÐÜ$×+Ò+Ü—‘Ø ×'Ñ'Ð(@Ð)@ÐAóˆLô Ð$×+Ñ+Ó,Ð/FÖFÜ˜|¬U¯]©]×;Ô;ÜØ$×+Ñ+Ð,CÕDÜ—M‘M÷ô ð ! 3Õ&Ø"×)Ñ)Ð*AÕBöCò ñ) !6ñ, rW   c                óV  € V^ ,          P                   p\        V4      pV Uu. uF(  pVP                  P                  V P                  4      NK*  	  upVn        V^ ,          pW5P                  n        VP                  P                  RV P                   VP                  n        V# u upi ©r   N)Úkernel_groupÚOuterLoopFusedKernelrR  Úfrom_loop_levelr'  ÚinnerÚkernelrL  )r1  rY  ra  Úouter_loop_fused_kernelÚproxyÚouter_fused_proxys   &&    rX   Úmerge_outer_fusion_kernelsÚ6OuterLoopFusedSchedulerNode.merge_outer_fusion_kernels?  s¨   € ð -¨QÕ/×<Ñ<ˆÜ"6°|Ó"DÐñ /ó)
á.�ð �O‰O×+Ñ+¨D×,HÑ,HÖIÙ.ñ)
ÐÔ%ð 2°!Õ4ÐØ-D×#Ñ#Ô*Ø,=×,GÑ,G×,MÑ,MØ*ˆd×*Ñ*ð-
Ð×#Ñ#Ô)ð !Ð ùò)
s   £.B&)r+  r'  )r  r  r  r  Úclassmethodr(  r0  r%  r]  ri  r
  r  Ú__classcell__©r4  r  s   @@rX   r  r  �  s:   ù‡ € Ø÷)Qó ð)Q÷V4ó 4ò&&ò]÷~!ò !rW   r  c                   óN   a € ] tR tRt o R
V 3R lR lltR tR tR tR tRt	V t
R	# )ÚRecordOptimizationContextiQ  c                ó    <€ V ^8„  d   QhRS[ /# )r’   Ú	func_name©rp   )r–   r  s   "€rX   r—   Ú&RecordOptimizationContext.__annotate__R  s   ø€ ÷ 8ñ 8¡#ñ 8rW   c                ó.   € Wn         R V n        R V n        R # r  )rq  Úcurrent_nodeÚopt_ctx)r1  rq  s   &&rX   r0  Ú"RecordOptimizationContext.__init__R  s   € Ø"ŒØ26ˆÔØ37ˆŽrW   c                óH  € \         P                  '       g   \        R 4      h\         P                  P                  '       g   \        R4      h\         P                  P                  V n        V P                  f   \        R4      h\        P
                  V P                  P                  9   d2   V P                  P                  \        P
                  ,          V n        M\	        4       V n        V P                  f   \        R4      hV P                  V P                  n	        V # )zexpected V.interpreterz#expected V.interpreter.current_nodez&expected self.current_node is not Nonez!expected self.opt_ctx is not None)
r5   Úinterpreterr…   ru  rA   ÚkeyÚmetarv  rq  Úops_namer7  s   &rX   Ú	__enter__Ú#RecordOptimizationContext.__enter__W  sË   € Ü�}�}ˆ}Ü Ð!9Ó:Ð:Ü�}‰}×)×)Ð)Ü Ð!FÓGÐGäŸM™M×6Ñ6ˆÔØ×ÑÒ$Ü Ð!IÓJÐJÜ×"Ñ" d×&7Ñ&7×&<Ñ&<Ô<Ø×,Ñ,×1Ñ1Ô2E×2IÑ2IÕJˆD�Lä.Ó0ˆDŒLØ�<‰<ÒÜ Ð!DÓEÐEØ $§¡ˆ�‰ÔØˆrW   c                óÜ   € V P                   '       g   \        R 4      hV P                  '       g   \        R4      hV P                  V P                   P                  \        P
                  &   R# )úexpected self.current_nodezexpected self.opt_ctxN)ru  r…   rv  r{  rA   rz  ©r1  Úexc_typeÚexc_valÚexc_tbs   &&&&rX   Ú__exit__Ú"RecordOptimizationContext.__exit__i  sP   € Ø× × Ð Ü Ð!=Ó>Ð>Ø�|�|ˆ|Ü Ð!8Ó9Ð9Ø:>¿,¹,ˆ×Ñ×ÑÔ2×6Ñ6Ó7rW   c                ó   € V P                   # r  )rv  r7  s   &rX   Úget_opt_ctxÚ%RecordOptimizationContext.get_opt_ctxp  s   € Ø�|‰|ÐrW   c                óT   € V P                   '       g   \        R 4      hV P                   # )r€  )ru  r…   r7  s   &rX   Úget_fx_nodeÚ%RecordOptimizationContext.get_fx_nodes  s&   € Ø× × Ð Ü Ð!=Ó>Ð>Ø× Ñ Ð rW   )ru  rq  rv  N)rS   )r  r  r  r  r0  r}  r…  rˆ  r‹  r
  r  r  s   @rX   ro  ro  Q  s)   ø‡ € ÷8ò 8ò
ò$Gò÷!ð !rW   ro  c                 óp  € W3p\         ;QJ d    R  V 4       F  '       g   K   RM	  RM! R  V 4       4      '       d   \        R4      h\        V4      pVe   \        V,          MRV R2pV\        P
                  \        P                  39   d   RV R2pV RV RV R	V  R
V RV R2# V RV R
V  R
V R2# )c              3   ój   "  € T F)  p\        V\        4      ;'       d    VP                  x € K+  	  R # 5ir  )r§   rG   r«   )r  Úargs   & rX   r  Ú!arith_promoted.<locals>.<genexpr>{  s&   é € Ð
JÁT¸cŒ:�cœ>Ó*×9Ð9¨s¯z©zÔ9ÃTùs   ‚3ž3TFz*Promotion of vector types is not supportedú	decltype(r›   z!static_cast<std::make_unsigned_t<z>>Ú(ú) r�   r¦   )rf   r…   rI   rH   rƒ   Úint32Úint64)ÚopÚaÚbÚargsr‡   Úcpp_typer   s   &&&    rX   Úarith_promotedr›  y  sÇ   € Øˆ6€Dß
ƒsÑ
JÁTÓ
J‡s‡s‚sÑ
JÁTÓ
J×JÒJÜÐIÓJÐJä˜dÓ#€EØ&+Ò&7Œ|˜EÖ"¸yÈÈÈ1Ð=M€HØ”—‘œeŸk™kÐ*Ô*à2°8°*¸BÐ?ˆØ�˜1˜T˜F ! A 3 b¨¨¨A¨d¨V°1°Q°C°rÐ:Ð:ØˆZ�q˜˜˜1˜R˜D  !  AÐ&Ð&rW   c                   ó.  a € ] tR tRt o Rt]R 4       t]R 4       t]R 4       t]RWR l4       t	]RWR l4       t
]R	 4       t]R
 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t ]R 4       t!]R  4       t"]R! 4       t#]R" 4       t$]R# 4       t%]R$ 4       t&]R% 4       t']R& 4       t(]R' 4       t)]R( 4       t*]R) 4       t+]R* 4       t,]R+ 4       t-]R, 4       t.]R- 4       t/]R. 4       t0]R/ 4       t1]R0 4       t2]R1 4       t3]R2 4       t4]R3 4       t5]R4 4       t6]R5 4       t7]R6 4       t8]R7 4       t9]R8 4       t:]R9 4       t;]R: 4       t<]R; 4       t=]R< 4       t>]R= 4       t?]R> 4       t@]R? 4       tA]R@ 4       tB]RA 4       tC]RB 4       tD]RC 4       tE]RD 4       tF]RE 4       tG]RF 4       tH]RG 4       tI]RH 4       tJ]RI 4       tK]V 3RJ lRK l4       tL]V 3RL lRM l4       tM]V 3RN lRO l4       tN]V 3RP lRQ l4       tO]RR 4       tP]RS 4       tQV 3RT lRU ltRRVtSV tTR# )XÚCppOverridesi‡  zMap element-wise ops to C++c                ó   € \        R W4      # )r[   ©r›  ©r—  r˜  s   &&rX   ÚaddÚCppOverrides.addŠ  ó   € ä˜c 1Ó(Ð(rW   c                ó   € \        R W4      # )Ú-rŸ  r   s   &&rX   rÑ   ÚCppOverrides.subŽ  r£  rW   c                ó   € \        R W4      # )r\   rŸ  r   s   &&rX   rV  ÚCppOverrides.mul’  r£  rW   Nc                ó  € \        V \        4      '       g   \        R 4      hVf   V P                  p\        P
                  P                  WV4      p\        P
                  P                  P                  \        P
                  P                  V4      pVP                  RW3RV/4       V'       dX   V\        9   dM   V\        P                  8X  d8   \        P                  '       g"    \        P
                  P!                  WWQ4       V# )ú&expected isinstance(x, CppCSEVariable)Úto_dtyper¯   ©r§   rG   r…   r‡   r5   re  Úget_to_dtype_exprÚcseÚgenerateÚcomputeÚupdate_on_argsr‚   rƒ   rm   r   Úemulate_precision_castsÚcache_dtype_convert©Úxr‡   r¯   Úuse_compute_typesÚexprÚcsevars   &&&&  rX   r«  ÚCppOverrides.to_dtype–  sÁ   € ä˜!œ^×,Ò,Ü Ð!IÓJÐJØÒØŸ™ˆIÜ�x‰x×)Ñ)¨!°IÓ>ˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜j¨1¨*°{ÀIÐ6NÔOçØœÔ&ØœUŸ[™[Ô(Ü×2×2Ð2ð"ôF �H‰H×(Ñ(¨°vÔEØˆrW   c                ó  € \        V \        4      '       g   \        R 4      hVf   V P                  p\        P
                  P                  WVRR7      p\        P
                  P                  P                  \        P
                  P                  V4      pVP                  RW3RV/4       V\        9   dL   V\        P                  8X  d7   \        P                  '       g!   \        P
                  P!                  WWQ4       V# )rª  T©ÚroundingÚround_to_intr¯   r¬  r´  s   &&&&  rX   r½  ÚCppOverrides.round_to_intË  s½   € ô ˜!œ^×,Ò,Ü Ð!IÓJÐJØÒØŸ™ˆIÜ�x‰x×)Ñ)¨!°IÈÐ)ÓMˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜n¨q¨j¸;È	Ð:RÔSà”]Ô"ØœUŸ[™[Ô(Ü×2×2Ð2ä�H‰H×(Ñ(¨°vÔEØˆrW   c                ój   € V\         9  d   \        V R \         R24      hR\         V,           RV  R2# )z missing from z.DTYPE_TO_CPPzc10::bit_cast<ú>(r›   )rH   r…   r  )rµ  r‡   r¯   s   &&&rX   Úto_dtype_bitcastÚCppOverrides.to_dtype_bitcastÝ  s=   € àœÔ$Ü  E 7¨.¼¸
À-Ð!PÓQÐQØ¤¨UÕ 3Ð4°B°q°c¸Ð;Ð;rW   c                óÚ   € \        V \        4      '       dQ   V P                  \        P                  \        P
                  \        P                  \        P                  39   d   V  # R V  R2# )z	std::abs(r›   ©r§   rG   r‡   rƒ   Úuint8Úuint16Úuint32Úuint64©rµ  s   &rX   ÚabsÚCppOverrides.absã  sU   € ô �aœ×(Ò(¨Q¯W©WÜ�K‰KÜ�L‰LÜ�L‰LÜ�L‰Lð	9
ô .
ð �SˆMØ˜1˜#˜QÐÐrW   c                ó   € R V  R2# )z	std::sin(r›   rV   rÉ  s   &rX   ÚsinÚCppOverrides.sinñ  ó   € ð ˜1˜#˜QÐÐrW   c                ó   € R V  R2# )z	std::cos(r›   rV   rÉ  s   &rX   ÚcosÚCppOverrides.cosö  rÏ  rW   c                ó   € R V  RV  R2# )r‘  z)(-r›   rV   rÉ  s   &rX   ÚnegÚCppOverrides.negû  ó   € à˜1˜#˜S   1Ð%Ð%rW   c                ó   € R V  R2# )z	std::exp(r›   rV   rÉ  s   &rX   ÚexpÚCppOverrides.expÿ  s   € ð ˜1˜#˜QÐÐrW   c                ó   € R V  R2# )z
std::exp2(r›   rV   rÉ  s   &rX   Úexp2ÚCppOverrides.exp2  ó   € à˜A˜3˜aÐ Ð rW   c                ó   € R V  R2# )zstd::expm1(r›   rV   rÉ  s   &rX   Úexpm1ÚCppOverrides.expm1	  ó   € à˜Q˜C˜qÐ!Ð!rW   c                ó   € R V  R2# )z	std::erf(r›   rV   rÉ  s   &rX   ÚerfÚCppOverrides.erf  rÏ  rW   c                ó   € R V  R2# )z
std::erfc(r›   rV   rÉ  s   &rX   ÚerfcÚCppOverrides.erfc  rÝ  rW   c                ó   € R V  R2# )zcalc_erfinv(r›   rV   rÉ  s   &rX   ÚerfinvÚCppOverrides.erfinv  ó   € ð ˜a˜S Ð"Ð"rW   c                ó   € R V  R2# )z
std::sqrt(r›   rV   rÉ  s   &rX   ÚsqrtÚCppOverrides.sqrt  ó   € ð ˜A˜3˜aÐ Ð rW   c                ó   € R V  R2# )z1 / std::sqrt(r›   rV   rÉ  s   &rX   ÚrsqrtÚCppOverrides.rsqrt   s   € ð   ˜s !Ð$Ð$rW   c                ó†   € \         P                  P                  pVR 8X  d	   V  RV  R2# Vf   RV  R2# \        RV: 24      h)Úaccuracyú + decltype(ú)(1)zstd::log1p(r›   ú8unrecognized config cpp.inject_log1p_bug_TESTING_ONLY = ©r   ÚcppÚinject_log1p_bug_TESTING_ONLYr…   ©rµ  Úbugs   & rX   Úlog1pÚCppOverrides.log1p%  sW   € ä�j‰j×6Ñ6ˆØ�*ÔØ�S˜ Q C tÐ,Ð,ØŠ[Ø    1Ð%Ð%ä ØJÈ3É'ÐRóð rW   c                ó   € R V  R2# )z	std::tan(r›   rV   rÉ  s   &rX   ÚtanÚCppOverrides.tan1  rÏ  rW   c                ó   € R V  R2# )z
std::tanh(r›   rV   rÉ  s   &rX   ÚtanhÚCppOverrides.tanh6  rï  rW   c                ó2   € \         '       d   RV  R2# RV  R2# )z�
On windows std::signbit only support float type.
Ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/signbit?view=msvc-170
z std::signbit(static_cast<float>(r¦   zstd::signbit(r›   rT   rÉ  s   &rX   ÚsignbitÚCppOverrides.signbit;  s/   € ÷ ‹{ð /¨q¨c°Ð4ð	
ð !   1Ð%ð	
rW   c                ó   € R V  RV R2# )z	std::pow(r¡   r›   rV   r   s   &&rX   ÚpowÚCppOverrides.powH  s   € à˜1˜#˜R ˜s !Ð$Ð$rW   c                ó   € R V  R2# )z	std::log(r›   rV   rÉ  s   &rX   ÚlogÚCppOverrides.logL  rÏ  rW   c                ó   € R V  R2# )zstd::nearbyint(r›   rV   rÉ  s   &rX   ÚroundÚCppOverrides.roundQ  s   € ð !   1Ð%Ð%rW   c                ó   € R V  R2# )zstd::floor(r›   rV   rÉ  s   &rX   ÚfloorÚCppOverrides.floorV  ó   € ð ˜Q˜C˜qÐ!Ð!rW   c                ó   € R V  RV R2# )úfloor_divide_integral(r¡   r›   rV   r   s   &&rX   ÚfloordivÚCppOverrides.floordiv[  s   € ð (¨ s¨"¨Q¨C¨qÐ1Ð1rW   c                ó   € R V  R2# )z
std::ceil(r›   rV   rÉ  s   &rX   ÚceilÚCppOverrides.ceil`  rï  rW   c                ó   € R V  R2# )zstd::trunc(r›   rV   rÉ  s   &rX   ÚtruncÚCppOverrides.trunce  r  rW   c                ó   € V  R V 2# ©ú / rV   r   s   &&rX   ÚtruncdivÚCppOverrides.truncdivj  s   € ð ��C˜�sˆ|ÐrW   c                ó   € R V  RV R2# )z
std::fmod(r¡   r›   rV   r   s   &&rX   ÚfmodÚCppOverrides.fmodp  s   € ð ˜A˜3˜b   1Ð%Ð%rW   c                ó   € R V  R2# )zstd::isinf(r›   rV   rÉ  s   &rX   ÚisinfÚCppOverrides.isinfu  r  rW   c                ó   € R V  R2# )zstd::isnan(r›   rV   rÉ  s   &rX   ÚisnanÚCppOverrides.isnanz  r  rW   c                ó   € R V  R2# )zstd::lgamma(r›   rV   rÉ  s   &rX   ÚlgammaÚCppOverrides.lgamma  rë  rW   c                ó   € R V  R2# )z
std::acos(r›   rV   rÉ  s   &rX   ÚacosÚCppOverrides.acos„  rï  rW   c                ó   € R V  R2# )zstd::acosh(r›   rV   rÉ  s   &rX   ÚacoshÚCppOverrides.acosh‰  r  rW   c                ó   € R V  R2# )z
std::cosh(r›   rV   rÉ  s   &rX   ÚcoshÚCppOverrides.coshŽ  rï  rW   c                ó   € R V  R2# )z
std::sinh(r›   rV   rÉ  s   &rX   ÚsinhÚCppOverrides.sinh“  rï  rW   c                ó   € R V  R2# )z
std::asin(r›   rV   rÉ  s   &rX   ÚasinÚCppOverrides.asin˜  rï  rW   c                ó   € R V  R2# )zstd::asinh(r›   rV   rÉ  s   &rX   ÚasinhÚCppOverrides.asinh�  r  rW   c                ó   € R V  RV R2# )zstd::atan2(r¡   r›   rV   ©rµ  Úys   &&rX   Úatan2ÚCppOverrides.atan2¢  ó   € ð ˜Q˜C˜r !  AÐ&Ð&rW   c                ó   € R V  R2# )z
std::atan(r›   rV   rÉ  s   &rX   ÚatanÚCppOverrides.atan§  rï  rW   c                ó   € R V  R2# )zstd::atanh(r›   rV   rÉ  s   &rX   ÚatanhÚCppOverrides.atanh¬  r  rW   c                ó   € R V  RV R2# )zstd::copysign(r¡   r›   rV   rC  s   &&rX   ÚcopysignÚCppOverrides.copysign±  s   € ð   ˜s " Q C qÐ)Ð)rW   c           	     ó6  € R V  R2R V  R23p\         ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       d-   \        ;QJ d    . R V 4       F  NK  	  5# ! R V 4       4      # \        4       p\        P                  P
                  P                  \        P                  V P                  R7      p\        P                  P
                  P                  V P                  V P                  R7      pVP                  RV R	24       VP                  R
V RV  RV R24       \        P                  P                  P                  V4       WC3p\        W4       F.  w  rg\        P                  P
                  P                  Wg4       K0  	  WC3# )úfrexp(ú)[0]ú)[1]c              3   óx   "  € T F0  p\         P                  P                  P                  V4      R Jx € K2  	  R # 5ir  ©r5   re  r®  Útry_get©r  Ú	cache_keys   & rX   r  Ú%CppOverrides.frexp.<locals>.<genexpr>º  ó(   é € ÐWÉJ¸yŒq�x‰x�|‰|×#Ñ# IÓ.°dÕ:ËJùó   ‚8:FTc              3   ót   "  € T F.  p\         P                  P                  P                  V4      x € K0  	  R # 5ir  rV  rX  s   & rX   r  rZ  »  ó&   é € ÐUÉ*¸YœŸ™Ÿ™×-Ñ-¨i×8Ð8Ë*ùó   ‚68)r‡   Úshapezint32_t rÁ   rÝ   z = std::frexp(rš   rÞ   )r#  r¨   r7   r5   re  r®  Únewvarrƒ   r”  r`  r‡   rÉ   r°  ÚspliceÚzipÚput)rµ  Ú
cache_keysr¹   ÚexponentÚmantissaÚcse_varsrY  Úcse_vars   &       rX   ÚfrexpÚCppOverrides.frexp¶  s;  € ð ˜a˜S Ð%¨°¨s°$Ð'7Ð7ˆ
ß‹3ÑWÉJÓW�3�3Š3ÑWÉJÓW×WÒWß”5ÑUÉ*ÓU—5ÐU�5ÑUÉ*ÓUÓUÐUä‹~ˆÜ—8‘8—<‘<×&Ñ&¬U¯[©[ÀÇÁÐ&ÓHˆÜ—8‘8—<‘<×&Ñ&¨Q¯W©W¸A¿G¹GÐ&ÓDˆØ�‰˜ ( ¨1Ð-Ô.Ø�‰˜˜x˜j¨°q°c¸¸X¸JÀbÐIÔJÜ	�‰×Ñ×Ñ Ô%ØÐ'ˆÜ"% jÖ";ÑˆIÜ�H‰H�L‰L×Ñ˜YÖ0ñ #<àÐ!Ð!rW   c                ó   € R V  RV R2# )zstd::hypot(r¡   r›   rV   rC  s   &&rX   ÚhypotÚCppOverrides.hypotÈ  rG  rW   c                ó   € R V  R2# )zstd::log10(r›   rV   rÉ  s   &rX   Úlog10ÚCppOverrides.log10Í  rá  rW   c                ó   € R V  R2# )z
std::log2(r›   rV   rÉ  s   &rX   Úlog2ÚCppOverrides.log2Ñ  rÝ  rW   c                ó   € R V  RV R2# )zstd::ldexp(r¡   r›   rV   )rµ  Úns   &&rX   ÚldexpÚCppOverrides.ldexpÕ  rG  rW   c                ó   € R V  RV R2# )zstd::nextafter(r¡   r›   rV   rC  s   &&rX   Ú	nextafterÚCppOverrides.nextafterÚ  s   € ð !   2 a S¨Ð*Ð*rW   c                ó¶   € \         P                  P                  pVR 8X  d   R# VR8X  d   V  R2# VR8X  d	   V  RV  R2# Vf
   RV  RV  R	2# \        R
V: 24      h)Úcompile_errorúcompile error!Úruntime_errorú	; throw 1rô  rõ  rö  ú	std::max(ú, decltype(ú)(0))ú7unrecognized config cpp.inject_relu_bug_TESTING_ONLY = ©r   rù  Úinject_relu_bug_TESTING_ONLYr…   rû  s   & rX   ÚreluÚCppOverrides.reluß  s|   € ä�j‰j×5Ñ5ˆØ�/Ô!Ù#Ø�OÔ#Ø�S˜	�?Ð"Ø�JÔØ�S˜ Q C tÐ,Ð,ØŠ[Ø˜q˜c ¨Q¨C¨uÐ5Ð5ä ØIÈ#ÉÐQóð rW   c                ó   € R V  RV R2# )zmin_propagate_nan(r¡   r›   rV   r   s   &&rX   ÚminimumÚCppOverrides.minimumï  ó   € ð $ A 3 b¨¨¨1Ð-Ð-rW   c                ó   € R V  RV R2# )zmax_propagate_nan(r¡   r›   rV   r   s   &&rX   ÚmaximumÚCppOverrides.maximumô  rŒ  rW   c                ó   € R V  RV R2# )r�  r¡   r›   rV   r   s   &&rX   ÚfmaximumÚCppOverrides.fmaximumù  s   € ð ˜1˜#˜R ˜s !Ð$Ð$rW   c                ó   € V  R V RV 2# )ú ? ú : rV   )r—  r˜  Úcs   &&&rX   ÚwhereÚCppOverrides.whereþ  s   € ð ��C˜�s˜#˜a˜SÐ!Ð!rW   c                ó   € R V  RV R2# )zmod(r¡   r›   rV   r   s   &&rX   ÚmodÚCppOverrides.mod  s   € à�a�S˜˜1˜#˜QÐÐrW   c                ó0   € \        V \        V,          4      # r  )rP   rH   )Úvalr‡   s   &&rX   ÚconstantÚCppOverrides.constant  s   € ä˜C¤¨eÕ!4Ó5Ð5rW   c                ó  € \        \        P                  P                  V 4      4      p\        P                  P                  P                  \        P                  P                  V\        V 4      R 7      p\        P                  ! W14      # )©Úbounds)
rD   r5   re  Úrename_indexingr®  r¯  r°  r'   r3   r«  )r·  r‡   Úidx_strr¬   s   &&  rX   Ú
index_exprÚCppOverrides.index_expr  s`   € äœŸ™×0Ñ0°Ó6Ó7ˆÜ�h‰h�l‰l×#Ñ#Ü�H‰H×Ñ˜gÔ.CÀDÓ.Ið $ó 
ˆô �|Š|˜CÓ'Ð'rW   c                ó,   € \         P                  W4      # r  )r�  r¥  ©r·  r‡   s   &&rX   Ú
value_exprÚCppOverrides.value_expr  s   € ô ×&Ñ& tÓ3Ð3rW   c           	     ó^  € \        4       p\        P                  P                  P	                  4       pVP                  R V R24       \        P                  P                  V4      ;_uu_ 4        VP                  4       ;_uu_ 4        V! 4       pVP                  RV R24       RRR4       RRR4       VP                  R4       \        P                  P                  P                  V4       \        VRV R24      pV  RV RV 2#   + '       g   i     Ll; i  + '       g   i     Lw; i)	rÝ   ú = [&]úreturn rÁ   Nr‘  z())r”  z() : )r7   r5   re  r®  ra  rÉ   Úswap_buffersrË   r°  rb  rP   )ÚmaskÚbodyÚotherr¹   Úbody_varÚresultÚ
other_codes   &&&    rX   ÚmaskedÚCppOverrides.masked  sÜ   € ä‹~ˆô —8‘8—<‘<×&Ñ&Ó(ˆØ�‰˜˜x˜j¨Ð/Ô0Ü�X‰X×"Ñ" 4×(Õ(¨$¯+©+¯-­-Ù“VˆFØ�N‰N˜W V H¨AÐ.Ô/÷ +8×(ð 	�‰�sÔÜ	�‰×Ñ×Ñ Ô%ô " %¨9°X°J¸cÐ)BÓCˆ
Ø��s˜8˜* E¨*¨Ð6Ð6÷ +8¯-ú×(×(ús$   Á/DÂD		Â%DÄ	DÄDÄD,	c                ó   € V  R V 2# )ú && rV   r   s   &&rX   Úlogical_andÚCppOverrides.logical_and*  ó   € ð ��D˜˜ˆ}ÐrW   c                ó   € R V  2# )Ú!rV   ©r—  s   &rX   Úlogical_notÚCppOverrides.logical_not/  ó   € à�1�#ˆwˆrW   c                ó   € V  R V 2# )r    rV   r   s   &&rX   Ú
logical_orÚCppOverrides.logical_or3  r»  rW   c                ó   € V  R V 2# )ú != rV   r   s   &&rX   Úlogical_xorÚCppOverrides.logical_xor8  r»  rW   c                ó   € R V  RV  RV R2# )r‘  ú)(ú & r›   rV   r   s   &&rX   Úbitwise_andÚCppOverrides.bitwise_and=  ó   € à˜1˜#˜R ˜s # a S¨Ð*Ð*rW   c                ó   € R V  RV  R2# )r‘  z)(~r›   rV   r¾  s   &rX   Úbitwise_notÚCppOverrides.bitwise_notA  rÖ  rW   c                ó   € R V  RV  RV R2# )r‘  rÊ  ú | r›   rV   r   s   &&rX   Ú
bitwise_orÚCppOverrides.bitwise_orE  rÎ  rW   c                ó   € R V  RV  RV R2# )r‘  rÊ  rŸ   r›   rV   r   s   &&rX   Úbitwise_xorÚCppOverrides.bitwise_xorI  rÎ  rW   c                ó(  € \        4       pVP                  R 4       VP                  4       ;_uu_ 4        \        V P                  ,          pVP                  RV RV R24       VP                  RV RV RV R24       VP                  4       ;_uu_ 4        VP                  RV  R	24       R
R
R
4       VP                  RV  RV RV  RV R2	4       R
R
R
4       VP                  R4       V#   + '       g   i     LI; i  + '       g   i     L6; i)ú[&]()úconstexpr decltype(ú) max_shift = sizeof(z) * CHAR_BIT;ú$if ((static_cast<std::make_signed_t<ú>>(ú) < 0) || (ú >= max_shift))úreturn decltype(z)(0);Nz#)(static_cast<std::make_unsigned_t<z) << rÞ   ú()©r7   rÉ   rË   rH   r‡   ©r—  r˜  r¹   Úscalar_ts   &&  rX   Úbitwise_left_shiftÚCppOverrides.bitwise_left_shiftM  sø   € ä‹~ˆØ�‰�wÔØ�[‰[�]�]Ü# A§G¡GÕ,ˆHØ�N‰NØ% a SÐ(=¸h¸ZÀ}ÐUôð �N‰NØ6°x°jÀÀAÀ3ÀkÐRSÐQTÐTcÐdôð —‘—•Ø—‘Ð!1°!°°EÐ:Ô;÷ à�N‰NØ" 1 #Ð%HÈÈ
ÐRUÐVWÐUXÐX]Ð^_Ð]`Ð`bÐcô÷ ð 	�‰�tÔØˆ÷ —ú÷ �]ús$   ´A#DÂC.	Â-&DÃ.C>Ã9DÄD	c                ó.  € \        4       pVP                  R 4       VP                  4       ;_uu_ 4        \        V P                  ,          pVP                  RV RV RV R24       VP                  RV RV RV R24       VP                  4       ;_uu_ 4        VP                  R	V  R
V  R24       RRR4       VP                  R	V  R
V  RV R24       RRR4       VP                  R4       V#   + '       g   i     LF; i  + '       g   i     L6; i)rÚ  rÛ  rÜ  z ) * CHAR_BIT - std::is_signed_v<z>;rÝ  rÞ  rß  rà  rá  rÊ  z >> max_shift);Nú >> rÞ   râ  rã  rä  s   &&  rX   Úbitwise_right_shiftÚ CppOverrides.bitwise_right_shifta  sø   € ä‹~ˆØ�‰�wÔØ�[‰[�]�]Ü# A§G¡GÕ,ˆHØ�N‰NØ% a SÐ(=¸h¸ZÐGgÐhpÐgqÐqsÐtôð �N‰NØ6°x°jÀÀAÀ3ÀkÐRSÐQTÐTcÐdôð —‘—•Ø—‘Ð!1°!°°B°q°c¸ÐIÔJ÷ à�N‰NÐ-¨a¨S°°1°#°T¸!¸¸BÐ?Ô@÷ ð 	�‰�tÔØˆ÷	 —ú÷ �]ús$   ´A&DÂC1	Â3#DÃ1DÃ<DÄD	c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r’   ÚseedrJ  ©r”   r•   )r–   r  s   "€rX   r—   ÚCppOverrides.__annotate__t  s#   ø€ ÷ 8ñ 8‘5—:‘:ð 8¡u§z¡zñ 8rW   c                ó   € R V  RV R2# ©znormalized_rand_cpu(r¡   r›   rV   ©rî  rJ  s   &&rX   ÚrandÚCppOverrides.rands  s   € à% d V¨2¨f¨X°QÐ7Ð7rW   c          
      óœ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r’   rî  Úbase_offsetÚthreads_per_roundÚtidÚvecrï  )r–   r  s   "€rX   r—   rð  x  sO   ø€ ÷ 
=ñ 
=Ù�j‰jð
=á—Z‘Zð
=ñ !Ÿ:™:ð
=ñ �Z‰Zð	
=ñ
 �Z‰Zñ
=rW   c                ó   € R V  RV R2# rò  rV   )rî  r÷  rø  rù  rú  s   &&&&&rX   Ú
rand_eagerÚCppOverrides.rand_eagerw  s   € ð & d V¨2¨k¨]¸!Ð<Ð<rW   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rí  rï  )r–   r  s   "€rX   r—   rð  …  s#   ø€ ÷ .ñ .‘E—J‘Jð .©¯
©
ñ .rW   c                ó   € R V  RV R2# )z
randn_cpu(r¡   r›   rV   ró  s   &&rX   ÚrandnÚCppOverrides.randn„  s   € à˜D˜6  F 8¨1Ð-Ð-rW   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rí  rï  )r–   r  s   "€rX   r—   rð  ‰  s'   ø€ ÷ Añ A™Ÿ
™
ð A©E¯J©Jñ ArW   c           	     ó    € R V  RV RV RV R2	# )zrandint64_cpu(r¡   r›   rV   )rî  rJ  ÚlowÚhighs   &&&&rX   Ú	randint64ÚCppOverrides.randint64ˆ  s#   € à ˜v R¨ x¨r°#°°b¸¸¸aÐ@Ð@rW   c                ó   € R V  RV  RV  R2# )r‘  z)(1) / (decltype(z)(1) + std::exp(-r¦   rV   rÉ  s   &rX   ÚsigmoidÚCppOverrides.sigmoidŒ  s   € à˜1˜#Ð.¨q¨cÐ1BÀ1À#ÀRÐHÐHrW   c                ól  € \        4       pR V  R2pR V  R2pVP                  R4       VP                  4       ;_uu_ 4        VP                  RV  RV RV R24       VP                  RV  R	V RV R24       VP                  R
4       RRR4       VP                  R4       V#   + '       g   i     L#; i)r‘  ú)(0)rö  rÚ  úauto left = z > 0 ? r•  rÁ   úauto right = z < 0 ? úreturn left - right;Nrâ  ©r7   rÉ   rË   )rµ  r¹   Úscalar_zeroÚ
scalar_ones   &   rX   ÚsignÚCppOverrides.sign�  s®   € ô ‹~ˆØ! !  DÐ)ˆØ    4Ð(ˆ
Ø�‰�wÔØ�[‰[�]�]Ø�N‰N˜\¨!¨¨G°J°<¸sÀ;À-ÈqÐQÔRØ�N‰N˜]¨1¨#¨W°Z°LÀÀKÀ=ÐPQÐRÔSØ�N‰NÐ1Ô2÷ ð 	�‰�tÔØˆ÷ �]ús   Á AB#Â#B3	c          
      óL   <€ V ^8„  d   QhRS[ RS[ RS[RS[S[ S[3,          RR/# )r’   rÎ   r†   rù   Ú
extra_metar¾   N)rp   r9   Údictr   )r–   r  s   "€rX   r—   rð  ž  sE   ø€ ÷ "ñ "áð"ñ ð"ñ ð	"ñ
 ™™c˜•Nð"ð 
ñ"rW   c                ó   € \         hr  )ÚNotImplementedError)r1  rÎ   r†   rù   r  s   &&&&&rX   Úpartial_accumulateÚCppOverrides.partial_accumulatež  s
   € ô "Ð!rW   rV   ©NT)Ur  r  r  r  r  Ústaticmethodr¡  rÑ   rV  r«  r½  rÁ  rÊ  rÍ  rÑ  rÔ  rØ  rÛ  rß  rã  ræ  ré  rí  rñ  rý  r   r  r  r	  r  r  r  r  r  r  r"  r%  r(  r+  r.  r1  r4  r7  r:  r=  r@  rE  rI  rL  rO  rj  rm  rp  rs  rw  rz  r‡  rŠ  rŽ  r‘  r—  rš  rž  r¥  r©  rµ  r¹  r¿  rÃ  rÇ  rÌ  rÐ  rÔ  r×  ræ  rê  rô  rü  r   r  r	  r  r  r
  r  r  s   @rX   r�  r�  ‡  s  ø‡ € Ù%àñ)ó ð)ð ñ)ó ð)ð ñ)ó ð)ð ó2ó ð2ðh óó ðð  ñ<ó ð<ð
 ñ
 ó ð
 ð ñ ó ð ð ñ ó ð ð ñ&ó ð&ð ñ ó ð ð ñ!ó ð!ð ñ"ó ð"ð ñ ó ð ð ñ!ó ð!ð ñ#ó ð#ð ñ!ó ð!ð ñ%ó ð%ð ñ	ó ð	ð ñ ó ð ð ñ!ó ð!ð ñ	
ó ð	
ð ñ%ó ð%ð ñ ó ð ð ñ&ó ð&ð ñ"ó ð"ð ñ2ó ð2ð ñ!ó ð!ð ñ"ó ð"ð ñó ðð ñ&ó ð&ð ñ"ó ð"ð ñ"ó ð"ð ñ#ó ð#ð ñ!ó ð!ð ñ"ó ð"ð ñ!ó ð!ð ñ!ó ð!ð ñ!ó ð!ð ñ"ó ð"ð ñ'ó ð'ð ñ!ó ð!ð ñ"ó ð"ð ñ*ó ð*ð ñ"ó ð"ð  ñ'ó ð'ð ñ"ó ð"ð ñ!ó ð!ð ñ'ó ð'ð ñ+ó ð+ð ñó ðð ñ.ó ð.ð ñ.ó ð.ð ñ%ó ð%ð ñ"ó ð"ð ñ ó ð ð ñ6ó ð6ð ñ(ó ð(ð ñ4ó ð4ð
 ñ7ó ð7ð  ñó ðð ñó ðð ñó ðð ñó ðð ñ+ó ð+ð ñ&ó ð&ð ñ+ó ð+ð ñ+ó ð+ð ñó ðð& ñó ðð" ÷8ó ð8ð ÷
=ó ð
=ð ÷.ó ð.ð ÷Aó ðAð ñIó ðIð ñ
ó ð
÷"ö "rW   r�  rù  c                   ó^  a a€ ] tR tRt oRtV 3R lt]R 4       t]R 4       t]R 4       t	]R 4       t
]R 4       t]R	 4       t]R
 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t ]R 4       t!]R 4       t"]R  4       t#]R! 4       t$]R" 4       t%]R# 4       t&]R$ 4       t']R% 4       t(]R& 4       t)]R' 4       t*]R( 4       t+]R) 4       t,]R* 4       t-]R+ 4       t.]R, 4       t/]R- 4       t0]R. 4       t1]R/ 4       t2]R0 4       t3]R1 4       t4]R2 4       t5]R3 4       t6]R4 4       t7]R5 4       t8]R6 4       t9]R7 4       t:]R8 4       t;]R9 4       t<]R: 4       t=]R; 4       t>]R< 4       t?]R= 4       t@]R> 4       tA]R? 4       tB]R@ 4       tC]RA 4       tD]RB 4       tE]RC 4       tF]RD 4       tG]RE 4       tH]RF 4       tI]RG 4       tJ]RH 4       tK]RI 4       tL]RJ 4       tM]RK 4       tN]RL 4       tO]RWRM l4       tP]RWRN l4       tQ]RO 4       tR]RP 4       tS]RQ 4       tT]RR 4       tU]RS 4       tV]WRT 4       tX]WRU 4       tYRVtZVt[V ;t\# )XÚCppVecOverridesi«  z.Map element-wise ops to aten vectorization C++c                óú   <a€ \         SV `  V 4      oVV3R  lp\        \        4      P	                  4        FA  w  rE\        VRR4      \        J g   K  VR9  g   K%  \        SWC! VP                  4      4       KC  	  S# )c                 ó   <a € VV V3R  lpV# )c                  ó(  <€ V  Uu. uFW  p\        V\        \        P                  34      '       g-   \        V\        4      '       g   KA  VP
                  '       d   KU  VNKY  	  ppV  Uu. uF1  p\        V\        4      '       g   K  VP
                  '       g   K/  VNK3  	  pp\        V 4      pV'       dì   V'       dä   . pV  FÛ  p\        V\        \        P                  34      '       d¢   \        V\        P                  4      '       d9   VP                  '       g'   \        P                  ! V\        P                  4      pM%\        P                  ! V\        P                  4      p\        V\        4      '       d   VP                  MTpVP                  V4       KÝ  	  V'       dF   \!        V4      ^8X  d   \#        V4      pM)S
\$        P&                  J d   \#        VR,          4      VR&   V'       dÄ   V'       d¼   \        \(        P*                  \,        4      '       g   \/        R4      hV Uu. uF€  p\        V\        4      '       df   VP
                  '       gT   S
\$        P0                  \$        P2                  \$        P4                  39  d    \(        P*                  P7                  V4      MTNK‚  	  ppV'       d	   S
! V/ VB # \9        \$        S4      p\;        VS
P<                  4      pVf   \/        R4      hV! V / VB # u upi u upi u upi )r’   ºé   NNú+expected isinstance(V.kernel, CppVecKernel)z expected scalar_func is not None)r§   rk   r”   r•   rG   r«   r$  Ú	is_numberr3   r¥  rƒ   r•  rž  r4   rù   ÚappendrÛ   rM   r  r—  r5   re  ÚCppVecKernelr…   rô  r   r  Ú	broadcastr/  rD  r  )r™  Úkwargsr�  ÚscalarsÚvectorsÚnew_argsÚnew_argÚ
scalar_opsÚscalar_funcr4  Úfuncr1  s   *,       €€€rX   ÚwrapperÚ6CppVecOverrides.__new__.<locals>.wrap.<locals>.wrapper¾  st  ø€ ñ  $óá#˜Ü! #¬¬U¯Z©ZÐ'8×9Ò9Ü" 3¬×7ô ð ADÇ
Å
÷ �CÙ#ð ð ñ  $óá#˜Ü! #¤~×6ô à;>¿:½:÷ �CÙ#ð ð ô
   ›:�ßŸwØ!�HÛ#˜Ü% c¬C´·±Ð+<×=Ò=Ü)¨#¬u¯z©z×:Ò:À3Ç=Ç=À=Ü&)§n¢n°S¼%¿+¹+Ó&F¡ä&)§l¢l°3¼¿¹Ó&D Ü/9¸#¼x×/HÒ/H #§)¢)Èc˜CØ Ÿ™¨Ö,ñ  $÷ ô ˜8“}¨Ô)Ü#/°Ó#9™Ø¤×!6Ñ!6Ó6Ü'3°H¸RµLÓ'A˜ ™÷ ŸwÜ%¤a§h¡h´×=Ò=Ü,ØIóð ñ$ (0ó ñ (0˜Gô !+¨7´N× CÒ CØ(/¯¯¨Ø$(ä$3×$8Ñ$8Ü$3×$9Ñ$9Ü$3×$=Ñ$=ð("ô%"ô	 ŸH™H×.Ñ.¨wÔ7ð ")ò)ñ (0ð ð  ÷$ Ù Ð4¨VÑ4Ð4ô "'¤¸Ó!=�JÜ")¨*°d·m±mÓ"D�KØ"Ò*Ü,Ð-OÓPÐPÙ&¨Ð7°Ñ7Ð7ùòGùòùòF s0   †>LÁ	LÁLÁ)L
ÂL
ÂL
È0,LÉALrV   )r1  r2  r4  r1  s   f €€rX   ÚwrapÚ%CppVecOverrides.__new__.<locals>.wrap±  s   ù€ ÷D8ðL ˆNrW   r4  N)rµ  r¥  r©  )	r/  Ú__new__Úvarsr  ÚitemsrD  r  ÚsetattrÚ__func__)r&  r™  Úkargsr4  rÎ   Úmethodr1  r4  s   &*,   @€rX   r6  ÚCppVecOverrides.__new__®  sq   ù€ Ü‰w‰˜sÓ#ˆöS	ôj !¤Ó1×7Ñ7Ö9‰LˆDÜ�v˜{¨DÓ1´\ÕAÀdð Sö Gô
 ˜˜d D¨¯©Ó$9Ö:ñ :ð ˆrW   c                ó   € V  R V 2# )ró   rV   r   s   &&rX   r¡  ÚCppVecOverrides.add  ó   € à��C˜�sˆ|ÐrW   c                ó   € V  R V 2# )ú - rV   r   s   &&rX   rÑ   ÚCppVecOverrides.sub  r@  rW   c                ó   € V  R V 2# ©rž   rV   r   s   &&rX   rV  ÚCppVecOverrides.mul  r@  rW   c                ó   € V  R V 2# r   rV   r   s   &&rX   ÚtruedivÚCppVecOverrides.truediv  r@  rW   c                óØ   € \        V \        4      '       dQ   V P                  \        P                  \        P
                  \        P                  \        P                  39   d   V  # V  R 2# )z.abs()rÄ  rÉ  s   &rX   rÊ  ÚCppVecOverrides.abs   sP   € ä�aœ×(Ò(¨Q¯W©WÜ�K‰KÜ�L‰LÜ�L‰LÜ�L‰Lð	9
ô .
ð �SˆMØ��Fˆ|ÐrW   c                ó   € V  R 2# )z.sin()rV   rÉ  s   &rX   rÍ  ÚCppVecOverrides.sin,  ó   € à��Fˆ|ÐrW   c                ó   € V  R 2# )z.cos()rV   rÉ  s   &rX   rÑ  ÚCppVecOverrides.cos0  rN  rW   c                ó   € V  R 2# )z.exp()rV   rÉ  s   &rX   rØ  ÚCppVecOverrides.exp4  rN  rW   c                ó   € V  R 2# )z.exp2()rV   rÉ  s   &rX   rÛ  ÚCppVecOverrides.exp28  ó   € à��Gˆ}ÐrW   c                ó   € V  R 2# )z.expm1()rV   rÉ  s   &rX   rß  ÚCppVecOverrides.expm1<  ó   € à��Hˆ~ÐrW   c                ó   € V  R 2# )z.erf()rV   rÉ  s   &rX   rã  ÚCppVecOverrides.erf@  rN  rW   c                ó   € V  R 2# )z.erfc()rV   rÉ  s   &rX   ræ  ÚCppVecOverrides.erfcD  rU  rW   c                ó   € V  R 2# )z	.erfinv()rV   rÉ  s   &rX   ré  ÚCppVecOverrides.erfinvH  ó   € à��IˆÐrW   c                ó   € V  R 2# )z.sqrt()rV   rÉ  s   &rX   rí  ÚCppVecOverrides.sqrtL  rU  rW   c                ó8  € \        \        P                  \        4      '       g   \	        R 4      h\        V \
        4      '       g   \	        R4      hV P                  f   \	        R4      h\        P                  P                  V P                  4       RV  RV R2# )r%  rª  úexpected x.dtype is not Noner’  ú == r›   ©r§   r5   re  r(  r…   rG   r‡   Ú_get_mask_typerC  s   &&rX   ÚeqÚCppVecOverrides.eqP  óz   € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ˜!œ^×,Ò,Ü Ð!IÓJÐJØ�7‰7Š?Ü Ð!?Ó@Ð@Ü—(‘(×)Ñ)¨!¯'©'Ó2Ð3°1°Q°C°t¸A¸3¸aÐ@Ð@rW   c                ó&  € \        \        P                  \        4      '       g   \	        R 4      h\        V \
        4      '       g   \	        R4      hV P                  \        P                  8X  dY   VP                  \        P                  8w  d   \	        R4      h\        \        P                  P                  W34      w  r#V RV 2# V P                  f   \	        R4      h\        P                  P                  V P                  4       RV  RV R2# )r%  rª  zexpected y.dtype == torch.boolrÆ  rc  r’  r›   )r§   r5   re  r(  r…   rG   r‡   rƒ   ro   rO   r°  rf  )rµ  rD  Úx_castÚy_casts   &&  rX   ÚneÚCppVecOverrides.neZ  sÑ   € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ˜!œ^×,Ò,Ü Ð!IÓJÐJØ�7‰7”e—j‘jÔ Ø�w‰wœ%Ÿ*™*Ô$Ü$Ð%EÓFÐFÜ1´!·(±(×2BÑ2BÀQÀFÓK‰NˆFØ�X˜T & Ð*Ð*à�w‰wŠÜ$Ð%CÓDÐDÜ—h‘h×-Ñ-¨a¯g©gÓ6Ð7°q¸¸¸4À¸sÀ!ÐDÐDrW   c                ó8  € \        \        P                  \        4      '       g   \	        R 4      h\        V \
        4      '       g   \	        R4      hV P                  f   \	        R4      h\        P                  P                  V P                  4       RV  RV R2# )r%  rª  rc  r’  rÂ   r›   re  rC  s   &&rX   ÚltÚCppVecOverrides.ltj  óz   € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ˜!œ^×,Ò,Ü Ð!IÓJÐJØ�7‰7Š?Ü Ð!?Ó@Ð@Ü—(‘(×)Ñ)¨!¯'©'Ó2Ð3°1°Q°C°s¸1¸#¸QÐ?Ð?rW   c                ó8  € \        \        P                  \        4      '       g   \	        R 4      h\        V \
        4      '       g   \	        R4      hV P                  f   \	        R4      h\        P                  P                  V P                  4       RV  RV R2# )r%  rª  rc  r’  z > r›   re  rC  s   &&rX   ÚgtÚCppVecOverrides.gtt  rr  rW   c                ó8  € \        \        P                  \        4      '       g   \	        R 4      h\        V \
        4      '       g   \	        R4      hV P                  f   \	        R4      h\        P                  P                  V P                  4       RV  RV R2# )r%  rª  rc  r’  ú <= r›   re  rC  s   &&rX   ÚleÚCppVecOverrides.le~  ri  rW   c                ó8  € \        \        P                  \        4      '       g   \	        R 4      h\        V \
        4      '       g   \	        R4      hV P                  f   \	        R4      h\        P                  P                  V P                  4       RV  RV R2# )r%  rª  rc  r’  ú >= r›   re  rC  s   &&rX   ÚgeÚCppVecOverrides.geˆ  ri  rW   c                ó   € V  R V 2# ©rË  rV   rC  s   &&rX   Úand_ÚCppVecOverrides.and_’  r@  rW   c                ó   € V  R 2# )z.rsqrt()rV   rÉ  s   &rX   rñ  ÚCppVecOverrides.rsqrt–  rX  rW   c                ó   € V  R V R2# )z.pow(r›   rV   r   s   &&rX   r	  ÚCppVecOverrides.powš  s   € à��E˜!˜˜AˆÐrW   c                ó   € V  R 2# )z.log()rV   rÉ  s   &rX   r  ÚCppVecOverrides.logž  rN  rW   c                ó   € V  R 2# )z.round()rV   rÉ  s   &rX   r  ÚCppVecOverrides.round¢  rX  rW   c                ó   € V  R 2# )z.floor()rV   rÉ  s   &rX   r  ÚCppVecOverrides.floor¦  rX  rW   c                ó   € V  R 2# )z.ceil()rV   rÉ  s   &rX   r  ÚCppVecOverrides.ceilª  rU  rW   c                ó   € V  R 2# )z.trunc()rV   rÉ  s   &rX   r  ÚCppVecOverrides.trunc®  rX  rW   c                ó   € V  R V R2# )z.fmod(r›   rV   r   s   &&rX   r%  ÚCppVecOverrides.fmod²  s   € à��F˜1˜#˜QÐÐrW   c                ó   € V  R 2# )z	.lgamma()rV   rÉ  s   &rX   r.  ÚCppVecOverrides.lgamma¶  r_  rW   c                ó*   € \        W4      w  rV  R V 2# r  ©rL   r   s   &&rX   r¹  ÚCppVecOverrides.logical_andº  ó   € ä,¨QÓ2‰ˆØ��C˜�sˆ|ÐrW   c                ó   € R V  2# ©Ú~rV   r¾  s   &rX   r¿  ÚCppVecOverrides.logical_not¿  rÁ  rW   c                ó*   € \        W4      w  rV  R V 2# ©rÓ  r•  r   s   &&rX   rÃ  ÚCppVecOverrides.logical_orÃ  r—  rW   c                ó*   € \        W4      w  rV  R V 2# ©rŸ   r•  r   s   &&rX   rÇ  ÚCppVecOverrides.logical_xorÈ  r—  rW   c                ó*   € \        W4      w  rV  R V 2# r  r•  r   s   &&rX   rÌ  ÚCppVecOverrides.bitwise_andÍ  r—  rW   c                ó   € R V  2# r™  rV   r¾  s   &rX   rÐ  ÚCppVecOverrides.bitwise_notÒ  rÁ  rW   c                ó*   € \        W4      w  rV  R V 2# r�  r•  r   s   &&rX   rÔ  ÚCppVecOverrides.bitwise_orÖ  r—  rW   c                ó*   € \        W4      w  rV  R V 2# r   r•  r   s   &&rX   r×  ÚCppVecOverrides.bitwise_xorÛ  r—  rW   c                ó   € V  R V 2# )z << rV   r   s   &&rX   ræ  Ú"CppVecOverrides.bitwise_left_shiftà  ó   € à��D˜˜ˆ}ÐrW   c                ó   € V  R V 2# )ré  rV   r   s   &&rX   rê  Ú#CppVecOverrides.bitwise_right_shiftä  r¬  rW   c                ó    € \        \        P                  \        4      '       g   \	        R 4      h\        P                  P                  W4       # )r%  )r§   r5   re  r(  r…   Úload)rÎ   rJ  s   &&rX   Ú	load_seedÚCppVecOverrides.load_seedè  s5   € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ—(‘(—-‘- Ó-Ð.Ð/rW   c                ó˜   € \        \        P                  \        4      '       g   \	        R 4      h\        4       pRV  R2p\        WV4      # )r%  z)result[offset_idx] = normalized_rand_cpu(ú, offset[offset_idx]);©r§   r5   re  r(  r…   r7   rF   ©rî  rJ  r¹   Úrand_functions   &&  rX   rô  ÚCppVecOverrides.randî  sG   € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ‹~ˆà7¸°vÐ=SÐTð 	ô ˜F¨-Ó8Ð8rW   c                ó˜   € \        \        P                  \        4      '       g   \	        R 4      h\        4       pRV  R2p\        WV4      # )r%  zresult[offset_idx] = randn_cpu(r´  rµ  r¶  s   &&  rX   r   ÚCppVecOverrides.randnø  sB   € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ‹~ˆØ9¸$¸Ð?UÐVˆÜ˜F¨-Ó8Ð8rW   c                óÂ   € \        \        P                  \        4      '       g   \	        R 4      h\        4       pRV  RV RV R2p\        WV\        P                  4      # )r%  z#result[offset_idx] = randint64_cpu(z, offset[offset_idx], r¡   rÞ   )	r§   r5   re  r(  r…   r7   rF   rƒ   r•  )rî  rJ  r  r  r¹   r·  s   &&&&  rX   r  ÚCppVecOverrides.randint64   sZ   € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ‹~ˆØ=¸d¸VÐCYÐZ]ÐY^Ð^`ÐaeÐ`fÐfhÐiˆÜ˜F¨-¼¿¹ÓEÐErW   c                ó   € V P                   VP                   8w  d   \        R 4      h\        V P                   4      '       dr   RV  R2p\        P                  P                  VP                   4      ^8  d5   V R^\        P                  P                  ,          ^,
           RV RV R2pRV  RV R2# V  R\        P                  W4       R	V 2# )
z;remainder vec implementation expect the same inputs' dtype.r‘  r›   z::blend<rÀ  ú(1), zremainder_integral(r¡   z - (z) * )	r‡   r…   r   r5   re  Ú_get_raw_num_vectorsÚtiling_factorr  r  ©r—  r˜  Ú_ts   && rX   Ú	remainderÚCppVecOverrides.remainder  sÊ   € à�7‰7�a—g‘gÔÜ ØMóð ô ˜AŸG™G×$Ò$à˜Q˜C˜qÐ!ˆBÜ�x‰x×,Ñ,¨Q¯W©WÓ5¸Ô9Ø�d˜( A¬¯©×)?Ñ)?Õ$?À1Õ#DÐ"EÀRÈÀtÈ5ÐQRÐPSÐSTÐU�Ø(¨¨¨2¨a¨S°Ð2Ð2Ø��Dœ×1Ñ1°!Ó7Ð8¸¸Q¸CÐ@Ð@rW   c                ó   € V  R 2# )z.tan()rV   r¾  s   &rX   r   ÚCppVecOverrides.tan  rN  rW   c           	     ó�   € \         P                  P                  '       d#   R V  R2pR V  R2pR V  R2pV RV RV RV  RV 2	# V  R2# )	r‘  rö  z)(2)z)(-2)z / (z + (rž   z).exp()) - z.tanh())r   rù  Úuse_decompose_tanh)r—  Úvec_oneÚvec_twoÚvec_minus_twos   &   rX   r  ÚCppVecOverrides.tanh  so   € ä�:‰:×(×(Ð(Ø! !  DÐ)ˆGØ! !  DÐ)ˆGØ'¨ s¨%Ð0ˆMà�)˜4 ˜y¨¨]¨O¸3¸q¸cÀÈWÈIÐVðð �S˜�=Ð rW   c                ó   € V  R 2# )z.reciprocal()rV   r¾  s   &rX   Ú
reciprocalÚCppVecOverrides.reciprocal&  s   € à��MÐ"Ð"rW   c                ó   € V  R 2# )z.atan()rV   rÉ  s   &rX   rI  ÚCppVecOverrides.atan*  rU  rW   c                ó   € V  R 2# )z.acos()rV   rÉ  s   &rX   r1  ÚCppVecOverrides.acos.  rU  rW   c                ó   € V  R 2# )z.asin()rV   rÉ  s   &rX   r=  ÚCppVecOverrides.asin2  rU  rW   c                ó   € V  R 2# )z.cosh()rV   rÉ  s   &rX   r7  ÚCppVecOverrides.cosh6  rU  rW   c                ó   € V  R 2# )z.sinh()rV   rÉ  s   &rX   r:  ÚCppVecOverrides.sinh:  rU  rW   c                ó   € V  R 2# )z.log10()rV   rÉ  s   &rX   rp  ÚCppVecOverrides.log10>  rX  rW   c                ó   € V  R 2# )z.log2()rV   rÉ  s   &rX   rs  ÚCppVecOverrides.log2B  rU  rW   c                ó   € V  R V R2# )z.nextafter(r›   rV   rC  s   &&rX   rz  ÚCppVecOverrides.nextafterF  s   € à��K ˜s !Ð$Ð$rW   c                ó   € V  R V R2# )z
.copysign(r›   rV   r   s   &&rX   rO  ÚCppVecOverrides.copysignJ  s   € à��J˜q˜c Ð#Ð#rW   c                ó   € V  R V R2# )z.atan2(r›   rV   r   s   &&rX   rE  ÚCppVecOverrides.atan2N  ó   € à��G˜A˜3˜aÐ Ð rW   c                ó   € V  R V R2# )z.hypot(r›   rV   r   s   &&rX   rm  ÚCppVecOverrides.hypotR  rä  rW   c           
     ó<   € R V  R2pR V  R2pV RV RV  RV RV  R2
# )r‘  rö  z)(0.5)z * ((ró   z)/(rB  z)).log()rV   )rµ  rÉ  Úvec_one_halfs   &  rX   rL  ÚCppVecOverrides.atanhV  sE   € ð ˜a˜S Ð%ˆØ" 1 # VÐ,ˆØ�˜u W I¨S°°°3°w°i¸sÀ1À#ÀXÐNÐNrW   c                ó¶  € R V  R2p\        4       pVP                  R4       VP                  4       ;_uu_ 4        VP                  RV  R24       VP                  RV R24       VP                  R4       VP                  R4       VP                  R	4       VP                  R
V  R24       RRR4       VP                  R4       V#   + '       g   i     L#; i)r‘  r›   rÚ  zauto abs_x = z.abs();zauto one = z(1);zauto inv_abs_x = one / abs_x;zaauto correction = (one / (one + inv_abs_x)) / ((one + inv_abs_x * inv_abs_x).sqrt() + inv_abs_x);z1auto result = abs_x.log1p() + correction.log1p();zreturn result.copysign(rÞ   Nrâ  r  )rµ  Úvec_tr¹   s   &  rX   r@  ÚCppVecOverrides.asinh]  s¾   € à˜A˜3˜aÐ ˆÜ‹~ˆØ�‰�wÔØ�[‰[�]�]ð
 �N‰N˜]¨1¨#¨WÐ5Ô6Ø�N‰N˜[¨¨¨tÐ4Ô5Ø�N‰NÐ:Ô;Ø�N‰NðFôð
 �N‰NÐNÔOØ�N‰NÐ4°Q°C°rÐ:Ô;÷ ð 	�‰�tÔØˆ÷! �]ús   ºA3CÃC	c                ó   € V  R 2# )z.acosh()rV   rÉ  s   &rX   r4  ÚCppVecOverrides.acosht  rX  rW   c                ó¶   € \         P                  P                  pVR 8X  d   R# VR8X  d   V  R2# VR8X  d	   V  RV  R2# Vf
   RV  RV  R	2# \        R
V: 24      h)r}  r~  r  r€  rô  rõ  rö  zat::vec::clamp_min(r‚  rƒ  r„  r…  rû  s   & rX   r‡  ÚCppVecOverrides.relux  s|   € ä�j‰j×5Ñ5ˆØ�/Ô!Ù#Ø�OÔ#Ø�S˜	�?Ð"Ø�JÔØ�S˜ Q C tÐ,Ð,ØŠ[Ø(¨¨¨;°q°c¸Ð?Ð?ä ØIÈ#ÉÐQóð rW   c                ó   € R V  RV  RV  R2# )r‘  z)(1)/(decltype(z)(1) + z.neg().exp())rV   rÉ  s   &rX   r	  ÚCppVecOverrides.sigmoid‰  s   € à˜1˜#˜_¨Q¨C¨w°q°c¸ÐGÐGrW   c                ó   € V  R 2# )z.neg()rV   rÉ  s   &rX   rÔ  ÚCppVecOverrides.neg�  rN  rW   c           	     ó   € \        V P                  4      '       d0   V P                  VP                  8w  d   \        R 4      hRV  RV R2# \        ;QJ d    R W3 4       F  '       d   K   RM	  RM! R W3 4       4      '       g   \        R4      hRV R2pV R	V R
V R\	        \
        P                  P                  4       R2pRV  RV R2# )zDdiv_floor_floating_vec implementation expect the same inputs' dtype.zdiv_floor_floating_vec(r¡   r›   c              3   óL   "  € T F  p\        VP                  4      x € K  	  R # 5ir  )r   r‡   )r  Úitems   & rX   r  Ú+CppVecOverrides.floordiv.<locals>.<genexpr>š  s   é € ÐHÁ¸Ô(¨¯©×4Ð4Ãùó   ‚"$FTz=expected all(is_integer_dtype(item.dtype) for item in [a, b])r‘  ú::set(r¾  r  )r   r‡   r…   r#  rE   r5   re  Ú	num_elemsrÁ  s   && rX   r  ÚCppVecOverrides.floordiv‘  sÐ   € ä˜!Ÿ'™'×"Ò"Ø�w‰w˜!Ÿ'™'Ô!Ü$ØZóð ð -¨Q¨C¨r°!°°AÐ6Ð6ç“CÑHÀ!ÁÓH—C—C’CÑHÀ!ÁÓH×HÒHÜ$ØSóð ð ˜Q˜C˜qÐ!ˆBØ�$�f˜R˜D  a S¨¬;´q·x±x×7IÑ7IÓ+JÐ*KÈ1ÐMˆAØ+¨A¨3¨b°°°1Ð5Ð5rW   c           	     óz   € R V R2pV RV RV R\        \        P                  P                  4       R2pV  RV 2# )r‘  r›   rú  r¾  r¡   r!  )rE   r5   re  rû  rÁ  s   && rX   r"  ÚCppVecOverrides.truncdiv£  sP   € ð ˜˜˜1ÐˆØˆd�&˜˜˜E !  B¤{´1·8±8×3EÑ3EÓ'FÐ&GÀqÐIˆØ��C˜�sˆ|ÐrW   c                ó  € V P                   \        P                  8X  dY   VP                   \        P                  8w  d   \        R 4      h\	        \
        P                  P                  W34      w  r#V RV 2# RV  RV R2# )úexpected b.dtype == torch.boolrË  úat::vec::minimum(r¡   r›   ©r‡   rƒ   ro   r…   rO   r5   re  r°  ©r—  r˜  Úa_castÚb_casts   &&  rX   rŠ  ÚCppVecOverrides.minimumª  ón   € à�7‰7”e—j‘jÔ Ø�w‰wœ%Ÿ*™*Ô$Ü$Ð%EÓFÐFÜ1´!·(±(×2BÑ2BÀQÀFÓK‰NˆFØ�X˜S  Ð)Ð)à& q c¨¨A¨3¨aÐ0Ð0rW   c                ó  € V P                   \        P                  8X  dY   VP                   \        P                  8w  d   \        R 4      h\	        \
        P                  P                  W34      w  r#V RV 2# RV  RV R2# )r   rÓ  úat::vec::maximum(r¡   r›   r  r  s   &&  rX   rŽ  ÚCppVecOverrides.maximum´  r  rW   c                ó&   € R V  RV  RV RV  RV R2# )r‘  ú
)::blendv(r¡   rÂ   r›   rV   r   s   &&rX   r‘  ÚCppVecOverrides.fmaximum¾  s)   € à˜1˜#˜Z¨ s¨"¨Q¨C¨r°!°°C¸°s¸!Ð<Ð<rW   c                ó   € V  R V  2# rE  rV   r¾  s   &rX   ÚsquareÚCppVecOverrides.squareÂ  r@  rW   c                óÎ  € \        \        P                  \        4      '       g   \	        R 4      hVP
                  \        P                  8X  dc   VP
                  \        P                  8w  d   \	        R4      h\        \        P                  P                  WV34      w  r4pRV RV RV RV R2	# RV RV RV R\        P                  P                  WP
                  4       R2	# )r%  zexpected c.dtype == torch.boolr‘  r  r¡   r›   )r§   r5   re  r(  r…   r‡   rƒ   ro   rO   r°  Ú_get_mask_cast)r—  r˜  r–  Úblendv_aÚblendv_bÚblendv_cs   &&&   rX   r—  ÚCppVecOverrides.whereÆ  sÉ   € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOØ�7‰7”e—j‘jÔ Ø�w‰wœ%Ÿ*™*Ô$Ü$Ð%EÓFÐFÜ+?Ü—‘× Ñ  1¨ )ó,Ñ(ˆH ð ˜x˜j¨
°8°*¸B¸x¸jÈÈ8È*ÐTUÐVÐVà˜q˜c ¨A¨3¨b°°°2´a·h±h×6MÑ6MÈa×QXÑQXÓ6YÐ5ZÐZ[Ð\Ð\rW   c                óœ  € \        4       pR V  R2pR V  R2pR V  RV RV RV RV  R2pR V  RV RV RV  RV R2pVP                  R4       VP                  4       ;_uu_ 4        VP                  RV R	24       VP                  R
V R	24       VP                  R4       RRR4       VP                  R4       V#   + '       g   i     L#; i)r‘  r  rö  r  r¡   rÂ   r›   rÚ  r  rÁ   r  r  Nrâ  r  )rµ  r¹   Úvec_zerorÉ  Úblendv_lÚblendv_rs   &     rX   r  ÚCppVecOverrides.signÔ  sâ   € ä‹~ˆØ˜q˜c Ð&ˆØ˜a˜S Ð%ˆØ˜q˜c ¨H¨:°R¸°yÀÀ8À*ÈCÐPQÈsÐRSÐTˆØ˜q˜c ¨H¨:°R¸°yÀÀ1À#ÀSÈÈ
ÐRSÐTˆØ�‰�wÔØ�[‰[�]�]Ø�N‰N˜\¨(¨°1Ð5Ô6Ø�N‰N˜]¨8¨*°AÐ6Ô7Ø�N‰NÐ1Ô2÷ ð 	�‰�tÔØˆ÷ �]ús   Á$<B;Â;C	c                óŒ  € V\         P                  \         P                  \         P                  \         P                  \         P
                  \         P                  \         P                  \         P                  \         P                  \         P                  \         P                  39  d   \        \         R V 24      h\        V \        4      '       g   \        R4      hV P                   p\"        P$                  P'                  WV4      p\"        P$                  P(                  P+                  \"        P$                  P,                  V4      pVP/                  RW3RV/4       V'       dW   V\0        9   dL   V\         P                  8X  d7   \2        P4                  '       g!   \"        P$                  P7                  WWQ4       V# )ú does not support rª  r«  r¯   )rƒ   ro   Úfloat64rm   Úbfloat16Úfloat16rÅ  Úint8r”  r•  Úfloat8_e4m3fnÚfloat8_e5m2r…   r  r§   rG   r‡   r5   re  r­  r®  r¯  r°  r±  r‚   r   r²  r³  r´  s   &&&&  rX   r«  ÚCppVecOverrides.to_dtypeã  s*  € àÜ�J‰JÜ�M‰MÜ�K‰KÜ�N‰NÜ�M‰MÜ�K‰KÜ�J‰JÜ�K‰KÜ�K‰KÜ×ÑÜ×Ñð
ô 
ô !¤H :Ð-?À¸wÐ!GÓHÐHÜ˜!œ^×,Ò,Ü Ð!IÓJÐJØ—G‘Gˆ	Ü�x‰x×)Ñ)¨!°IÓ>ˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜j¨1¨*°{ÀIÐ6NÔOçØœÔ&ØœUŸ[™[Ô(Ü×2×2Ð2ä�H‰H×(Ñ(¨°vÔEØˆrW   c                ó�  € V\         P                  \         P                  \         P                  39  d   \	        \
         R V 24      h\        V \        4      '       g   \	        R4      hV P                  p\        P                  P                  WVRR7      p\        P                  P                  P                  \        P                  P                  V4      pVP                  RW3RV/4       V\         9   dL   V\         P"                  8X  d7   \$        P&                  '       g!   \        P                  P)                  WWQ4       V# )r  rª  Tr»  r½  r¯   )rƒ   rÅ  r!  r”  r…   r  r§   rG   r‡   r5   re  r­  r®  r¯  r°  r±  r‚   rm   r   r²  r³  r´  s   &&&&  rX   r½  ÚCppVecOverrides.round_to_int  sð   € àÜ�K‰KÜ�J‰JÜ�K‰Kð
ô 
ô
 !¤H :Ð-?À¸wÐ!GÓHÐHÜ˜!œ^×,Ò,Ü Ð!IÓJÐJØ—G‘Gˆ	Ü�x‰x×)Ñ)¨!°IÈÐ)ÓMˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜n¨q¨j¸;È	Ð:RÔSà”]Ô"ØœUŸ[™[Ô(Ü×2×2Ð2ä�H‰H×(Ñ(¨°vÔEØˆrW   c                ó„   € \         P                  P                  pVR 8X  d	   V  RV  R2# Vf   V  R2# \        RV: 24      h)rô  rõ  rö  z.log1p()r÷  rø  rû  s   & rX   rý  ÚCppVecOverrides.log1p  sT   € ä�j‰j×6Ñ6ˆØ�*ÔØ�S˜ Q C tÐ,Ð,ØŠ[Ø�S˜�>Ð!ä ØJÈ3É'ÐRóð rW   c                ó0  a€ \        \        P                  \        4      '       g   \	        R 4      h\        4       p\        P                  P                  P                  4       p\        P                  P                  V 4      ;_uu_ 4       pVP                  RV R24       \        P                  P                  V4      ;_uu_ 4        VP                  4       ;_uu_ 4        V! 4       pVP                  RV R24       RRR4       RRR4       RRR4       VP                  R4       \        P                  P                  P                  V4       XP                  oV R2pV3R lpVP                  '       d   Tp	MV! V4      p	\!        V\"        S,          4      p
V! V
4      p\        X\$        4      '       g   \	        V4      hVP                  '       Edy   \        4       pVP                  R4       \        P                  P                  V4      ;_uu_ 4        VP                  4       ;_uu_ 4        VP                  R	V R
24       VP                  4       ;_uu_ 4        VP                  RV R24       RRR4       VP                  R4       VP                  4       ;_uu_ 4        \        P                  P                  P'                  \        P                  P                  V	4      p\        P                  P                  P'                  \        P                  P                  V4      p\        V\$        4      '       g   \	        V4      h\        V\$        4      '       g   \	        V4      hSVn        SVn        \        P                  P(                  pVP                  RVP+                  W\V4       R24       RRR4       RRR4       RRR4       VP                  R4       \        P                  P                  P'                  \        P                  P                  V4      p\        V\$        4      '       g   \	        R4      hRVn        M§VP                  '       dL   \        P                  P                  P'                  \        P                  P                  V  RV	 RV 24      pMJ\        P                  P                  P'                  \        P                  P                  V  RV RV
 24      pVP-                  RWW&3/ 4       V#   + '       g   i     EL; i  + '       g   i     EL$; i  + '       g   i     EL0; i  + '       g   i     ELÜ; i  + '       g   i     EL«; i  + '       g   i     EL·; i  + '       g   i     ELÃ; i)r%  rÝ   r¬  r­  rÁ   Nrâ  c                 óÀ   <€ S\         P                  8X  d%   \        P                  P	                  4        R V  R2# \        P                  P                  S4       RV  R2# )ú::from(r›   r’  )rƒ   ro   r5   re  rf  Ú_get_vec_type)r¹   r‡   s   &€rX   Úmaskify_or_vecifyÚ1CppVecOverrides.masked.<locals>.maskify_or_vecify5  s]   ø€ ð œEŸJ™JÔ&ô —8‘8×*Ñ*Ó,Ð-¨W°T°F¸!Ð<ðô Ÿ™×.Ñ.¨uÓ5Ð6°a¸°v¸QÐ?ðrW   ú[&]úif (z.all_zero())Úelseú+expected isinstance(csevar, CppCSEVariable)Tr”  r•  rµ  )r§   r5   re  r(  r…   r7   r®  ra  rµ  rÉ   r®  rË   r°  rb  r‡   r«   rP   rH   rG   r¯  Ú	overridesr—  r±  )r¯  r°  r±  r¹   r¬   Únew_maskr³  Ú	body_coder-  Úbody_code_vecr´  Úother_code_vecÚbody_vec_varÚother_vec_varr3  r¸  r‡   s   &&&             @rX   rµ  ÚCppVecOverrides.masked$  sà  ø€ äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ‹~ˆÜ�h‰h�l‰l×!Ñ!Ó#ˆÜ�X‰X�_‰_˜T×"Ô" hØ�N‰N˜U 3 % vÐ.Ô/Ü—‘×&Ñ& t×,Õ,¨d¯k©k¯m­mÙ›�Ø—‘ ¨¨°Ð2Ô3÷ /<×,÷ #ð
 	�‰�sÔÜ	�‰×Ñ×Ñ Ô%à—‘ˆØ�e˜2�Jˆ	õ	ð �=�=ˆ=Ø%‰Má-¨iÓ8ˆMÜ! %¬°eÕ)<Ó=ˆ
á*¨:Ó6ˆÜ˜(¤N×3Ò3Ü  Ó*Ð*Ø�?�?‰?Ü“>ˆDØ�N‰N˜5Ô!Ü—‘×&Ñ& t×,Õ,¨d¯k©k¯m­mØ—‘  h Z¨|Ð<Ô=Ø—[‘[—]•]Ø—N‘N W¨^Ð,<¸AÐ#>Ô?÷ #à—‘˜vÔ&Ø—[‘[—]•]ä#$§8¡8§<¡<×#8Ñ#8ÜŸ™×(Ñ(Ø%ó$�Lô %&§H¡H§L¡L×$9Ñ$9ÜŸ™×(Ñ(Ø&ó%�Mô & l´N×CÒCÜ,¨\Ó:Ð:Ü% m´^×DÒDÜ,¨]Ó;Ð;Ø).�LÔ&Ø*/�MÔ'ô Ÿ™×*Ñ*ð ð —N‘NØ! )§/¡/°(È-Ó"XÐ!YÐYZÐ[ô÷) #÷ /<×,ð8 �N‰N˜4Ô Ü—X‘X—\‘\×*Ñ*Ü—‘× Ñ ØóˆFô ˜f¤n×5Ò5Ü$Ð%RÓSÐSØ ˆF�MØ�]�]ˆ]Ü—X‘X—\‘\×*Ñ*Ü—‘× Ñ  T F¨#¨m¨_¸CÀÐ?OÐ"Pó‰Fô —X‘X—\‘\×*Ñ*Ü—‘× Ñ  T F¨#¨i¨[¸¸J¸<Ð"HóˆFð
 	×Ñ˜h¨°UÐ(CÀRÔHØˆ÷W /<¯m¨mú×,×,Ð,ú÷ #×"Ð"ú÷@ #—]�]ú÷ #—]�]ú÷ /<¯m¨mú×,×,Ð,ús‘   Â
=T4ÃT 	Ã TÃ=T 	ÄT4ÈVÈ1.U0	ÉUÉ51U0	Ê&DUÎ:U0	ÏVÔTÔ	T 	Ô T1Ô+	T4Ô4U	ÕUÕ	U0	ÕU-Õ'	U0	Õ0VÕ;	VÖV	c                óˆ  € \        \        P                  \        4      '       g   \	        R 4      h\        P                  P                  V 4      p\        P                  P                  \        P                  P                  ,          p\        P                  P                  W#4      pV^ 8X  d   \        P                  W4      # Ve¯   \        P                  P                  P                  \        P                  P                  \        V4      \        V 4      R7      p\         P"                  ! WQ4      p\        V\$        4      '       d   VP&                  p\        P                  P)                  Wd4      pM9\        P                  P+                  RW!\        P                  P                  4      pVP-                  RW3/ 4       V# )r%  Nr¡  r¥  )r§   r5   re  r(  r…   r£  ÚitervarsÚ
tiling_idxÚ_try_get_const_strider�  r¥  r®  r¯  r°  rD   r'   r3   r«  r4   rù   ÚarangeÚ_load_or_store_non_contiguousr±  )r·  r‡   r“   Ú
tiling_varÚstriderZ  rù   r¸  s   &&      rX   r¥  ÚCppVecOverrides.index_expry  s3  € äœ!Ÿ(™(¤L×1Ò1Ü Ð!NÓOÐOÜ—‘×(Ñ(¨Ó.ˆÜ—X‘X×&Ñ&¤q§x¡x×':Ñ':Õ;ˆ
Ü—‘×/Ñ/°ÓBˆØ�QŒ;Ü×*Ñ*¨4Ó7Ð7ØÒÜ—(‘(—,‘,×'Ñ'Ü—‘× Ñ ¤%¨£,Ô7LÈTÓ7Rð (ó ˆCô —L’L Ó,ˆEÜ˜%¤×*Ò*ØŸ™�Ü—X‘X—_‘_ UÓ3‰Fä—X‘X×;Ñ;Ø�e¤A§H¡H×$4Ñ$4óˆFð 	×Ñ˜l¨T¨M¸2Ô>ØˆrW   c                ó,   € \         P                  W4      # r  )r  r¥  r¨  s   &&rX   r©  ÚCppVecOverrides.value_expr’  s   € ô ×)Ñ)¨$Ó6Ð6rW   c                óR  € R V  R2R V  R23p\         ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       d-   \        ;QJ d    . R V 4       F  NK  	  5# ! R V 4       4      # \        V P                  ,          p\        P
                  P                  '       d   \        P
                  P                  M\        P
                  P                  p\        4       p\        P
                  P                  P                  \        P                  R7      p\        P
                  P                  P                  V P                  R7      pVP                  RV 3/ R	7       VP                  RV 3/ R	7       \        P
                  P                  V P                  4      pV^8X  d   R
V R2MRV RV R2pTP                  V^8X  d   RV R2MRV RV R24       VP                  V RV R24       VP                  R4       VP!                  4       ;_uu_ 4        VP                  RV R\        P
                  P                   R24       VP                  V  R\#        V4       R24       VP                  R\        P
                  P                   R24       VP                  RV R\        P
                  P                   R24       VP                  R\#        V4       R24       VP!                  4       ;_uu_ 4        VP                  R4       RRR4       TP                  V^8X  d   V R\#        V4       R2MV R V R!\#        V4       R24       VP                  V R"V R#\#        V4       R24       RRR4       VP                  R$4       \        P
                  P$                  P'                  V4       We3p	\)        W4       F.  w  r«\        P
                  P                  P+                  W«4       K0  	  We3#   + '       g   i     Ló; i  + '       g   i     L ; i)%rR  rS  rT  c              3   óx   "  € T F0  p\         P                  P                  P                  V4      R Jx € K2  	  R # 5ir  rV  rX  s   & rX   r  Ú(CppVecOverrides.frexp.<locals>.<genexpr>›  r[  r\  FTc              3   ót   "  € T F.  p\         P                  P                  P                  V4      x € K0  	  R # 5ir  rV  rX  s   & rX   r  rH  œ  r^  r_  ©r‡   rj  )r*  úat::vec::Vectorized<rŽ   úat::vec::VectorizedN<r¡   zat::vec::Vectorized<int32_t> rÁ   zat::vec::VectorizedN<int32_t, ú> r�   rÚ  ú__at_align__ std::array<ú	> tmpbuf;ú.store(tmpbuf.data(), rÞ   z!__at_align__ std::array<int32_t, z> tmpbuf_exponent;z> tmpbuf_mantissa;rá   râ   z@tmpbuf_mantissa[i] = std::frexp(tmpbuf[i], &tmpbuf_exponent[i]);Nz? = at::vec::Vectorized<int32_t>::loadu(tmpbuf_exponent.data(), z! = at::vec::VectorizedN<int32_t, z!>::loadu(tmpbuf_exponent.data(), rÀ   z ::loadu(tmpbuf_mantissa.data(), z();)r#  r¨   rH   r‡   r5   re  Ú	tail_sizerÀ  r7   r®  ra  rƒ   r”  r±  Ú_get_num_vectorsrÉ   rË   rE   r°  rb  rc  rd  )rµ  re  rˆ   rI  r¹   rf  rg  Ún_vecÚ
mantissa_trh  rY  ri  s   &           rX   rj  ÚCppVecOverrides.frexp˜  s‡  € à˜a˜S Ð%¨°¨s°$Ð'7Ð7ˆ
ß‹3ÑWÉJÓW�3�3Š3ÑWÉJÓW×WÒWß”5ÑUÉ*ÓU—5ÐU�5ÑUÉ*ÓUÓUÐUä˜aŸg™gÕ&ˆÜ%&§X¡X×%7×%7Ð%7Œq�x‰x×!Ò!¼Q¿X¹X×=SÑ=SˆÜ‹~ˆÜ—8‘8—<‘<×&Ñ&¬U¯[©[Ð&Ó9ˆÜ—8‘8—<‘<×&Ñ&¨Q¯W©WÐ&Ó5ˆØ×Ñ ¨!¨°bÐÔ9Ø×Ñ ¨!¨°bÐÔ9Ü—‘×)Ñ)¨!¯'©'Ó2ˆð ˜Œzð # 6 (¨!Ñ,à(¨¨°°5°'¸Ð;ð 	ð
 	�‰à˜Œzð ,¨H¨:°QÑ7à1°%°¸¸8¸*ÀAÐFô	
ð
 	�‰˜*˜ Q x j°Ð2Ô3Ø�‰�wÔØ�[‰[�]�]Ø�N‰NØ*¨6¨(°"´Q·X±X×5KÑ5KÐ4LÈIÐVôð �N‰N˜a˜SÐ 6´{À4Ó7HÐ6IÈÐLÔMØ�N‰NØ3´A·H±H×4JÑ4JÐ3KÐK]Ð^ôð �N‰NØ*¨6¨(°"´Q·X±X×5KÑ5KÐ4LÐL^Ð_ôð �N‰NÐ1´+¸dÓ2CÐ1DÀFÐKÔLØ—‘—•Ø—‘ØVô÷ ð �N‰Nà˜A”:ð �*Ð[Ô\gÐhlÓ\mÐ[nÐnpÑqà �zÐ!BÀ5À'ÐIjÔkvÐw{Ók|Ðj}Ð}ð  Aôð
 �N‰NØ�*˜C 
˜|Ð+KÌKÐX\ÓL]ÐK^Ð^`Ðaô÷+ ð0 	�‰�uÔÜ	�‰×Ñ×Ñ Ô%ØÐ'ˆÜ"% jÖ";ÑˆIÜ�H‰H�L‰L×Ñ˜YÖ0ñ #<àÐ!Ð!÷# —ú÷ �]ús&   È"C$PÌP	ÌA&PÐPÐPÐP&	c                ó   a€ V3R  lpV# )c                  óð  <€ V'       d   \        R 4      h\        P                  p\        V\        4      '       g   \        R4      h\        4       pVP                  R4       V ^ ,          P                  pVP                  V4      pVP                  '       d   VP                  MVP                  p. p\        V,          pSP                  R9   p	V	'       d   RMTp
SP                  R8X  d   \        V R ,          ,          MT
p
VP                  4       ;_uu_ 4        \        V 4       FÄ  w  r¼\        V\        4      '       d™   VP                   '       g   \        R4      hVP                  V8w  d   \        R4      hVP                  RV RVP                   R	V R
24       VP                  V RV R\#        V4       R24       VP%                  RV R24       K³  VP%                  V4       KÆ  	  VP                  RV
 RVP                   R24       S! V!  pVP                  R\#        V4       R24       VP                  4       ;_uu_ 4        VP                  RV R
24       RRR4       R\#        V4       2pV	'       d   RV RV R2pMV^8X  d   RV
 R2pM	RV
 RV R2pVP                  RV RV R24       RRR4       VP                  R4       V#   + '       g   i     Lz; i  + '       g   i     L6; i)!zexpected not kwargsz)expected isinstance(kernel, CppVecKernel)rÚ  ro   rÁ  zexpected arg.is_veczexpected arg.dtype == vec_dtyperN  r¡   z> tmpbufrÁ   z.store(tmpbufz	.data(), rÞ   Útmpbufz[i]z> tmpbuf_out;rá   râ   ztmpbuf_out[i] = Nztmpbuf_out.data(), úat::vec::VecMask<Ú,z>::fromrK  z>::loaduz at::vec::VectorizedN<r­  r’  râ  )r(  r+  r  éþÿÿÿ)r…   r5   re  r§   r(  r7   rÉ   r‡   rR  rQ  rÀ  rH   r  rË   rÌ   rG   r«   rE   r'  )r™  r*  re  r¹   Ú	vec_dtyperS  rI  Úscalar_argsrˆ   Úoutput_maskÚoctypeÚargidxr�  ÚresÚ	load_argsÚload_fnr0  s   *,              €rX   rd  Ú)CppVecOverrides._scalarize.<locals>.innerÓ  sº  ø€ ßÜ$Ð%:Ó;Ð;Ü—X‘XˆFÜ˜f¤l×3Ò3Ü$Ð%PÓQÐQÜ“>ˆDØ�N‰N˜7Ô#Ø˜Q�Ÿ™ˆIØ×+Ñ+¨IÓ6ˆEØ'-×'7×'7Ð'7�6×#Ò#¸V×=QÑ=QˆDØˆKÜ! )Õ,ˆFØ%×.Ñ.ð 3ñ ˆK÷
  +‘V°ˆFð  ×(Ñ(Ð,>Ô>ô ˜T "�XÖ&àð ð
 —‘—•Ü#,¨T¦?‘K�FÜ! #¤~×6Ò6Ø"ŸzŸz˜zÜ"0Ð1FÓ"GÐGØŸ9™9¨	Ô1Ü"0Ð1RÓ"SÐSØŸ™Ø6°v°h¸bÀ×AUÑAUÐ@VÐV^Ð_eÐ^fÐfgÐhôð Ÿ™Ø"˜e =°°¸	Ä+ÈdÓBSÐATÐTVÐWôð $×*Ñ*¨V°F°8¸3Ð+?Ö@à#×*Ñ*¨3Ö/ñ $3ð —‘Ø.¨v¨h°b¸×9MÑ9MÐ8NÈmÐ\ôñ " ;Ñ/�Ø—‘Ð!5´kÀ$Ó6GÐ5HÈÐOÔPØ—[‘[—]•]Ø—N‘NÐ%5°c°U¸!Ð#<Ô=÷ #à1´+¸dÓ2CÐ1DÐE�	ßØ 1°&°¸¸5¸'ÀÐI‘GØ˜a”ZØ 4°V°H¸HÐE‘Gà 6°v°h¸bÀÀÀxÐP�GØ—‘ ¨¨	°°9°+¸RÐ@ÔA÷= ð> �N‰N˜4Ô ØˆK÷ #—]ú÷+ —ús,   Ä8K%ÅC9K%É
K	É AK%ËK"ËK%Ë%K5	rV   )r&  r0  rd  s   &f rX   Ú
_scalarizeÚCppVecOverrides._scalarizeÑ  s   ø€ õ8	ðt ˆrW   c                ó  € \        \        4      p\        \        4      P                  4        F[  w  r#\	        V\
        4      '       g   K  W!9  g   K%  V P                  VP                  4      pW$n        \        W\        V4      4       K]  	  R # r  )
r7  r  r�  r8  r§   r  re  r:  r  r9  )r&  Úvec_varsrÎ   r<  r1  s   &    rX   Ú_initialize_scalarizeÚ%CppVecOverrides._initialize_scalarize  s^   € äœÓ(ˆÜ ¤Ó.×4Ñ4Ö6‰LˆDÜ˜&¤,×/Ô/°DÖ4HØ—~‘~ f§o¡oÓ6�Ø $”Ü˜¤<°Ó#5Ö6ó	 7rW   rV   r  )]r  r  r  r  r  r6  r  r¡  rÑ   rV  rH  rÊ  rÍ  rÑ  rØ  rÛ  rß  rã  ræ  ré  rí  rg  rm  rp  rt  rx  r|  r€  rñ  r	  r  r  r  r  r  r%  r.  r¹  r¿  rÃ  rÇ  rÌ  rÐ  rÔ  r×  ræ  rê  r±  rô  r   r  rÃ  r   r  rÎ  rI  r1  r=  r7  r:  rp  rs  rz  rO  rE  rm  rL  r@  r4  r‡  r	  rÔ  r  r"  rŠ  rŽ  r‘  r  r—  r  r«  r½  rý  rµ  r¥  r©  rj  rk  re  ri  r
  r  rl  rm  s   @@rX   r  r  «  s—  ù‡ € Ù8õ`ðD ñó ðð ñó ðð ñó ðð ñó ðð ñ	ó ð	ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñAó ðAð ñEó ðEð ñ@ó ð@ð ñ@ó ð@ð ñAó ðAð ñAó ðAð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ ó ð ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ0ó ð0ð
 ñ9ó ð9ð ñ9ó ð9ð ñFó ðFð ñAó ðAð ñó ðð ñ	!ó ð	!ð ñ#ó ð#ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ%ó ð%ð ñ$ó ð$ð ñ!ó ð!ð ñ!ó ð!ð ñOó ðOð ñó ðð, ñó ðð ñó ðð  ñHó ðHð ñó ðð ñ6ó ð6ð" ñó ðð ñ1ó ð1ð ñ1ó ð1ð ñ=ó ð=ð ñó ðð ñ]ó ð]ð ñó ðð óó ðð< óó ðð* ñ	ó ð	ð ñRó ðRðh ñó ðð0 ñ7ó ð7ð
 ñ6"ó ð6"ðp ñ;ó ð;ðz ñ7ó ÷7ð 7rW   r  Úcppvecc                   ó@   a € ] tR tRt o ]R 4       t]R 4       tRtV tR# )ÚCppTile2DOverridesi  c                óÈ   € \        \        P                  \        4      '       g   \	        R 4      h\        P                  P                  V 4      p \        P                  W4      # )z.expected isinstance(V.kernel, CppTile2DKernel))r§   r5   re  ÚCppTile2DKernelr…   Útransform_indexingr  r¥  r¨  s   &&rX   r¥  ÚCppTile2DOverrides.index_expr  sD   € äœ!Ÿ(™(¤O×4Ò4Ü Ð!QÓRÐRÜ�x‰x×*Ñ*¨4Ó0ˆÜ×)Ñ)¨$Ó6Ð6rW   c                ó,   € \         P                  W4      # r  )rm  r¥  r¨  s   &&rX   r©  ÚCppTile2DOverrides.value_expr%  s   € ô "×,Ñ,¨TÓ9Ð9rW   rV   N)	r  r  r  r  r  r¥  r©  r
  r  r  s   @rX   rm  rm    s+   ø‡ € Øñ7ó ð7ð ñ:ó ö:rW   rm  c                   óø  a a€ ] tR tRt oRt]t]tRt	Rt
V 3R lt]]3R ltR tR2V3R lR	 llt]P$                  R
 4       tR3V3R lR lltV3R lR ltV3R lR ltV3R lR ltR tV3R lR ltV3R lR ltR tR2R ltR tV3R lR ltR2V3R lR lltR t R4R  lt!R5R! lt"R" t#R# t$R$ t%R% t&R& t'R' t(])V3R( lR) l4       t*R* t+]P$                  R+ 4       t,R, t-R5R- lt.R. t/R6V3R/ lR0 llt0R1t1Vt2V ;t3# )7Ú	CppKerneli,  a  
Base class for C++ kernel code generation in PyTorch Inductor.
This class is responsible for generating C++ code from the intermediate representation.

Args:
    args: Kernel arguments used for code generation
    num_threads: Number of threads for parallel execution
rÝ   rÁ   c                óÔ  <€ \         SV `  V4       / V n        . V n        R V n        . V n        . V n        R V n        \        4       V n	        . V n
        \        4       V n        \        4       V n        \        4       V n        \        4       V n        \        4       V n        RV n        \        4       V n        \        4       V n        \'        V P(                  V P*                  RR7      V n        \'        V P(                  V P*                  RR7      V n        \'        V P(                  V P*                  RR7      V n        \        4       V n        \        4       V n        W n        / V n        . V n        R # )NFÚtmp_acc)Úname_prefixÚwelford_helperÚcascade_helper)r/  r0  Úactive_rangesÚinner_itervarsÚcall_rangesrW  r<  Úreduction_depthr=   Úreduction_prefixÚreduction_prefix_generatorsÚreduction_suffixÚparallel_reduction_prefixÚparallel_reduction_suffixÚlocal_reduction_initÚlocal_reduction_storesÚis_reductionÚnon_parallel_reduction_prefixÚnon_parallel_reduction_suffixr8   Únewvar_prefixÚsuffixÚreduction_cseÚwelford_helper_cseÚcascade_helper_cseÚpreloadsÚ
poststoresÚnum_threadsÚreduction_omp_decÚreduction_var_names)r1  r™  r�  r4  s   &&&€rX   r0  ÚCppKernel.__init__;  s5  ø€ Ü‰Ñ˜Ôð HJˆÔð 35ˆÔØ:>ˆÔØ(*ˆŒØ,.ˆŒØ#ˆÔÜ .Ó 0ˆÔð <>ˆÔ(Ü .Ó 0ˆÔÜ)7Ó)9ˆÔ&Ü)7Ó)9ˆÔ&Ü$2Ó$4ˆÔ!Ü&4Ó&6ˆÔ#Ø!ˆÔÜ-;Ó-=ˆÔ*Ü-;Ó-=ˆÔ*Ü  ×!3Ñ!3°T·[±[ÈiÔXˆÔÜ"%Ø×Ñ §¡Ð9Iô#
ˆÔô #&Ø×Ñ §¡Ð9Iô#
ˆÔô 'Ó(ˆŒÜ(Ó*ˆŒØ&ÔØ=?ˆÔØ.0ˆÖ rW   c                óX  € \         P                  P                  '       d.   V P                  '       g   V P                  P	                  R 4       V R2p\         P                  P                  '       d   RM	\        4       pV R2p	V P                  P	                  V RV RV! W44       R24       V P                  P                  \        VVVVVV4      4       V P                  P	                  V	 RV R24       V P                  P                  RV R2R	R
V RV! W1W”R7       R2R.4       R# )z(int max_threads = omp_get_max_threads();Ú_localÚmax_threadsz	_arr[tid]r�   rÀ   rÁ   zfor (int tid = 0; tid < z; tid++)rã   rä   ©r¯   r~   N)r   rù  Údynamic_threadsr‚  rÉ   r,   r„  rb  rê   r…  rƒ  ræ   )
r1  r·   rÚ   r†   r‡   Úreduction_combine_fnÚreduction_init_fnÚ	acc_localr�  Úacc_local_in_arrays
   &&&&&&&   rX   Ú_gen_parallel_reduction_buffersÚ)CppKernel._gen_parallel_reduction_buffersd  s?  € ô �:‰:×%×%Ð%¨d×.L×.LÐ.LØ×*Ñ*×4Ñ4Ø:ôð �e˜6�Nˆ	ä#ŸZ™Z×7×7Ð7‰MÔ=QÓ=Sð 	ð !$˜u IÐ.ÐØ×!Ñ!×+Ñ+Øˆj˜˜)˜ CÑ(9¸.Ó(PÐ'QÐQRÐSô	
ð 	×&Ñ&×-Ñ-Ü"ØØØØØØ!óô		
ð 	×#Ñ#×-Ñ-Ð1CÐ0DÀCÈ	À{ÐRSÐ.TÔUØ×&Ñ&×1Ñ1à*¨;¨-°xÐ@ØØ�s�e˜3Ñ3°NÐI[ÔmÐnÐnoÐpØð	ö	
rW   c                ó^   € V P                    F  p\        V P                  W R 24       K  	  R# )r•  N)r’  rð   Ústores)r1  Úvar_names   & rX   Ú%update_stores_with_parallel_reductionÚ/CppKernel.update_stores_with_parallel_reduction�  s'   € Ø×0Ô0ˆHÜ˜TŸ[™[¨(°jÀÐ4GÖHó 1rW   c                ó.   <€ V ^8„  d   QhRS[ R,          /# ©r’   r¹   N©r7   )r–   r  s   "€rX   r—   ÚCppKernel.__annotate__‘  s   ø€ ÷ ñ ™\¨DÕ0ñ rW   c                óô  € Ve   \        R4      h\        4       p\        P                  ! 4       ;_uu_ 4       p\	        V R4      '       dL   VP                  V P                  4       V P                  V4       VP                  VP                  4       4       VP                  V P                  4       VP                  V P                  4       VP                  V P                  4       R R R 4       \	        V R4      '       d   VP                  V P                  4       V P                  '       d9   V P                   F(  pV P                  V,          w  rE\!        WV R2WE4      pK*  	  V#   + '       g   i     L‰; i)Nzexpected code is NoneÚcodegen_inner_loopsÚ_tail)r…   r7   rÇ   rÈ   rª   rb  rŽ  r©  rÊ   rË   Úloadsr°  r   r�  r|  r{  r×   )r1  r¹   rÓ   rZ  ÚstartÚends   &&    rX   Úgen_bodyÚCppKernel.gen_body‘  s  € ØÒÜ Ð!8Ó9Ð9Ü‹~ˆÜ×!Ò!×#Ô# uÜ�tÐ2×3Ò3Ø—‘˜DŸM™MÔ*Ø×(Ñ(¨Ô.Ø×#Ñ# D§K¡K£MÔ2Ø�K‰K˜Ÿ
™
Ô#Ø�K‰K˜Ÿ™Ô%Ø�K‰K˜Ÿ™Ô$÷ $ô �4Ð.×/Ò/Ø�K‰K˜Ÿ™Ô(à××ÐØ×*Ô*�Ø!×/Ñ/°Õ4‘
�Ü1°$À¸uÀE¸]ÈEÓW’ñ +ð ˆ÷ $×#ús   ·B/E'Å'E7	c              #  óJ  "  € V P                   pV'       do   \        P                  ! W4      p\        V\        4      '       dC   VP
                  p\        V\        4      '       g   \        R4      h\        P                  Vn
        Wn          Vx € W n         R#   Y n         i ; i5i)z>Context manager to add an additional mask to loads and stores.z)expected isinstance(mask, CppCSEVariable)N)Ú
_load_maskr3   r€  r§   r4   rù   rG   r…   rƒ   ro   r‡   )r1  r¯  Úpriors   && rX   rµ  ÚCppKernel.masked¦  sv   é € ð —‘ˆßÜ—8’8˜DÓ(ˆDÜ˜$¤×)Ò)Ø—z‘z�Ü! $¬×7Ò7Ü(Ð)TÓUÐUô #ŸZ™Z�”
àŒð	$ØŠJà#ŽOø˜e�Oüs   ‚B	B#ÂB ÂB#ÂB Â B#c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r’   r“   rï  )r–   r  s   "€rX   r—   r§  º  s   ø€ ÷ ñ Ù—Z‘ZñrW   c                ód   € V P                   V,          pWUV,          V,           /p\        W4      pV# r  )r<  r1   )r1  r“   ÚscaleÚitervar_idxrJ  r¬   ÚreplacementÚ	new_indexs   &&&&&   rX   Úscale_index_with_offsetÚ!CppKernel.scale_index_with_offsetº  s3   € ð �m‰m˜KÕ(ˆØ %�K¨&Õ0Ð1ˆÜ˜uÓ2ˆ	ØÐrW   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# ©r’   r“   r¾   )r”   r•   rp   )r–   r  s   "€rX   r—   r§  Â  s   ø€ ÷ 2ñ 2¡%§*¡*ð 2±ñ 2rW   c                ó6   € \        V P                  V4      4      # )zŽ
Convert an index expr to a string that can be used in cpp code.
e.g. a sympy expression "s2" may actually appear as "ks1" in the cpp kernel.
)rD   r£  ©r1  r“   s   &&rX   Úindex_to_strÚCppKernel.index_to_strÂ  s   € ô
 �T×)Ñ)¨%Ó0Ó1Ð1rW   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r’   r“   Úitervar©r”   r•   ÚSymbol)r–   r  s   "€rX   r—   r§  É  s#   ø€ ÷ 	
ñ 	
©u¯z©zð 	
ÁEÇLÁLñ 	
rW   c                ó¦   a a€ \         ;QJ d*    VV 3R lVP                   4       F  '       g   K   R# 	  R# ! VV 3R lVP                   4       4      # )zM
Check if an index has free symbol CppCSEVariable that depends on `itervar`.
c              3   óX  <"  € T FŸ  pVP                   SP                  P                  9   g   K*  \        SP                  P                  VP                   ,          \        4      '       g   Kg  SP                  P                  VP                   ,          P                  S4      x € K¡  	  R # 5ir  )rÎ   r®  Úvarname_mapr§   rG   Ú
depends_on)r  ÚsrÄ  r1  s   & €€rX   r  Ú6CppKernel.index_indirect_depends_on.<locals>.<genexpr>Í  su   øé € ð 
á'�Ø�v‰v˜Ÿ™×-Ñ-Ñ-ô =ô ˜4Ÿ8™8×/Ñ/°·±Õ7¼×Hô =ˆD�H‰H× Ñ  §¡Õ(×3Ñ3°G×<Ð<Û'ùs   ƒ'B*¯8B*Á,>B*TF)rf   Úfree_symbols©r1  r“   rÄ  s   f&frX   Úindex_indirect_depends_onÚ#CppKernel.index_indirect_depends_onÉ  sL   ù€ ÷ ‹sõ 
à×'Ò'ó
�sŒsð 	
Šsð 	
ˆsõ 
à×'Ò'ó
ó 
ð 	
rW   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÃ  rÅ  )r–   r  s   "€rX   r—   r§  Ô  s#   ø€ ÷ 
ñ 
¡e§j¡jð 
¹5¿<¹<ñ 
rW   c                óP   € W!P                   9   ;'       g    V P                  W4      # r  )rÍ  rÏ  rÎ  s   &&&rX   Úindex_depends_onÚCppKernel.index_depends_onÔ  s-   € Ø×,Ñ,Ñ,÷ 
ð 
°×0NÑ0NØó1
ð 	
rW   c                óT   € \        \        V P                  V P                  4      4      # r  )r  rc  r<  rW  r7  s   &rX   Ú
var_rangesÚCppKernel.var_rangesÙ  s   € Ü”C˜Ÿ™ t§{¡{Ó3Ó4Ð4rW   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[RS[/# )r’   r·  rI  ÚlowerÚupper)r”   r•   ro   )r–   r  s   "€rX   r—   r§  Ü  s;   ø€ ÷ (:ñ (:á�j‰jð(:ñ �j‰jð(:ñ ð	(:ñ
 ñ(:rW   c                óD  € V'       g   V'       g   R # \        V\        P                  4      pV'       dL   \        P                  ! V\
        P                  4      P                  p\        P                  P                  pEM\        V P                  V4      4      pV P                  P                  V4      eK   \        P                  ! V\
        P                  4      P                  p\        P                  P                  pMŠ\        P                  P                  p	 V P                  \        P                  n
        \        P                  ! V\
        P                  4      P                  pV	\        P                  n
        V P                  pV'       d/   \        P                  P!                  V P                  V4      4      MR p
T P#                  Yc'       d   RMR W P$                  4      pV P                  P'                  W{RR7       R #   T	\        P                  n
        i ; i)NÚ0F)Ú
assignment)r   r   ÚTMPr3   r¥  rƒ   r•  rù   r5   re  r°  rD   r£  r®  rW  r«  ÚsexprÚindirect_assertr±  r¯  )r1  r·  rI  rÙ  rÚ  Úindirectr¸  rì   r¤  Úprior_computeÚsize_strrÏ   s   &&&&&       rX   Úcheck_boundsÚCppKernel.check_boundsÜ  sY  € ÷ ŸÙä& t¬T¯X©XÓ6ˆßä—^’^ D¬%¯+©+Ó6×<Ñ<ˆFÜ—X‘X×%Ñ%ŠFô ˜D×0Ñ0°Ó6Ó7ˆGØ�x‰x×Ñ Ó(Ò4ÜŸš¨¬e¯k©kÓ:×@Ñ@�ÜŸ™×)Ñ)‘ä !§¡× 0Ñ 0�ð5Ø'+§z¡z”A—H‘HÔ$Ü Ÿ^š^¨D´%·+±+Ó>×DÑD�Fà'4”A—H‘HÔ$ØŸ™�çAF”1—8‘8—>‘> $×"6Ñ"6°tÓ"<Ô=ÈDˆà×#Ñ#Øž5‘C d¨H·o±oó
ˆð 	�‰×Ñ˜&°5ÐÖ9øð (5”A—H‘HÕ$ús   ÄAH ÈHc                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# ©r’   rÎ   r“   ©rp   r”   r•   )r–   r  s   "€rX   r—   r§  	  s   ø€ ÷ ñ ™ð ¡U§Z¡Zñ rW   c                ó4  € V P                   P                  V4      pV P                  V4      pV R \        V4       R2pV P                  P                  V P                  V\        P                  P                  V4      R7      pVP                  RWV3/ 4       V# )Ú[Ú]rJ  r°  )r™  Úinputr£  rE   r®  r¯  r«  r5   ÚgraphÚ	get_dtyper±  )r1  rÎ   r“   r¬   rÏ   r¸  s   &&&   rX   r°  ÚCppKernel.load	  s†   € Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆØ��aœ EÓ*Ð+¨1Ð-ˆØ—‘×"Ñ" 4§:¡:¨t¼1¿7¹7×;LÑ;LÈTÓ;RÐ"ÓSˆØ×Ñ˜f t°5Ð&9¸2Ô>ØˆrW   c                ód   € \         P                  P                  ;'       g    V P                  ^8g  # ©r$  )r   rù  r˜  r�  r7  s   &rX   Ú_use_parallel_atomic_addÚ"CppKernel._use_parallel_atomic_add	  s%   € Ü�z‰z×)Ñ)×BÐB¨T×-=Ñ-=ÀÑ-BÐBrW   c                ó  € R V9  d   \        R4      hV P                  P                  V4      pV P                  V4      pVf   V R\	        V4       RV R2pM‰VR8X  du   V P                  4       '       g   V R\	        V4       RV R2pMW\        P                  P                  V4      pR\        V,           R	V R
2pRV R\	        V4       RV R2pM\        RV 24      hV P                  P                  \        W4      4       R# )Úbufúexpected "buf" in nameNrê  ú] = rÁ   Ú
atomic_addz] += zstatic_cast<rÀ  r›   zatomic_add(&z], rÞ   ústore mode=)r…   r™  Úoutputr£  rE   rò  r5   rí  rî  rH   r  r   rÉ   r;   )r1  rÎ   r“   rù   Úmoder¬   rÏ   r‡   s   &&&&&   rX   ÚstoreÚCppKernel.store	  s	  € Ø˜ÔÜ Ð!9Ó:Ð:Ø�i‰i×Ñ˜tÓ$ˆØ×$Ñ$ UÓ+ˆØŠ<Ø�U˜!œK¨Ó.Ð/¨t°E°7¸!Ð<‰DØ�\Ô!Ø×0Ñ0×2Ò2Ø˜˜a¤¨EÓ 2Ð3°5¸¸¸qÐA‘äŸ™×)Ñ)¨$Ó/�à&¤|°EÕ':Ð&;¸2¸e¸WÀAÐF�Ø% c U¨!¬K¸Ó,>Ð+?¸sÀ5À'ÈÐL‘ä%¨°D°6Ð&:Ó;Ð;Ø�‰×Ñœl¨4Ó6Ö7rW   c                óJ   € V P                   P                  R V RV R24       R# )r’  z" ? 0 : (throw std::runtime_error("z"), 0));N)r°  rÉ   )r1  ÚcondÚmsgs   &&&rX   Údevice_assert_asyncÚCppKernel.device_assert_async$	  s'   € Ø�‰×ÑØ�ˆvÐ7¸°u¸HÐEö	
rW   c                óV   <€ V ^8„  d   QhRS[ S[,          RS[RS[RS[P                  /# )r’   r·   rÚ   Úrtyper‡   )r9   rp   rƒ   r‡   )r–   r  s   "€rX   r—   r§  )	  s<   ø€ ÷ ñ á™3Õðñ ðñ ð	ñ
 �{‰{ñrW   c                ó.   aaaaa€ RR VVVVV3R lllpV# )Nc                ó2   € V ^8„  d   QhR\         R,          /# ©r’   rI  Nr  )r–   s   "rX   r—   Ú5CppKernel._gen_reduction_prefix.<locals>.__annotate__7	  s   € ÷ 	ñ 	œ˜d�
ñ 	rW   c                 óP   <€ V f   S RS RS! SS4       R2# \        SSSSV S4      # )Nr�   rÀ   rÁ   )rê   )rI  r·   rÚ   r‡   rç   r  s   &€€€€€rX   rd  Ú.CppKernel._gen_reduction_prefix.<locals>.inner7	  sH   ø€ ØŠ|Ø"˜ 1 S E¨©W°U¸EÓ-BÐ,CÀ1ÐEÐEä-ØØØØØØóð rW   r  rV   )r1  r·   rÚ   r  r‡   rç   rd  s   &fffff rX   Ú_gen_reduction_prefixÚCppKernel._gen_reduction_prefix)	  s   ü€ ÷	ö 	ð ˆrW   c                ó.   <€ V ^8„  d   QhRS[ R,          /# r  r  )r–   r  s   "€rX   r—   r§  F	  s   ø€ ÷ 7ñ 7©c°D­jñ 7rW   c                ón   € V P                    F$  pV P                  P                  V! V4      4       K&  	  R # r  )r€  r  rb  )r1  rI  Úgen_fns   && rX   Úfinalize_reduction_prefixÚ#CppKernel.finalize_reduction_prefixF	  s*   € Ø×6Ô6ˆFØ×!Ñ!×(Ñ(©°«Ö6ó 7rW   c                óx  € VR 8X  d   R# VR8X  dª   V\         P                  8X  d•   V P                  f   \        R4      h\        P
                  ! \        P                  V P                  V P                  R 4      pRp\        P                  P                  P                  \        P                  ! WE4      4      # R# )rg   Tra   Nú%expected self.call_ranges is not Noneé   F)rƒ   rm   r}  r…   rS  rT  rU  rV  r~  r5   rí  ÚsizevarsÚguard_or_falser”   ÚGt)r1  r†   r‡   Ú
use_scalarÚreduction_sizeÚ
chunk_sizes   &&&&  rX   Úneed_use_acc_helperÚCppKernel.need_use_acc_helperJ	  s˜   € ð Ð-Ô-Ùð ˜UÔ" u´·±Ô';Ø×ÑÒ'Ü$Ð%LÓMÐMÜ&×-Ò-Ü—‘˜d×.Ñ.¨t×/CÑ/CÐ/EÐFóˆNð
 ˆJä—7‘7×#Ñ#×2Ñ2´5·8²8¸NÓ3WÓXÐXÙrW   c           
     ó²  € V'       d   \        W54      MTp\        V4      pVR9  d   \        R4      hRp	\        Wy4      p
VR 8X  d   RMRpV'       d   \        V,          pM0\	        V R4      '       d   V P                  V4      M\        V,          pV RV RV	 R	V R
V R2
pVR8X  d   V# \        V
\        P                  4      '       d   V
^8:  d   RV 2# V# )rg   ra   z4expected reduction_type in ["welford_reduce", "sum"]r  ÚWelfordHelperÚCascadeSumHelperr,  Ú<r¡   rM  r’  rÞ   zstatic )rg   ra   )	r   rE   r…   rH   rª   r,  r§   r”   rX  )r1  r†   r®   Úhelper_ranger‡   r�  r  Únum_range_threadÚnum_range_thread_exprr  Ú
num_chunksÚhelper_typeÚh_typeÚhelper_init_lines   &&&&&&&       rX   Ú_acc_helper_initÚCppKernel._acc_helper_inita	  s  € ÷ 3>ŒG�LÔ.À<ð 	ô !,Ð,<Ó =ÐØÐ!:Ô:Ü Ð!WÓXÐXØˆ
ÜÐ-Ó:ˆ
ð Ð!1Ô1ñ à#ð 	÷
 Ü! %Õ(‰Fô ˜4 ×1Ò1ð ×"Ñ" 5Ô)ä! %Õ(ð ð ˆm˜1˜V˜H B z l°"°Z°LØØ$Ð%Øðð 	ð ˜UÔ"Ø#Ð#Ü�j¤%§-¡-×0Ò0°ZÀ1´_ð Ð-Ð.Ð/Ð/à#Ð#rW   c           
     óT  € \         P                  P                  '       d   R M	\        4       pV P                  P                  V P                  WWERV4      4       V P                  P                  V P                  WWEWv4      4       V'       d   TMV R2pVR8X  dK   V P                  P                  V RV RV R24       V P                  P                  V RV RV R24       R# V P                  P                  V R	V R24       V P                  P                  V R
V R24       R# )r–  NÚ_vecrg   z = welford_combine(rš   rÞ   z_local = welford_combine(z	_local, &z = cascade_sum_final(&z_local = cascade_sum_final(&)
r   rù  r˜  r,   r‡  rÉ   r(  r„  rˆ  r…  )	r1  r†   r·   r®   r!  r‡   r  r�  r³  s	   &&&&&&&  rX   Ú_use_acc_helperÚCppKernel._use_acc_helperŽ	  s:  € ô $ŸZ™Z×7×7Ð7‰MÔ=QÓ=Sð 	ð 	×*Ñ*×4Ñ4Ø×!Ñ!Ø¨LÀÀzóô	
ð
 	×!Ñ!×+Ñ+Ø×!Ñ!Ø¨LÀóô	
÷
 #‘¨3¨%¨t¨ˆØÐ-Ô-Ø×.Ñ.×8Ñ8Ø�(Ð-¨f¨X°S¸¸ÀBÐGôð ×'Ñ'×1Ñ1Ø�(Ð3°F°8¸9ÀZÀLÐPRÐSöð ×.Ñ.×8Ñ8Ø�(Ð0°°¸BÐ?ôð ×'Ñ'×1Ñ1Ø�(Ð6°z°lÀ"ÐEörW   c                ó.  € VR9   pRpV'       d   \        V\        4      '       d   Vw  rFVe   \        \        V4      pW#WG33pMW#V3pW€P                  P
                  9   d   V P                  P
                  V,          # V P                  P                  V P                  RV 2RR7      p	V P                  P                  V	 4       RV n
        V'       d   TMTp
\        W:4      pV P                  P                  V P                  W›W:\        4      4       V P                  W1R4      '       dã   \         P"                  ! \$        P&                  V P(                  V P*                  R 4      pVR8X  d-   V P,                  P                  V P.                  RV 2RR7      pM+V P0                  P                  V P.                  RV 2RR7      pRV 2pV P3                  VV	VVVRR7       V P4                  P7                  V	 R	\9        W9WN4       R
24       MÑVe   TpMžV P*                  f   \;        R4      hV P<                  V P*                  ,          p\?        V P*                  ^,           \A        V P<                  4      4       F4  pWðP(                  V,          ,          V P<                  V,          ,           pK6  	  V P4                  P7                  V	 R	\9        W9WOR7       R
24       V PC                  W›W:4       \E        W94      pVV P                  P
                  V&   V# )re   Nú
reduction F©ÚwriteTrg   Úscalar_©r  rÀ   rÁ   ú)expected self.reduction_depth is not None)r“   ©re   rd   )#r§   r¨   r   r9   r‹  Úreduction_cacher¯  r«  r’  r'  r†  r�   r€  r  rŒ   r  rS  rT  rU  rV  rW  r~  rŒ  r°  r�  r,  r   rÉ   rµ   r…   r<  rQ  rÛ   r�  r©   )r1  r‡   r¯   r†   rù   Úargmax_or_argminÚlogical_indexÚlogical_index_keyÚreduction_keyr·   Ú
init_dtyperÚ   r  r®   Úscalar_helper_valr“   rï   r³  s   &&&&&             rX   Ú	reductionÚCppKernel.reduction®	  sï  € Ø)Ð-AÑAÐØˆß¤
¨5´%× 8Ò 8Ø#(Ñ ˆEàÒ$Ü $¤[°-Ó @ÐØ%¸Ð7QÐQ‰Mà%°uÐ<ˆMØ×.Ñ.×>Ñ>Ô>Ø×%Ñ%×5Ñ5°mÕDÐDà× Ñ ×)Ñ)Ø�J‰J˜* ] OÐ4¸Eð *ó 
ˆð 	× Ñ ×'Ñ'¨3¨%Ô1Ø ˆÔß"2‘Y¸ˆ
Ü% nÓAˆØ×(Ñ(×/Ñ/Ø×&Ñ&Ø˜~¼>óô	
ð ×#Ñ# N¸4×@Ò@ä&×-Ò-Ü—‘˜dŸk™k¨$×*>Ñ*>Ð*@ÐAóˆNð Ð!1Ô1Ø!×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ó ‘
ð "×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ó �
ð #*¨*¨Ð 6ÐØ× Ñ ØØØ!ØØØð !ô ð �K‰K×!Ñ!Ø�%�sÔ,¨^À%Ó[Ð\Ð\]Ð^õð Ò(Ø%‘à×'Ñ'Ò/Ü(Ð)TÓUÐUØŸ™ d×&:Ñ&:Õ;�Ü˜t×3Ñ3°aÕ7¼¸T¿]¹]Ó9KÖL�AØ!§K¡K°¥NÕ2°T·]±]À1Õ5EÕE’Eñ Mà�K‰K×!Ñ!Ø�%�sÔ,¨^À%ÔUÐVÐVWÐXôð 	×,Ñ,¨S¸NÔWÜ" >Ó7ˆØ<Bˆ×Ñ×*Ñ*¨=Ñ9ØˆrW   c                óÊ   € V P                  V4      pV P                  P                  V4      pV P                  P	                  \        W R \        V4       RV R24      4       R# )rê  r÷  rÁ   N)r£  r™  rú  r�  rÉ   r;   rE   )r1  rÎ   r“   rù   r¬   s   &&&& rX   Ústore_reductionÚCppKernel.store_reductionö	  s[   € Ø×$Ñ$ UÓ+ˆØ�i‰i×Ñ˜tÓ$ˆØ×Ñ×'Ñ'Ü˜  a¬°EÓ(:Ð';¸4À¸wÀaÐHÓIö	
rW   c           	     óð  € V P                   '       d€   V P                   \        V4      \        V4      ,           8w  d0   \        V P                    R \        V4       R\        V4       24      hV P                  \	        V4      8w  d   \        R4      hMª\        V4      \        V4      ,           V n         V P                    Uu. uF  q0P                  V4      NK  	  upV n        \        \	        V P                  4      4       Uu. uF  p\        \        P                  V4      NK  	  upV n        \	        V4      V n        V P                  RV P                   V P                  V P                  R 3# u upi u upi )rd  ró   z-expected self.reduction_depth == len(lengths)N)r}  r¨   r…   r~  rÛ   r£  rW  rQ  r/   r   ÚXBLOCKr<  )r1  ÚlengthsÚreduction_lengthsrµ  rv  s   &&&  rX   Ú
set_rangesÚCppKernel.set_rangesý	  sK  € Ø××ÐØ×Ñ¤5¨£>´EÐ:KÓ4LÕ#LÔLÜ$Ø×'Ñ'Ð(¨¬U°7«^Ð,<¸CÄÐFWÓ@XÐ?YÐZóð ð ×#Ñ#¤s¨7£|Ô3Ü$Ð%TÓUÐUð 4ô  % W›~´Ð6GÓ0HÕHˆDÔØ<@×<LÒ<LÓMÑ<L°q×/Ñ/°Ö2Ñ<LÑMˆDŒKô œs 4§;¡;Ó/Ô0óá0�Aô /¬t¯{©{¸AÖ>Ù0ñˆDŒMô
 $' w£<ˆDÔ à�M‰MÐ0˜D×0Ñ0Ð1Ø�M‰M˜$×.Ñ.Ð0Ð1ð
ð 	
ùò Nùòs   Ã E.Ä#E3c                ó°   € V P                   f   \        R4      h\        V P                   4      p\        P                  P
                  P                  V4      # )Nr  )r}  r…   r0   r5   rí  r  Úoptimization_hint)r1  r·  s   & rX   Ú	size_hintÚCppKernel.size_hint
  sE   € Ø×ÑÒ#Ü Ð!HÓIÐIÜ˜T×-Ñ-Ó.ˆÜ�w‰w×Ñ×1Ñ1°$Ó7Ð7rW   c                óì  aaaaaaaaaa€ \        V \        4      '       g   \        R 4      h\        4       oV P                  f   \        R4      h\        VP
                  \        4      '       d,   VP
                  P                  VP                  4       S4      oM V P                  VP                  4       S4      oVP                  RJ;'       d(    VP                  SP                  ,          P                  o\        P                  ! 4       ;_uu_ 4       pSP                  '       d=   S'       d   SP                  4        MSP!                  S4       VP#                  S4       M<S^8”  d6   SP%                  4       '       d    VP'                  SP)                  4       4       R VVV3R lloRR loRR VVVVVV3R llloRR VV3R	 llloRR
 VV3R llloVP'                  SP)                  4       4       \        VP
                  \        4      '       Ed5   \        \*        P,                  \.        4      '       Ed   \*        P,                  P0                  '       dð   \*        P,                  P0                  pVP3                  4        FÁ  p\5        VP7                  4       P8                   Uu. uF  pV P;                  V4      NK  	  up4      p\<        VP7                  4       P>                  ,          p	RV	 R\A        V4       R2p
VPC                  4       pSPE                  RV	 RV RV
 R24       SPE                  V	 RV RV R24       KÃ  	  S! V4       RRR4       R# u upi   + '       g   i     R# ; i)z)expected isinstance(self, CppKernelProxy)Nr  c                ó$   € V ^8„  d   QhR\         /# )r’   Ú
_loop_nest©r@  )r–   s   "rX   r—   Ú2CppKernel.codegen_loops_impl.<locals>.__annotate__8
  s   € ÷ .ñ .¤xñ .rW   c                 ó
  <a € V V3R  lpS P                  4       p\        V\        4      '       d   VP                   F  o S! S 4       K  	  R# \        V\        4      '       g   \        R4      hS P                  e   V! 4       '       d   VP                  4        \        P                  ! 4       ;_uu_ 4       pVP                  SP                  4       4       VP                  S4       RRR4       R#   + '       g   i     R# ; i)c                  ó¸   <€ SP                   '       g   \        R 4      hSP                   SP                  ,          p V P                  ;'       d    V P                  # )úexpected _loop_nest.loops)rL  r…   r   r†  Úparallel)ÚrootrN  Ú	par_depths    €€rX   Úis_parallel_reductionÚOCppKernel.codegen_loops_impl.<locals>.gen_kernel.<locals>.is_parallel_reduction9
  sI   ø€ Ø%×+×+Ð+Ü,Ð-HÓIÐIØ%×+Ñ+¨I×,AÑ,AÕB�DØ×,Ñ,×>Ð>°·±Ð>rW   z+expected isinstance(kernel, CppKernelProxy)N)Ú
get_kernelr§   rb  rd  ÚCppKernelProxyr…   rL  r¢  rÇ   rÈ   rÊ   rË   r®  )rN  rW  re  rÓ   r¹   Úgen_loop_nestrV  s   f   €€€rX   Ú
gen_kernelÚ0CppKernel.codegen_loops_impl.<locals>.gen_kernel8
  s¾   ù€ ö?ð $×.Ñ.Ó0�Ü˜fÔ&:×;Ò;Ø&,§l¤l˜
Ù% jÖ1ó '3ô & f¬n×=Ò=Ü,ØIóð ð "×'Ñ'Ò3Ñ8M×8OÒ8OØ×DÑDÔFÜ#×-Ò-×/Ô/°5Ø×+Ñ+¨D¯K©K«MÔ:ØŸ™¨Ô-÷ 0×/×/Ò/ús   Â61C1Ã1D	c                 ó  € V'       d?   V P                   pV'       d   V P                  V,           pV# V P                  V,           pV# V P                  pV'       d   W@P                  ,           pV# W@P
                  ,           pV# r  )r�  rƒ  rˆ  r  r‚  r‡  )re  rT  Ú	is_suffixrŠ  Úprefixs   &&&  rX   Úget_reduction_prefix_suffixÚACppKernel.codegen_loops_impl.<locals>.get_reduction_prefix_suffixN
  sy   € ßØ#×4Ñ4�FßØ!'×!AÑ!AÀFÕ!J˜ð "�Mð "(×!EÑ!EÈÕ!N˜Ø!�Mà#×4Ñ4�FßØ!'×*JÑ*JÕ!J˜ð "�Mð "(×*NÑ*NÕ!N˜Ø!�MrW   c                ó0   € V ^8„  d   QhR\         R\        /# ©r’   rN  Údepth©r@  rk   )r–   s   "rX   r—   rP  ^
  s   € ÷ #ñ #Ü$ð#Ü-0ñ#rW   c           
      óê  <€ V P                  4       pV P                  '       g   \        R 4      hV P                  V,          p\        P                  ! 4       ;_uu_ 4       pVP
                  '       dU   V'       gM   S	! W4P                  RR7      pV'       d    VP                  SP                  4       4       SP                  V4       S
'       dn   VP                  '       d\   SP                  S4       VP                  '       d9   VP                  '       g   \        R4      hSP                  VP                  4       S! W4       S
'       dP   VP                  '       d>   VP                  '       d   SP                  VP                  4       SP                  4        VP
                  '       d,   V'       g$   SP                  S	! W4P                  RR7      4       RRR4       R#   + '       g   i     R# ; i)rS  F)r_  z&expected kernel.local_reduction_storesTN)rY  rL  r…   rÇ   rÈ   r†  rT  rÊ   rË   rb  r„  r…  Úclose)rN  re  Úin_reductionre  ÚloopÚstack_outerr  r¹   Úgen_loop_atra  Úis_reduction_loopÚthreadsÚworksharings   &&&    €€€€€€rX   Úgen_loop_with_reductionÚ=CppKernel.codegen_loops_impl.<locals>.gen_loop_with_reduction^
  s`  ø€ ð $×.Ñ.Ó0�Ø!×'×'Ð'Ü(Ð)DÓEÐEØ!×'Ñ'¨Õ.�Ü×)Ò)×+Ô+¨{Ø×(×(Ð(·Ù+FØ"§M¡M¸Uô,Ð(÷ ,Ø'×5Ñ5°d·k±k³mÔDØŸ™Ð$4Ô5ß(¨T¯]¯]¨]Ø#×,Ñ,¨WÔ5Ø!×6×6Ð6Ø#)×#@×#@Ð#@Ü&4Ø$Ló'"ð !"ð !ŸK™K¨×(CÑ(CÔDá 
Ô2ç(¨T¯]¯]¨]Ø!×8×8Ð8Ø ŸK™K¨×(EÑ(EÔFØ#×)Ñ)Ô+Ø×(×(Ð(·ØŸ™Ù7Ø &¯©Àôô÷1 ,×+×+Ò+úsD   ÁG!Á:AG!ÃG!Ã "G!ÄG!ÄAG!ÅG!Å/<G!Æ,G!Æ4#G!Ç!G2	c                ó0   € V ^8„  d   QhR\         R\        /# rd  rf  )r–   s   "rX   r—   rP  ƒ
  s   € ÷ 
Lñ 
L¬ð 
L¼ñ 
LrW   c                 ó®  <€ \         P                  ! 4       ;_uu_ 4       pV P                  '       g   \        R 4      hV P                  V,          pVP	                  4       pVf    RRR4       R# SP                  V4       VP                  SP                  4       4       S! W^,           VP                  4       RRR4       R#   + '       g   i     R# ; i)rS  N)	rÇ   rÈ   rL  r…   Úlinesræ   rÊ   rË   r†  )rN  re  rÓ   rj  Ú
loop_linesr¹   r[  s   &&   €€rX   rl  Ú1CppKernel.codegen_loops_impl.<locals>.gen_loop_atƒ
  s�   ø€ Ü×)Ò)×+Ô+¨uØ%×+×+Ð+Ü,Ð-HÓIÐIØ%×+Ñ+¨EÕ2�DØ!%§¡£�JØ!Ò)Ø÷ ,Ñ+ð —O‘O JÔ/Ø×'Ñ'¨¯©«Ô6Ù! *°a­i¸×9JÑ9JÔK÷ ,×+×+Ò+ús   ŸACÁ/A
CÃC	c                ó<   € V ^8„  d   QhR\         R\        R\        /# )r’   rN  re  ri  r?  )r–   s   "rX   r—   rP  �
  s+   € ÷ Mñ MÜ$ðMäðMô #ñMrW   c                 ó~   <€ V P                   e   V\        V P                   4      8X  d   S! V 4       R # S! WV4       R # r  )rL  rÛ   )rN  re  ri  r\  rp  s   &&&€€rX   r[  Ú3CppKernel.codegen_loops_impl.<locals>.gen_loop_nest�
  s5   ø€ ð
 ×#Ñ#Ò+¨u¼¸J×<LÑ<LÓ8MÔ/MÙ˜zÖ*á+¨J¸|ÖLrW   zstd::make_unique<z []>(r›   zstd::unique_ptr<z	 []> buf_rÀ   rÁ   z* z = buf_z.get();)FF)r   F)r   )#r§   rZ  r…   r,   r}  re  rb  Údecide_parallel_depthÚmax_parallel_depthrL  r   r†  rÇ   rÈ   rÿ   rh  rT  Úmark_parallelÚsinglerÊ   rË   r5   Úlocal_buffer_contextrK   Úlocal_buffersÚvaluesr0   Ú
get_layoutrI  r£  rH   r‡   rD   Úget_namerb  )r1  rR  r¹   ro  rÓ   r  Úlocal_bufferÚsize_valÚlocal_buf_sizeÚlocal_buf_dtypeÚallocateÚlocal_buffer_namer\  rl  r[  rp  ra  rm  rV  rn  s   &&ff        @@@@@@@@rX   Úcodegen_loops_implÚCppKernel.codegen_loops_impl
  sì  ÿù€ Ü˜$¤×/Ò/Ü Ð!LÓMÐMÜ&Ó(ˆØ×ÑÒ#Ü Ð!HÓIÐIÜ�i×&Ñ&Ô(<×=Ò=Ø!×(Ñ(×>Ñ>Ø×,Ñ,Ó.°ó‰Ið ×2Ñ2Ø×,Ñ,Ó.°óˆIð
 �O‰O 4Ð'÷ Dð DØ—‘ 	× 5Ñ 5Õ6×CÑCð 	ô ×!Ò!×#Ô# uØ×'×'Ð'ß$à×%Ñ%Õ'à×(Ñ(¨Ô1Ø×'Ñ'¨	Õ2Ø˜1”Ø×%Ñ%×'Ò'Ø×'Ñ'¨¯©«Ô6÷.ò .ô,"÷ #÷ #÷J
Ló 
L÷Mó Mð ×Ñ §¡£Ô.ô ˜9×+Ñ+Ô-A×BÓBÜœq×5Ñ5Ô7I×JÓJÜ×*Ñ*×8×8Ð8ô !"× 6Ñ 6× DÑ D�Ø$1×$8Ñ$8Ö$:�Lä%2ð -9×,CÑ,CÓ,E×,JÒ,Jóá,J ð !×0Ñ0°Ö:Ù,Jñó&�Nô '3°<×3JÑ3JÓ3L×3RÑ3RÕ&S�OØ!2°?Ð2CÀ5ÌÈ~ÓI^ÐH_Ð_`Ða�HØ(4×(=Ñ(=Ó(?Ð%Ø—K‘KØ*¨?Ð*;¸9ÐEVÐDWÐWZÐ[cÐZdÐdeÐfôð —K‘KØ*Ð+¨2Ð.?Ð-@ÀÐHYÐGZÐZaÐböñ %;ñ" ˜)Ô$÷O $Ñ#ùòr÷s $×#Ð#ús4   ÄM"Ä/AM"Æ CM"ÉAM"Ê*MËBM"ÍM"Í"M3	c                óT   € \         P                  V 4      pV P                  W1V4       R # r  )r@  Úbuildr‰  )r1  r¹   ro  rR  s   &&& rX   Úcodegen_loopsÚCppKernel.codegen_loopsµ
  s    € Ü—N‘N 4Ó(ˆ	Ø×Ñ 	°Ö=rW   c                ó    <€ V ^8„  d   QhRS[ /# ©r’   r¾   rr  )r–   r  s   "€rX   r—   r§  º
  s   ø€ ÷ !ñ !¡ñ !rW   c                óJ   € \         P                  P                  '       d   R # R# )ÚAOTI_TORCH_CHECKÚTORCH_CHECK)r5   rí  Úaot_moder7  s   &rX   Úassert_functionÚCppKernel.assert_function¹
  s   € ä�7‰7××ÐÙ%á rW   c                óv  € V P                   f   \        R4      hV P                   VP                  VP                  VP                  ,            pV P	                  4       p^p^ pV F„  p\
        P                  P                  P                  VRR7      pV^V,          8¼  g   WR8X  d    MDWB,          \        P                  P                  8  d    MV^,          pWX,          pWH,          pK†  	  \        P                  P                  '       d   V^ 8X  d   \        V4      ^ 8”  d   ^p\        WaP                  R7      # )Nr  i    ©Úfallback©rÿ   r   )r}  r…   r   rÿ   rJ  r5   rí  r  rI  r   rù  Úmin_chunk_sizer˜  rÛ   rý   )	r1  r{  rn  rW  ÚseqÚparre  r·  Úhints	   &&&      rX   rz  ÚCppKernel.decide_parallel_depthÀ
  s  € Ø×ÑÒ#Ü Ð!HÓIÐIØ×!Ñ!Ø×*Ñ*Ø"×.Ñ.Ð1C×1RÑ1RÕRð
ˆð
 �n‰nÓˆØˆØˆÛˆDÜ—7‘7×#Ñ#×5Ñ5°dÀTÐ5ÓJˆDØ�a˜'•kÔ! S¤^ÙØ�~¤§
¡
× 9Ñ 9Ô9áØ�Q�JˆEØ�KˆCØ�KŠCñ ô �:‰:×%×%Ð%¨%°1¬*¼¸V»Àq¼ØˆEÜØ ×.LÑ.Lô
ð 	
rW   c              #  ó  "  € V P                   V P                  V P                  V P                  3p\	        4       V n         \	        4       V n        \	        4       V n        V P                  P                  4       V n        R x € V P                  P                  V P                   4       V P                  P                  V P                  4       V P                  P                  V P                  4       Vw  V n         V n        V n        V n        R # 5ir  )r«  r°  r   r®  r=   Úcloner�  rb  )r1  r²  s   & rX   Úwrite_to_suffixÚCppKernel.write_to_suffixÞ
  s´   é € à—‘˜TŸ\™\¨4¯;©;¸¿¹ÐAˆÜ#Ó%ˆŒ
Ü%Ó'ˆŒÜ$Ó&ˆŒØ—8‘8—>‘>Ó#ˆŒÛØ×Ñ×$Ñ$ T§Z¡ZÔ0Ø×Ñ×$Ñ$ T§\¡\Ô2Ø×Ñ×$Ñ$ T§[¡[Ô1Ø<AÑ9ˆŒ�T”\ 4¤;°¶ùs   ‚DDc                ó   € \        V/ VB # r  ©rG   )r1  r™  r*  s   &*,rX   Úcreate_cse_varÚCppKernel.create_cse_varë
  s   € Ü˜tÐ. vÑ.Ð.rW   c                óö   € V'       d_   V\         P                  \         P                  39   d:   V\         P                  \         P                  39   d   R \
        V,           RV R2# R \
        V,           RV R2# )zc10::convert<z>(std::round(r¦   rÀ  r›   )rƒ   r!  rÅ  rm   rn   rH   )r1  Úsrcr‡   r¯   r¼  s   &&&&&rX   r­  ÚCppKernel.get_to_dtype_exprî
  se   € çØœ%Ÿ*™*¤e§k¡kÐ2Ô2ØœeŸk™k¬5¯<©<Ð8Ô8à"¤<°Õ#6Ð"7°}ÀSÀEÈÐLÐLØœ|¨EÕ2Ð3°2°c°U¸!Ð<Ð<rW   c                ó`   € V P                  W2V4      pV P                  P                  WQ4       R # r  )r­  r®  rd  )r1  ÚdstÚ	dst_dtyper©  r¯   r·  s   &&&&& rX   r³  ÚCppKernel.cache_dtype_convert÷
  s$   € Ø×%Ñ% c°iÓ@ˆØ�‰�‰�TÖrW   c                ó\   <€ V ^8„  d   QhRS[ RS[R,          RS[P                  R,          /# )r’   r¹   r`  Nr¬   )r7   rp   r”   rÆ  )r–   r  s   "€rX   r—   r§  û
  s6   ø€ ÷ 1ñ 1áð1ñ �d•
ð1ñ �\‰\˜DÕ ñ	1rW   c                ó¶  a a
€ Vf   RpS P                   '       g   R# . o
V
V 3R lpVeD   VS P                   9  d   \        R4      hS P                   V,          w  rVV! WVV4      '       g   R# M9S P                   P                  4        F  w  rxVw  rVV! WVV4      '       d   K   R# 	  RP                  S
4      p	V	'       d   VP	                  RV RV	 R	24       R# R# )
NrS   Tc                 ó`  <€ W8X  d   R # Rp\        SP                  4       F  w  rEW%8X  g   K  Tp M	  \        S4      \        J d*   V'       d"   V ^ 8X  d   VSP                  V,          8X  d   ^pSP                  V R\        V 4       24       SP                  V R\        V4       24       R# )FNr{  rÂ   T)rÌ   r<  r  ru  rW  r'  rE   )r¬  r­  r¬   Úvar_idrï   Ú_varÚ
conditionsr1  s   &&&   €€rX   ÚgenÚ)CppKernel.codegen_conditions.<locals>.gen  s¡   ø€ ØŒ|ÙØˆFÜ$ T§]¡]Ö3‘�Ø–;Ø�FÙñ 4ô
 �T“
œiÓ'ßØ˜Q”JØ˜4Ÿ;™; vÕ.Ô.à�à×Ñ   T¬+°eÓ*<Ð)=Ð>Ô?à×Ñ   S¬°SÓ)9Ð(:Ð;Ô<ÙrW   z"expected var in self.active_rangesFr¸  zif(r’  r¦   )r{  r…   r8  ÚjoinrÉ   )r1  r¹   r`  r¬   rµ  r¬  r­  r³  Ú_rangeÚjoined_conditionsr´  s   f&&&      @rX   Úcodegen_conditionsÚCppKernel.codegen_conditionsû
  sÛ   ù€ ð Š>ØˆFØ×!×!Ð!ÙØˆ
ö	ð* Š?Ø˜$×,Ñ,Ô,Ü$Ð%IÓJÐJØ×+Ñ+¨CÕ0‰JˆEÙ�u 3×'Ò'Ùð (ð !%× 2Ñ 2× 8Ñ 8Ö :‘�Ø#‘
�Ù˜5 t×,Ô,Ú ñ !;ð #ŸK™K¨
Ó3ÐßØ�N‰N˜S  ¨Ð*;Ð)<¸BÐ?Ô@ÙárW   )r±  r{  r}  r�  r°  r®  r|  r†  r<  r«  r„  r…  r‡  rˆ  r�  r‚  rƒ  r�  rŽ  rW  r‹  r~  r‘  r  r€  r�  r’  r   rŒ  r  )r$  éÿÿÿÿr   )NF©F©NN)4r  r  r  r  r  r�  r3  rD   rß  r‰  rŠ  r0  rµ   rŒ   r�  r¢  r®  rÇ   Úcontextmanagerrµ  rº  rÀ  rÏ  rÓ  rÖ  rä  r°  rò  rü  r  r  r  r  r(  r,  r=  r@  rF  rJ  r‰  r�  Úpropertyr•  rz  r¢  r¦  r­  r³  rº  r
  r  rl  rm  s   @@rX   ru  ru  ,  sJ  ù‡ € ñð €IØ€EØ€MØ€Fõ'1ð^ /Ø(ô'
òRI÷ò ð* ×Ññ$ó ð$÷&ò ÷2ð 2÷	
ð 	
÷
ð 
ò
5÷(:ð (:÷Tð òCô8ò&
÷
ð ÷:7ò 7òô.+$ôZò@FòP
ò
ò,8òZ%òx>ð ÷!ó ð!ò
ð< ×Ññ
Bó ð
Bò/ô=ò ÷1÷ 1ò 1rW   ru  c                   óÄ  a a€ ] tR tRt o]tR(V 3R lltV3R lR ltV3R lR ltV3R lR	 lt	V3R
 lR lt
]P                  3V3R lR lltV3R lR ltR(V3R lR lltR)V3R lR lltV3R lV 3R lltR*V3R lR lltR(R ltV3R lR ltR tR tV3R lR ltV3R lR  ltR! tR" tRRR]P6                  3V3R# lR$ lltR(V 3R% lltR*V 3R& lltR'tVt V ;t!# )+r(  i/  Nc                ó
  <€ \         SV `  W4       \        P                  ! 4       V n        V P                  '       g   \        R 4      hV^ 8:  d   \        R4      hW0n        W@n        WPn        V'       d	   WPn	        R# TV n	        R# )zexpected self.vec_isaz0Expect pass in Non-Zero tiling_factor explicitlyN)
r/  r0  r   Úpick_vec_isaÚvec_isar…   rÀ  r=  rQ  rû  )r1  r™  r�  rÀ  r=  rQ  r4  s   &&&&&&€rX   r0  ÚCppVecKernel.__init__2  sh   ø€ ô 	‰Ñ˜Ô+Ü"×/Ò/Ó1ˆŒØ�|�|ˆ|Ü Ð!8Ó9Ð9Ø˜AÔÜ Ð!SÓTÐTØ*ÔØ$ŒØ"Œß&/˜Ž°]ˆŽrW   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÃ  rÅ  )r–   r  s   "€rX   r—   ÚCppVecKernel.__annotate__E  s#   ø€ ÷ 4ñ 4©5¯:©:ð 4ÁÇÁñ 4rW   c                ó4  a € S P                  W4      '       d   R # V 3R lVP                   4        F9  p\        V\        4      '       g   \	        R4      hVP
                  '       g   K8   R # 	  \        WS P                  4      pVP                  '       d   V# R # )Nc              3   ó¶   <"  € T FN  p\        V\        P                  4      '       g   K%  SP                  P                  VP
                  ,          x € KP  	  R # 5ir  ©r   r   rÞ  r®  rÉ  rÎ   ©r  rË  r1  s   & €rX   r  Ú5CppVecKernel._try_get_const_stride.<locals>.<genexpr>H  s>   øé € ð 
á'�Ü˜a¤§¡×*ô )ˆD�H‰H× Ñ  §¡×(Ò(Û'ùó
   ƒ"Aª/Aú1expected isinstance(indirect_var, CppCSEVariable))	rÏ  rÍ  r§   rG   r…   r«   r%   rÀ  r&  )r1  r“   rÄ  Úindirect_varrB  s   f&&  rX   r>  Ú"CppVecKernel._try_get_const_strideE  s‹   ø€ Ø×)Ñ)¨%×9Ò9Ùô
à×'Ò'ö
ˆLô
 ˜l¬N×;Ò;Ü$ØGóð ð ×"×"Ò"Úñ
ô % U°T×5GÑ5GÓHˆØ×)×)Ð)ˆvÐ3¨tÐ3rW   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# ©r’   r‡   r¾   ©rƒ   r‡   rk   )r–   r  s   "€rX   r—   rÇ  V  s   ø€ ÷ ñ ¡e§k¡kð ±cñ rW   c                óØ   € \         P                  ! V P                  VP                  ,          ^,          V P                  P                  4       ,          4      pV^8  d   \        R4      hV# )é   zexpected num_vectors >= 1)Úmathr  rÀ  ÚitemsizerÄ  Ú	bit_widthr…   ©r1  r‡   Únum_vectorss   && rX   rR  ÚCppVecKernel._get_num_vectorsV  sT   € Ü—i’iØ×Ñ §¡Õ/°!Õ3°d·l±l×6LÑ6LÓ6NÕNó
ˆð ˜Œ?Ü Ð!<Ó=Ð=ØÐrW   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# rÒ  )rƒ   r‡   rm   )r–   r  s   "€rX   r—   rÇ  ^  s#   ø€ ÷ Rñ R©%¯+©+ð R¹%ñ RrW   c                óˆ   € V P                   VP                  ,          ^,          V P                  P                  4       ,          # )rÕ  )rÀ  r×  rÄ  rØ  )r1  r‡   s   &&rX   r¿  Ú!CppVecKernel._get_raw_num_vectors^  s0   € ð ×!Ñ! E§N¡NÕ2°QÕ6¸¿¹×9OÑ9OÓ9QÕQÐQrW   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# rÒ  ©rƒ   r‡   rp   )r–   r  s   "€rX   r—   rÇ  c  s#   ø€ ÷ Pñ P¡5§;¡;ð P±3ñ PrW   c                ó|   € V P                  V4      pV^8X  d   R\        V,           R2# R\        V,           RV R2# )r$  rK  rŽ   rL  rZ  )rR  rH   rÙ  s   && rX   r,  ÚCppVecKernel._get_vec_typec  sJ   € Ø×+Ñ+¨EÓ2ˆØ˜!ÔØ)¬,°uÕ*=Ð)>¸aÐ@Ð@à*¬<¸Õ+>Ð*?¸qÀÀÈQÐOÐOrW   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# rÒ  rà  )r–   r  s   "€rX   r—   rÇ  j  s#   ø€ ÷ Hñ H¡E§K¡Kð HÁ#ñ HrW   c                óz   € V\         P                  8X  d   R # V P                  V4      pR\        V,           RV R2# )rS   rY  rZ  rŽ   )rƒ   ro   rR  rH   rÙ  s   && rX   rf  ÚCppVecKernel._get_mask_typej  s<   € Ø”E—J‘JÔÙØ×+Ñ+¨EÓ2ˆØ"¤<°Õ#6Ð"7°q¸¸ÀQÐGÐGrW   c                ó@   <€ V ^8„  d   QhRS[ RS[P                  RS[/# )r’   r¯  r‡   r¾   )rG   rƒ   r‡   rp   )r–   r  s   "€rX   r—   rÇ  p  s+   ø€ ÷ Nñ N¡>ð N¹%¿+¹+ð NÉ#ñ NrW   c                ó¶   € VP                   \        P                  8w  d   \        \	        V4      4      hV P                  V4      pV R \        V,           RV R2# )z.template cast<rZ  r€   )r‡   rƒ   ro   r…   ÚreprrR  rH   )r1  r¯  r‡   rÚ  s   &&& rX   r  ÚCppVecKernel._get_mask_castp  sP   € Ø�:‰:œŸ™Ô#Ü ¤ d£Ó,Ð,Ø×+Ñ+¨EÓ2ˆØ��¤|°EÕ':Ð&;¸1¸[¸MÈÐMÐMrW   c          	      óh   <€ V ^8„  d   QhRS[ RS[P                  RS[P                  RS[R,          /# )r’   r¬   r“   r‡   Ú	load_maskN)rp   r”   r•   rƒ   r‡   rG   )r–   r  s   "€rX   r—   rÇ  v  s@   ø€ ÷ #ñ #áð#ñ �z‰zð#ñ �{‰{ð	#ñ
 " DÕ(ñ#rW   c                óF  € \         V,          pV P                  V4      pRpV'       dZ   VP                  '       g'   V P                  \        P
                  4       RV R2pM!V P                  V\        P
                  4       pV^ 8w  d   V R\        V4       2MTpV\        P                  8X  d/   V P                  4        RV R\        V P                  4       R2p	V	# V'       d   V RV RV RV R2M,V P                  V4       R	V R\        V P                  4       R2p	V	# )
aÏ  
Get a load line str that loads a vector from `var` at `index` of type `dtype`.
If `load_mask` is not None, we do a masked load accordingly.
Notes on the `dtype`:
1. We always load `self.tiling_factor` number of elements regardless of the `dtype`.
   It means we load half of the vector lanes for 16-bit data types and quarter of the
   vector lanes for 8-bit data types.
2. `torch.bool` and `torch.uint8` could mean masks and we load them as float mask vectors.
Nr+  r›   ró   r¡   z.template loadu<rZ  rÀ  ú::loadu()rH   rR  r«   rf  rƒ   rm   r  rE   ro   rû  r,  )
r1  r¬   r“   r‡   rë  rš  rÚ  Úload_mask_strÚloadbufrÏ   s
   &&&&&     rX   Ú_get_vec_load_lineÚCppVecKernel._get_vec_load_linev  s-  € ô    Õ&ˆØ×+Ñ+¨EÓ2ˆØˆßØ×#×#Ð#à#'×#6Ñ#6´u·{±{Ó#CÐ"DÀGÈIÈ;ÐVWÐ X‘à#'×#6Ñ#6°yÄ%Ç+Á+Ó#NÐ"O�Ø5:¸a´Z�S�E˜œ[¨Ó/Ð0Ñ1ÀSˆØ”E—J‘JÔà×)Ñ)Ó+Ð,¨G°G°9¸B¼{È4Ï>É>Ó?ZÐ>[Ð[\Ð]ˆDð ˆ÷ !ð !�/Ð!1°(°¸1¸[¸MÈÈGÈ9ÐTUÑVà×*Ñ*¨5Ó1Ð2°(¸7¸)À2ÄkÐRV×R`ÑR`ÓFaÐEbÐbcÐdð ð
 ˆrW   c                ó´   <€ V ^8„  d   QhRS[ R,          RS[P                  RS[P                  RS[R,          RS[ S[,          R,          RS[RS[R,          /# )	r’   r¬   Nr“   r‡   rì   Ústore_valueÚ
accu_storer¾   )rp   r”   r•   rƒ   r‡   r=   rG   ro   )r–   r  s   "€rX   r—   rÇ  ›  sz   ø€ ÷ Xñ Xá�4�ZðXñ �z‰zðXñ �{‰{ð	Xñ
  Õ%ðXñ ™>Õ)¨DÕ0ðXñ ðXñ 
˜$Õ	ñXrW   c                ó¾	  a aaa€ V'       d   Vf   \        R4      hV'       d   V'       g   \        R4      hSf   S P                  oR V 3R lloR V 3R lloR VVVV 3R	 llp\        4       pVP                  R
4       VP	                  4       ;_uu_ 4        S! V4      p	S! V4      p
R\
        V,           RV
 R2pVP                  V4       V'       d!   VP                  V R\        V	4       R24       \        S P                  S P                  ,           R24      p/ pV 3R lVP                   4        FJ  p\        V\        4      '       g   \        R4      hVP                  '       g   K8  V! V4      pV RV R2WÞ&   KL  	  S P                  VS P                  VR7      pRpS P                  e‡   V'       d   \        R4      h\        S P                  \        4      '       g   \        S P                  4      hS P                  P                  '       d   S P                   RV R2pMS P                   R2p\         P"                  ! 4       '       d    VP                  RS P$                   24       MVP                  RS P$                   24       VP                  RV R2V R\        S P&                  4       R2,           V R 2,           4       VP	                  4       ;_uu_ 4        \(        P*                  ! 4       ;_uu_ 4       p\        V4      pV F0  p\,        P.                  ! R!V ,           R!,           WÞ,          V4      pK2  	  Ve	   V RV R2MT pV'       d5   VP                  R"V R24       VP1                  VP	                  4       4       V'       d(   V'       d   R#MR$pVP                  V R%V R&V R'24       MVP                  R(V R)V R*24       RRR4       RRR4       V'       g)   S P3                  R+^ V4      pVP                  R,V R*24       RRR4       VP                  R-4       V'       d%   VP                  R*4       SP5                  V4       R# S P6                  P9                  SWƒR.7      p\        V\        4      '       g   \        R/4      hR0Vn        V#   + '       g   i     LÔ; i  + '       g   i     Lß; i  + '       g   i     Lº; i)1a  
Load or store a vector in a non-contiguous way. The vector is initialized from an array that is
filled in an inner loop over the tiling factor.
:param var: buffer to load from or store to, i.e. `var[transformed(index)]`. If None, we load the index
            as index expression, i.e. `transformed(index)`.
:param index: index into the `var` or the index expression by its own if `var` is None.
              The `index` could contain indirect indexing or the tiling itervar. When used in
              the inner loop, the index is transformed as follows:
              1. the index is linearized along the tiling dim.
              2. the indirect indexing vector variables are transformed into arrays over the tiling dim.
:param dtype: data type of `var` or `index` if `var` is None.
:param buffer: the code buffer to write the generated code to. If None, we write to `self.loads`.
:param store_value: the value to store. If None, we load the vector.
:param accu_store: whether accumulate the store_value to store_ptr. If True, a store_value should be provided
:return: a CppCSEVariable that represents the loaded vector or None if it is a store.
Nzstore var must be providedzexpected store_valuec                óD   € V ^8„  d   QhR\         P                  R\        /# rÒ  rÓ  )r–   s   "rX   r—   Ú@CppVecKernel._load_or_store_non_contiguous.<locals>.__annotate__¼  s   € ÷ 	&ñ 	&¤5§;¡;ð 	&´3ñ 	&rW   c                 ó†   <€ V P                   ^8  d%   SP                  ^V P                   ,          ,          # SP                  # ©é   )r×  rû  ©r‡   r1  s   &€rX   Úget_result_sizeÚCCppVecKernel._load_or_store_non_contiguous.<locals>.get_result_size¼  s1   ø€ Ø�~‰~ Ô!Ø—~‘~¨¨e¯n©nÕ)<Õ=Ð=à—~‘~Ð%rW   c                óD   € V ^8„  d   QhR\         P                  R\        /# rÒ  rÓ  )r–   s   "rX   r—   r÷  Â  s   € ÷ 	*ñ 	*¤5§;¡;ð 	*´3ñ 	*rW   c                 ó†   <€ V P                   ^8  d%   SP                  ^V P                   ,          ,          # SP                  # rù  )r×  rÀ  rû  s   &€rX   Úget_tiling_sizeÚCCppVecKernel._load_or_store_non_contiguous.<locals>.get_tiling_sizeÂ  s5   ø€ Ø�~‰~ Ô!Ø×)Ñ)¨Q°%·.±.Õ-@ÕAÐAà×)Ñ)Ð)rW   c                ó0   € V ^8„  d   QhR\         R\         /# )r’   Úvec_varr¾   r¥  )r–   s   "rX   r—   r÷  È  s   € ÷ 	ñ 	¤.ð 	´^ñ 	rW   c           	      óÈ  <€ V P                   '       g   \        R 4      h\        4       pVP                  R4       VP	                  4       ;_uu_ 4        V P
                  pVf   \        R4      hV\        P                  8X  d   \        P                  pS! V4      pS	! V4      pVP                  R\        V,           RV R24       V  R\        V4       R2pVP                  V4       VP                  R	4       RRR4       VP                  R
4       S
P                  P                  SV4      p\        V\        4      '       g   \        R4      hV#   + '       g   i     L`; i)zexpected vec_var.is_vecr/  Nzexpected vec_dtype is not NonerN  r¡   rO  rP  rÞ   zreturn tmpbuf;râ  r2  )r«   r…   r7   rÉ   rË   r‡   rƒ   ro   rm   rH   rE   r®  r¯  r§   rG   )r  r¹   r\  Úresult_sizeÚtiling_sizerÏ   r¸  rì   rü  r   r1  s   &      €€€€rX   Úvec_to_arrayÚ@CppVecKernel._load_or_store_non_contiguous.<locals>.vec_to_arrayÈ  s#  ø€ Ø—>—>�>Ü$Ð%>Ó?Ð?Ü“>ˆDØ�N‰N˜5Ô!Ø—‘—•Ø#ŸM™M�	ØÒ$Ü(Ð)IÓJÐJØ¤§
¡
Ô*Ü %§¡�IÙ-¨iÓ8�Ù-¨iÓ8�Ø—‘Ø.¬|¸IÕ/FÐ.GÀrÈ+ÈÐV_Ð`ôð "˜Ð"8¼À[Ó9QÐ8RÐRTÐU�Ø—‘˜tÔ$Ø—‘Ð/Ô0÷ ð �N‰N˜4Ô Ø—X‘X×&Ñ& v¨tÓ4ˆFÜ˜f¤n×5Ò5Ü$Ð%RÓSÐSØˆM÷% —ús   ÁB'EÅE!	r/  rN  r¡   rO  rP  rÞ   rN  c              3   ó¶   <"  € T FN  p\        V\        P                  4      '       g   K%  SP                  P                  VP
                  ,          x € KP  	  R # 5ir  rÊ  rË  s   & €rX   r  Ú=CppVecKernel._load_or_store_non_contiguous.<locals>.<genexpr>ò  s>   øé € ð !á+�AÜ! !¤T§X¡X×.ô -�—‘×$Ñ$ Q§V¡V×,Ò,Û+ùrÍ  rÎ  rê  rë  ©r·  rJ  zunexpected store with load maskz.is_masked(r›   z != 0z#pragma GCC unroll z#pragma unroll ú
for (long ú = 0; rÂ   rÃ   rÄ   rÆ   r0  ú+=Ú=r�   z tmpbuf[rà   ztmpbuf[r÷  rÁ   ztmpbuf.data()r­  râ  rJ  r2  T)r…   r«  r7   rÉ   rË   rH   rE   r.   r<  r=  rÍ  r§   rG   r«   rº  r±  r   Úis_gccrÀ  rû  rÇ   rÈ   rÐ   rÑ   rÊ   rð  rb  r®  r¯  )r1  r¬   r“   r‡   rì   ró  rô  r  r¹   r  r  Úresult_declareÚitervar_innerÚreplacementsrÏ  Ú	array_varrë  rÓ   Úindex_cÚrhsr±   Ú	load_liner¸  rü  r   s   f&&&f&&                @@rX   r@  Ú*CppVecKernel._load_or_store_non_contiguous›  s4  û€ ÷2 ˜3š;Ü Ð!=Ó>Ð>ßßÜ$Ð%;Ó<Ð<ØŠ>Ø—Z‘ZˆF÷	&ð 	&÷	*ð 	*÷	ó 	ô2 ‹~ˆØ�‰�uÔØ�[‰[�]�]Ù)¨%Ó0ˆKÙ)¨%Ó0ˆKà*¬<¸Õ+>Ð*?¸rÀ+ÀÈiÐXð ð �N‰N˜>Ô*ßØ—‘Ø"�mÐ#9¼+ÀkÓ:RÐ9SÐSUÐVôô /Ø—=‘= §¡Õ1Ð2°&Ð9óˆMð ˆLô!à×+Ò+ö!�ô
 " ,´×?Ò?Ü(ØKóð ð  ×&×&Ò&Ù ,¨\Ó :�IØ4=°;¸aÀ¸ÈaÐ1P�LÓ.ñ!ð ×0Ñ0Ø 4§?¡?¸=ð 1ó ˆEð ˆIØ�‰Ò*ßÜ(Ð)JÓKÐKÜ! $§/¡/´>×BÒBÜ(¨¯©Ó9Ð9Ø—?‘?×)×)Ð)Ø#'§?¡?Ð"3°;¸}¸oÈQÐ O‘Ià#'§?¡?Ð"3°5Ð 9�IÜ×!Ò!×#Ò#Ø—‘Ð!4°T×5GÑ5GÐ4HÐIÕJà—‘ °×1CÑ1CÐ0DÐEÔFØ�N‰NØ˜]˜O¨6Ð2Ø"�O 3¤{°4·>±>Ó'BÐ&CÀ2ÐFõGà"�O 3Ð'õ(ôð
 —‘—•¤
× 4Ò 4× 6Ô 6¸%Ü% eÓ,�ã$0�LÜ ŸfšfØ < .Õ1°EÕ9Ø$Õ2Øó’Gñ %1ð .1ª_˜˜˜Q˜w˜i qÑ)ÀWÀI�ßØ—N‘N T¨)¨°AÐ#6Ô7Ø×'Ñ'¨¯©«Ô6ßß*4¡$¸#�KØ—N‘N c U¨!¨K¨=¸ÀÀÈrÐ#RÕSà—N‘N W¨]¨O¸4À¸uÀAÐ#FÔG÷# !7—÷$ Ø ×3Ñ3°OÀQÈÓN�	Ø—‘ ¨¨°1Ð5Ô6÷K ðL 	�‰�tÔßØ�N‰N˜3ÔØ�M‰M˜$ÔÙà—X‘X×&Ñ& v¨tÐ&ÓAˆFÜ˜f¤n×5Ò5Ü$Ð%RÓSÐSØ ˆFŒMØˆM÷? !7× 6ú——ú÷c �]úsq   ÂCSÅ/ASÆ;ASÈ<SÉBSË(R9	ÌAR&Í;R&ÎR&Î#7R&ÏR9	Ï"SÏ2(SÒ&R6Ò1R9	Ò9S	ÓSÓS	c                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# rç  rè  )r–   r  s   "€rX   r—   rÇ  5  s   ø€ ÷ ñ ™ð ¡U§Z¡Zñ rW   c                óZ  <€ V P                   P                  V4      pV P                  V4      p\        P                  P                  V4      pV P                  V P                  ,          pV P                  W%4      pV^ 8X  d   \        S	V `)  W4      # V^8X  dE   V P                  W2W@P                  4      pV P                  P                  V P                  WtR7      pMV P!                  W2V4      p\#        V\$        4      '       g   \'        R4      hVP)                  RWV3/ 4       RVn        V# )r   rJ  r2  r°  T)r™  rì  r£  r5   rí  rî  r<  r=  r>  r/  r°  rð  r±  r®  r¯  r«  r@  r§   rG   r…   r±  r«   )
r1  rÎ   r“   r¬   r‡   rA  rB  rÏ   r¸  r4  s
   &&&      €rX   r°  ÚCppVecKernel.load5  sù   ø€ Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆÜ—‘×!Ñ! $Ó'ˆØ—]‘] 4§?¡?Õ3ˆ
Ø×+Ñ+¨EÓ>ˆØ�QŒ;ä‘7‘< Ó,Ð,Ø�qŒ[à×*Ñ*¨3°u¿o¹oÓNˆDØ—X‘X×&Ñ& t§z¡z°4Ð&ÓE‰Fà×7Ñ7¸ÀEÓJˆFÜ˜&¤.×1Ò1Ü Ð!NÓOÐOØ×Ñ˜f t°5Ð&9¸2Ô>ØˆŒØˆrW   c          
      óp   <€ V ^8„  d   QhRS[ S[,          RS[ RS[P                  RS[P
                  RS[/# )r’   rù   r¬   r“   r‡   rô  )rp   rG   r”   r•   rƒ   r‡   ro   )r–   r  s   "€rX   r—   rÇ  J  sJ   ø€ ÷ ,ñ ,á‘^Õ#ð,ñ ð,ñ �z‰zð	,ñ
 �{‰{ð,ñ ñ,rW   c           	     ó&  € \        V\        4      '       g4   \        V\        4      '       d   VP                  '       g   \	        V4      hV P
                  V P                  ,          pV R\        V4       2pV P                  W64      p\        4       p	V^8X  då   V'       dq   V\        P                  8X  d&   V P                  f   V P                  V4       RV R2M,V P                  V4       RV R\        V P                  4       R2p
RV RV
 R2pV\        P                  8X  d(   V P                  f   V	P                  V RV R24       V	# V	P                  V RV R\        V P                  4       R24        V	# V P!                  W#WIWR7       V	# )	a  
Get a store line buffer that stores `value` into `var` at `index` of `dtype`. It handles
both contiguous and non-contiguous store cases.
:param value: Vectorized type templaterized on `dtype`.
:param var: buffer to store into.
:index: index into the `var`.
ró   rí  r›   r¡   r’  ú.store(rÞ   )rì   ró  rô  )r§   rp   rG   r«   r…   r<  r=  rE   r>  r=   rƒ   rm   rQ  r,  rû  rÉ   r@  )r1  rù   r¬   r“   r‡   rô  rA  Úvar_exprrB  r¹   r°  s   &&&&&&     rX   Ú_get_store_lineÚCppVecKernel._get_store_lineJ  sˆ  € ô$ �uœc×"Ò"Ü˜5¤.×1Ò1°e·l·l°lä  Ó'Ð'Ø—]‘] 4§?¡?Õ3ˆ
Ø�U˜#œk¨%Ó0Ð1Ð2ˆØ×+Ñ+¨EÓ>ˆÜÓˆØ�QŒ;ßð ¤§¡Ô+°·±Ò0Fð ×)Ñ)¨%Ó0Ð1°¸(¸À1ÑEà ×.Ñ.¨uÓ5Ð6°h¸x¸jÈÌ;ÐW[×WeÑWeÓKfÐJgÐghÐið ð
 ˜E˜7 # d V¨1Ð-�ØœŸ™Ô#¨¯©Ò(>Ø—‘ % ¨°¨z¸Ð<Ô=ð ˆð —‘Ø�g˜W X J¨b´¸T¿^¹^Ó1LÐ0MÈRÐPõð ˆð ×.Ñ.Ø˜E¸Eð /ô ð ˆrW   c                ó\  a€ R S9  d   \        R4      h\        V\        4      '       g   \        V4      hVP                  '       g   V P	                  V4      pV P
                  P                  S4      pV P                  V4      p\        P                  P                  S4      pVfC   V P                  W5W&4      pV P                  P                  VP                  V3R l4      4       R# VR8X  Ed°   V P                  4       '       gH   V P                  V VVVRR7      pV P                  P                  VP                  V3R l4      4       R# V P!                  V4      pV P!                  \"        P$                  4      p	\&        V,          p
\(        P*                  ! V\"        P$                  4      P,                  p\        V\        4      '       d$   VP                  '       g   V P	                  V4      p\        V\        4      '       d   VP                  '       g   \        R4      hV P.                  '       d-   R	V
 R
V	 R
V RV R
V R
V R
\1        V P.                  4       R2pMR	V
 R
V	 R
V RV R
V R
V R2pV P                  P3                  \5        SV4      4       R# \7        RV 24      h)rõ  rö  Nc                 ó   <€ \        SV 4      # r  ©r;   ©rµ  rÎ   s   &€rX   Ú<lambda>Ú$CppVecKernel.store.<locals>.<lambda>…  s   ø€ ´,¸tÀQÔ2GrW   rø  T)rô  c                 ó   <€ \        SV 4      # r  r$  r%  s   &€rX   r&  r'  �  s   ø€ ´lÀ4ÈÔ6KrW   z;expected isinstance(index, CppCSEVariable) and index.is_veczatomic_add_vec<r¡   rÀ  rÞ   rù  )r…   r§   rG   r«   r)  r™  rú  r£  r5   rí  rî  r   r   rb  Úmaprò  rR  rƒ   r•  rH   r3   r¥  rù   rQ  rE   rÉ   r;   r  )r1  rÎ   r“   rù   rû  r¬   r‡   r¹   Ún_srcÚn_idxrˆ   rÏ   s   &f&&&       rX   rü  ÚCppVecKernel.storex  sR  ø€ Ø˜ÔÜ Ð!9Ó:Ð:Ü˜%¤×0Ò0Ü  Ó'Ð'Ø�|�|ˆ|à—N‘N 5Ó)ˆEØ�i‰i×Ñ˜tÓ$ˆØ×$Ñ$ UÓ+ˆÜ—‘×!Ñ! $Ó'ˆØŠ<Ø×'Ñ'¨°EÓAˆDØ�K‰K×Ñ˜tŸx™xÔ(GÓHÖIØ�\Õ!Ø×0Ñ0×2Ò2Ø×+Ñ+Ø�gØØØØ#ð ,ó �ð —‘×"Ñ" 4§8¡8Ô,KÓ#LÖMà×-Ñ-¨eÓ4�Ø×-Ñ-¬e¯k©kÓ:�Ü% eÕ,�ô Ÿš u¬e¯k©kÓ:×@Ñ@�Ü˜e¤^×4Ò4¸U¿\¿\¸\Ø ŸN™N¨5Ó1�EÜ" 5¬.×9Ò9¸e¿l¿l¸lÜ(ØUóð ð —>—>�>Ø,¨V¨H°B°u°g¸RÀ¸wÀbÈÈÈRÐPUÈwÐVXÐY^ÐX_Ð_aÔbmÐnr×n|Ñn|Ób}Ða~ð  Að  B‘Dà,¨V¨H°B°u°g¸RÀ¸wÀbÈÈÈRÐPUÈwÐVXÐY^ÐX_Ð_aÐb�DØ—‘×%Ñ%¤l°4¸Ó&>Ö?ä%¨°D°6Ð&:Ó;Ð;rW   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r½  rï  )r–   r  s   "€rX   r—   rÇ  ¥  s#   ø€ ÷ ñ ©U¯Z©Zð ¹E¿J¹Jñ rW   c                ó   € V# r  rV   r¿  s   &&rX   Ú_adjust_argreduce_indexÚ$CppVecKernel._adjust_argreduce_index¥  s   € ØˆrW   c                ó¢  € V\         9  d   \        R4      hVR/9   pRpV'       d   \        V\        4      '       d   Vw  rFV P                  V P
                  8¬  pV'       d   TMTp\        V\        4      '       g   \        V4      h\        V\        \        R4      34      '       g   \        V4      hVP                  '       g   V P                  V4      pVe   \        \        V4      p	W#WI33p
MW#V3p
W P                  P                  9   d   V P                  P                  V
,          # RpV R\        V,           R2p\        W84      pV P                  W84      pV P                  P!                  V P"                  RV
 2RR7      p\        V\        4      '       g   \        R	4      hV R
2pRV 2pRV 2pV ;P$                  V VV.,          un        RV n        V P(                  P+                  V P-                  WýW8\.        4      4       V P(                  P+                  V P-                  VVVVV P0                  4      4       V P3                  W1R4      pV'       Ed6   Ve   \        V4      hV P(                  P+                  V P-                  VVVVV P0                  4      4       V P
                  f   \        R4      h\4        P6                  ! \8        P:                  V P<                  V P
                  R 4      pVR8X  d-   V P>                  P!                  V P@                  RV
 2RR7      pM+V PB                  P!                  V P@                  RV
 2RR7      pRV 2p\E        V P<                  V P                  ,          V PF                  4      '       d{   V P                  V P
                  8¼  d^   \E        VV P<                  V P                  ,          4      \E        V P<                  V P                  ,          V PF                  4      ,          MTM\H        PJ                  ! ^ 4      pV P<                  V P                  ,          V PF                  ,          '       dE   V P                  V P
                  8¼  d(   \E        VV P<                  V P                  ,          4      MTM\H        PJ                  ! ^ 4      pRV 2pV PM                  VVVVVRR7       V PM                  W?VVV4       V PM                  VVVVV4       V PN                  '       d   TMTpV PN                  '       d   TMTpVR8X  d6   V PP                  PS                  V RV PU                  VVVV4       R24       EMV PP                  PS                  V RV PU                  VVVV4       R24       MçVe   TpRpM¡V P
                  f   \        R4      hV PV                  V P
                  ,          p\Y        V P
                  ^,           \[        V PV                  4      4       F5  pVV P<                  V,          ,          V PV                  V,          ,           pK7  	  TpRVRVRVRV/pV PP                  PS                  V RV PT                  ! VV3/ VB  R24       V P]                  VVVVV PT                  V P0                  R7       V P]                  VVVV\^        \.        R7       V'       d,   V P]                  VVVVV PT                  V P0                  R7       V\`        Pb                  8H  p V'       Ed‘   \e        V4      '       dZ   V Pg                  V4      R09  d   \        R4      hRV R2p!RV R2p"V Ph                  PS                  V R\_        W?V"4       R24       MöV'       d
   V RV R2p!MåV '       d'   VR19   d   RV R2p!MÏVR8w  d   \        R 4      hV R!2p!M·R"V PU                  VR#R$4      ,           R%,           p#V\`        Pb                  8H  p V '       d   \`        Pj                  MTp$R&\        V$,           R2pR'\        V$,           R(V Pg                  V$4       R2p%T p&V'       d   VR8w  d   \        R)4      hV R*V 2p&V% R+V R,V R-V# R(V& R2
p!V Ph                  PS                  V R\_        W?V!VR.7       R24       Tp'MˆTp'\e        V4      '       d3   RV' 2p(V Ph                  PS                  V' R\_        VV'V(4       R24       MCV'       d<   VR8w  d   \        R)4      hRV' 2p(V Ph                  PS                  V' RV' R*V( R24       \m        VV'4      p)V)V P                  P                  V
&   V)# )2a  
Perform vectorized reduction operation.

This method handles vectorized reduction for different reduction types.
It manages special cases for low-precision floating point types and
employs precision improvement techniques for certain reduction operations.

Args:
    dtype: The output data type for the reduction result
    src_dtype: The source data type of the input value
    reduction_type: Type of reduction operation (sum, min, max, etc.)
    value: The input value to reduce

Returns:
    The result of the reduction operation
z.expected reduction_type in VECTORIZABLE_RTYPESNzat::vecz::Vectorized<rŽ   r/  Fr0  z(expected isinstance(acc, CppCSEVariable)r+  Úmasked_Tr4  rg   r2  r3  ra   rÀ   rÁ   r­   r“   Úhorizontal_reductionr¯   )r™  rš  z4Welford reduction does not support VectorizedN (N>2)zwelford_vec_reduce_all(r›   z_vec_reduce_all(r½  z.all_zero()r_   z expected reduction_type == "min"z.all_masked()z	{ return rµ  rD  z; }rK  zat::vec::vec_reduce_all<r¡   z expected reduction_type == "sum"ró   z([](z& x, z& y) r—  r5  )r$  r’   )rf   ra   r`   )7ÚVECTORIZABLE_RTYPESr…   r§   r¨   r=  r~  rG   r  r«   r)  r   r‹  r6  rH   r�   Úreduction_acc_type_vecr¯  r«  r’  r†  r€  r'  r  rŒ   Úreduction_init_vecr  rS  rT  rU  rV  rW  rŒ  r°  r�  r   rÀ  r”   rX  r,  rQ  r   rÉ   Úreduction_combine_vecr<  rQ  rÛ   r�  rµ   rƒ   ro   r+   rR  r�  rm   r©   )*r1  r‡   r¯   r†   rù   r7  r8  r3  r;  r9  r:  Úvec_nsrú  rÚ   Úacc_type_vecr·   Úacc_vecÚ
masked_accÚmasked_acc_vecÚuse_acc_helperr  r®   Úmasked_helper_valÚhelper_vec_rangeÚmasked_helper_vec_ranger<  Úacc_vec_Úhelper_val_r“   Úindex_horizontal_reductionrï   r*  r°   r­   Úmasked_next_valueÚreduce_all_bodyr\  Úvec_reduce_all_funcÚ
result_vecÚtmpvarÚmasked_tmpvarr³  s*   &&&&&                                     rX   r=  ÚCppVecKernel.reduction¨  st	  € ð& Ô!4Ô4Ü Ð!QÓRÐRØ)Ð-AÑAÐØˆß¤
¨5´%× 8Ò 8Ø#(Ñ ˆEà#Ÿ™°$×2FÑ2FÑFÐß"2‘Y¸ˆ
Ü˜%¤×0Ò0Ü  Ó'Ð'Ü˜-¬.¼$¸t»*Ð)E×FÒFÜ  Ó/Ð/à�|�|ˆ|Ø—N‘N 5Ó)ˆEàÒ$Ü $¤^°]Ó CÐØ%¸Ð7QÐQ‰Mà%°uÐ<ˆMØ×.Ñ.×>Ñ>Ô>Ø×%Ñ%×5Ñ5°mÕDÐDàˆØ�˜¤l°5Õ&9Ð%:¸!Ð<ˆÜ% nÓAˆØ×2Ñ2°>ÓNˆà× Ñ ×)Ñ)Ø�J‰J˜* ] OÐ4¸Eð *ó 
ˆô ˜#œ~×.Ò.Ü Ð!KÓLÐLØ�E˜�,ˆØ˜s˜e�_ˆ
Ø" 7 )Ð,ˆØ× Ò ¨ u¨w¸Ð$GÕGÕ Ø ˆÔØ×(Ñ(×/Ñ/Ø×&Ñ&Ø˜~¼>óô	
ð
 	×(Ñ(×/Ñ/Ø×&Ñ&ØØØØØ×'Ñ'óô	
ð ×1Ñ1°.ÈÓOˆßˆ>ØÒ(Ü$ ]Ó3Ð3à×,Ñ,×3Ñ3Ø×*Ñ*Ø"Ø Ø"ØØ×+Ñ+óôð ×#Ñ#Ò+Ü$Ð%PÓQÐQÜ&×-Ò-Ü—‘˜dŸk™k¨$×*>Ñ*>Ð*@ÐAóˆNð Ð!1Ô1Ø!×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ó ‘
ð "×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ó �
ð #*¨*¨Ð 6Ðô ˜DŸK™K¨¯©Õ8¸$×:LÑ:L×MÒMð —‘¨$×*>Ñ*>Ô>ô ˜^¨T¯[©[¸¿¹Õ-IÓJÜ˜tŸ{™{¨4¯?©?Õ;¸T×=OÑ=OÓPöQñ (ô —]’] 1Ó%ð ð  —;‘;˜tŸ™Õ/°$×2DÑ2D×DÔDð —‘¨$×*>Ñ*>Ô>ô ˜^¨T¯[©[¸¿¹Õ-IÔJá'ô —]’] 1Ó%ð $ð #*¨*¨Ð 6ÐØ× Ñ ØØØ!ØØØð !ô ð × Ñ Ø ZÐ1AÀ5ôð × Ñ ØØØ!Ø'Øôð *.¯¯¨‘~¸WˆHØ/3¯~¯~¨~Ñ+À:ˆKØ Ô&Ø—‘×%Ñ%Ø�j  D×$>Ñ$>¸~ÈxÐY^Ð`kÓ$lÐ#mÐmnÐoöð —‘×%Ñ%Ø�j  D×$>Ñ$>¸~ÈxÐY^Ð`kÓ$lÐ#mÐmnÐoõð Ò(Ø%�Ø-2Ñ*à×'Ñ'Ò/Ü(Ð)TÓUÐUØŸ™ d×&:Ñ&:Õ;�Ü˜t×3Ñ3°aÕ7¼¸T¿]¹]Ó9KÖL�AØ! D§K¡K°¥NÕ2°T·]±]À1Õ5EÕE’Eñ Mà-AÐ*à˜eØ˜Ø&Ð(BØ˜Yð	ˆFð �K‰K×!Ñ!Ø�)˜3˜t×9Ò9¸.È'Ñ\ÐU[Ñ\Ð]Ð]^Ð_ôð 	×,Ñ,ØØØØØ!%×!;Ñ!;Ø"×5Ñ5ð 	-ô 	
ð 	×,Ñ,ØØØØÜ!2Ü,ð 	-ô 	
÷ à×0Ñ0ØØØØØ%)×%?Ñ%?Ø"&×"9Ñ"9ð 1ô ð œ5Ÿ:™:Ñ%ˆßÐä# N×3Ò3Ø×(Ñ(¨Ó/ð 8ô ô )ØNóð ð  7°w°i¸qÐA�
Ø&=¸nÐ=MÈQÐ$OÐ!Ø×%Ñ%×/Ñ/Ø�e˜3Ô0°ÐFWÓXÐYÐYZÐ[õ÷ "Ø .Ð/Ð/?À¸yÈÐJ‘
ßØ!ð &ô ð
 $% W I¨[Ð!9‘Jà%¨Ô.Ü,Ð-OÓPÐPØ$+ 9¨MÐ!:‘Jð  Ø×0Ñ0°ÀÀcÓJõKàõð  ð
  ¤5§:¡:Ñ-�ç+2œEŸKšK¸�	Ø,¬\¸)Õ-DÐ,EÀQÐG�Ø(@ÄÈiÕAXÐ@YÐY[Ð\`×\qÑ\qÐr{Ó\|Ð[}Ð}~Ð&Ð#Ø '˜y�
ß!Ø%¨Ô.Ü,Ð-OÓPÐPØ$+ 9¨C°Ð/?Ð!@�JØ 3Ð4°D¸¸¸UÀ3À%ÀuÈ_ÐL]Ð]_Ð`jÐ_kÐklÐm�
à×!Ñ!×+Ñ+Ø�%�sÔ,¨^À*ÐXaÔbÐcÐcdÐeôð ‰FàˆFÜ# N×3Ò3Ø")¨&¨Ð 2�Ø×%Ñ%×/Ñ/Ø�h˜cÔ"3°NÀFÈMÓ"ZÐ![Ð[\Ð]õ÷  Ø! UÔ*Ü(Ð)KÓLÐLØ")¨&¨Ð 2�Ø×%Ñ%×/Ñ/Ø�h˜c & ¨¨]¨O¸1Ð=ôô # >°6Ó:ˆØ<Bˆ×Ñ×*Ñ*¨=Ñ9ØˆrW   c                óp  a€ V P                  V4      pV P                  P                  S4      p\        P                  P                  S4      pVP                  '       d'   V\        P                  8w  d   \        P                  pMTp\        P                  P                  V4      p\        P                  P                  V4      p\        4       p	V P                  V P                  8¼  d3   V	P                  V R \!        V4       R\"        V,           RV R24       MìWV8w  dÆ   \"        V,          P%                  RR4       RV 2p
V\        P&                  8X  d'   V RV P                  \        P&                  4       R2pMNY‡u;8X  d   ^8X  d   M MR\"        V,           RV R	2pM(R\"        V,           R
V R
\"        V,           R
V RV R	2pV	P                  RV
 RV R24       T
pV	P)                  V P+                  W4W%4      4       V P,                  P)                  V	P/                  V3R l4      4       R# )rê  z] = static_cast<rÀ  rÞ   z::rÔ   z.template cast<bool,r€   zat::vec::convert<r›   rZ  rÝ   rÀ   rÁ   c                 ó   <€ \        SV 4      # r  r$  r%  s   &€rX   r&  Ú.CppVecKernel.store_reduction.<locals>.<lambda>Ý  s   ø€ ¼ÀTÈ1Ô8MrW   N)r£  r™  rú  r5   rí  rî  Úis_floating_pointrƒ   rn   rm   re  rR  r=   r=  r~  rÉ   rE   rH   Úreplacero   rb  r   r�  r)  )r1  rÎ   r“   rù   r¬   Ú	out_dtyper‡   Úout_num_vectorsÚsrc_num_vectorsr¹   Úconverted_valueÚconverts   &f&&        rX   r@  ÚCppVecKernel.store_reduction¸  s  ø€ Ø×$Ñ$ UÓ+ˆØ�i‰i×Ñ˜tÓ$ˆÜ—G‘G×%Ñ% dÓ+ˆ	Ø×&×&Ð&¨9¼¿¹Ô+DÜ—K‘K‰EàˆEÜŸ(™(×3Ñ3°IÓ>ˆÜŸ(™(×3Ñ3°EÓ:ˆÜÓˆØ�?‰?˜d×2Ñ2Ô2à�N‰NØ�%�qœ UÓ+Ð,Ð,<¼\È)Õ=TÐ<UÐUWÐX]ÐW^Ð^`Ðaõð
 Ô!ä# IÕ.×6Ñ6°t¸SÓAÐBÀ!ÀEÀ7ÐKð  ð ¤§
¡
Ô*Ø!& Ð';¸D×<QÑ<QÔRW×R\ÑR\Ó<]Ð;^Ð^aÐb‘Gà&Ö>¸Q×>à/´¸YÕ0GÐ/HÈÈ5È'ÐQRÐSñ  ð
 0´¸YÕ0GÐ/HÈØ.Ð/¨q´¸eÕ1DÐ0EÀQÀÐFWÐWYÐZ_ÐY`Ð`aðcð  ð —‘  Ð&7°s¸7¸)À1ÐEÔFØ'�Ø�K‰K˜×,Ñ,¨U¸ÓJÔKØ×Ñ×$Ñ$ T§X¡XÔ.MÓ%NÖOrW   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r’   Ú
scalar_varr¾   r¥  )r–   r  s   "€rX   r—   rÇ  ß  s   ø€ ÷ ñ ¡Nð ±~ñ rW   c                ól  € VP                   '       d   \        R 4      hVP                  \        P                  8X  dF   V P
                  P                  V P                  V P                  4        RVP                   R24      pMhVP                  f   \        R4      hV P
                  P                  V P                  V P                  VP                  4       RVP                   R24      p\        V\        4      '       g   \        R4      hVP                  Vn        VP                  Vn        RVn         V# )zexpected not scalar_var.is_vecr+  r›   z%expected scalar_var.dtype is not Noner’  z,expected isinstance(vec_var, CppCSEVariable)T)r«   r…   r‡   rƒ   ro   r®  r¯  r°  rf  rÎ   r,  r§   rG   Údependent_itervars)r1  rW  r  s   && rX   r)  ÚCppVecKernel.broadcastß  s  € Ø××ÐÜ Ð!AÓBÐBØ×ÑœuŸz™zÔ)Ø—h‘h×'Ñ'Ø—‘ ×!4Ñ!4Ó!6Ð 7°w¸z¿¹Ð>OÈqÐQó‰Gð ×ÑÒ'Ü$Ð%LÓMÐMØ—h‘h×'Ñ'Ø—‘Ø×%Ñ% j×&6Ñ&6Ó7Ð8¸¸*¿/¹/Ð9JÈ!ÐLóˆGô ˜'¤>×2Ò2Ü Ð!OÓPÐPØ"×(Ñ(ˆŒØ%/×%BÑ%BˆÔ"ØˆŒØˆrW   c                ó@   <€ V ^8„  d   QhRS[ RS[P                  RS[ /# )r’   r“   rB  r¾   )rG   r”   rÆ  )r–   r  s   "€rX   r—   rÇ  ô  s&   ø€ ÷ ñ ™Nð ±E·L±Lð Á^ñ rW   c           	     ót  € VP                   '       d   \        R 4      hVP                  f   \        R4      hV P                  P	                  V P
                  V P                  VP                  4       RV RV R24      p\        V\        4      '       g   \        R4      hVP                  Vn        RVn         V# )zexpected not index.is_vecz expected index.dtype is not Nonez	::arange(r¡   r›   r2  T)	r«   r…   r‡   r®  r¯  r°  r,  r§   rG   )r1  r“   rB  r¸  s   &&& rX   r?  ÚCppVecKernel.arangeô  s    € Ø�<�<ˆ<Ü Ð!<Ó=Ð=Ø�;‰;ÒÜ Ð!CÓDÐDØ—‘×"Ñ"Ø�L‰LØ×!Ñ! %§+¡+Ó.Ð/¨y¸¸¸rÀ&ÀÈÐKó
ˆô ˜&¤.×1Ò1Ü Ð!NÓOÐOØ—{‘{ˆŒØˆŒØˆrW   c                óÄ  € \         V,          pV P                  V4      p\        V4      '       d   R V R2# VR9   dº   V\        P                  8X  d   \        P
                  MTp\        V,          pV P                  W4      pVR8X  d4   \        V4      '       g   V\        P                  8X  d   RV R2MRV R2pM2\        V4      '       g   V\        P                  8X  d   RV R2MRV R2pV RV R	2# VR
8X  d   V P                  4        R2# \        W4      p	V RV	 R	2p
V\        P                  8X  d)   VR9  d   \        R4      hV P                  4        RV	 R	2# V
# )r   r€   rd   rz   ry   r|   rx   r{   r’  r›   rf   z	::from(0)z0expected reduction_type in ("min", "max", "sum")r+  r�   )r_   r`   ra   )r<   r,  r+   rƒ   ro   rm   rH   r5  r   rf  rŒ   r…   )r1  r†   r‡   r�   Úvec_typeÚcompute_dtyperˆ   rÚ   r�  Úscalar_initÚvec_inits   &&&        rX   r6  ÚCppVecKernel.reduction_init_vec  sx  € Ü0°Õ7ˆØ×%Ñ% kÓ2ˆä ×/Ò/Ø˜h˜Z sÐ+Ð+àÐ1Ô1à+0´E·J±JÔ+>œEŸKšKÀKˆMÜ! -Õ0ˆFØ×2Ñ2°>ÓIˆHØ Ô)ô & e×,Ò,°¼¿¹Ô0Cð +¨6¨(°-Ñ@à/°¨x°xÐ@ñ ô & e×,Ò,°¼¿¹Ô0Cð ,¨F¨8°=ÑAà/°¨x°xÐ@ð ð
 �Z˜q   QÐ'Ð'à˜UÔ"Ø×)Ñ)Ó+Ð,¨IÐ6Ð6ä$ ^Ó;ˆØ�Z˜q  ¨QÐ/ˆØ”E—J‘JÔØÐ%:Ô:Ü$Ð%WÓXÐXØ×)Ñ)Ó+Ð,¨G°K°=ÀÐBÐBØˆrW   c                ó.  € \         V,          pV P                  V4      p\        V4      '       d   R V R2# VR9   d¡   V P                  \        P
                  4      pV\        P                  8X  dE   V P                  \        P                  4      pR\        \        P                  ,           RV RV R2# V P                  V4      pR\        V,           RV RV R2# V\        P                  8X  d$   VR9  d   \        R4      hV P                  4        # V# )r   rŽ   zIndexValueVec<r¡   z7expected reduction_type in ("min", "max", "any", "sum")r�   )r_   r`   rf   ra   )r<   r,  r+   rR  rƒ   r•  ro   rm   rH   r…   rf  )r1  r†   r‡   r�   r_  r+  r*  s   &&&    rX   r5  Ú#CppVecKernel.reduction_acc_type_vec(  s  € Ü0°Õ7ˆØ×%Ñ% kÓ2ˆÜ ×/Ò/Ø˜h˜Z qÐ)Ð)ØÐ1Ô1Ø×)Ñ)¬%¯+©+Ó6ˆEØœŸ
™
Ô"ð ×-Ñ-¬e¯k©kÓ:�Ø'¬´U·[±[Õ(AÐ'BÀ"ÀUÀGÈ2ÈeÈWÐTUÐVÐVØ×)Ñ)¨+Ó6ˆEØ#¤L°Õ$=Ð#>¸bÀÀÀrÈ%ÈÐPQÐRÐRØ”E—J‘JÔØÐ%AÔAÜ$ØMóð ð ×)Ñ)Ó+Ð,Ð-ØˆrW   c                óŽ   <€ V ^8„  d   QhRS[ P                  S[,          R,          RS[R,          RS[P
                  R,          /# )r’   r“   Nr3  r¯   )r”   r•   rG   ro   rƒ   r‡   )r–   r  s   "€rX   r—   rÇ  >  sK   ø€ ÷ y&ñ y&ñ �z‰z™NÕ*¨TÕ1ðy&ñ # T�kðy&ñ —;‘; Õ%ñy&rW   c                ó4
  € V\         P                  8H  pVR8X  dJ   V P                  '       d    RV RV R\        V P                  4       R2# V'       d   V RV 2# RV RV R2# VR8X  dJ   V P                  '       d    RV RV R\        V P                  4       R2# V'       d   V R	V 2# R
V RV R2# VR8X  dŠ   V'       d;   V P                  '       d    RV R\        V P                  4       RV R2# RV RV R2# V P                  '       d    RV RV R\        V P                  4       R2# V'       d   RMRp	V RV	 RV 2# VR8X  d9   V P                  '       d    RV RV R\        V P                  4       R2# V RV 2# VR8X  d9   V P                  '       d    RV RV R\        V P                  4       R2# V RV 2# VR8X  dƒ   V'       dA   V P                  '       d#   RV RV R\        V P                  4       RV R2	# RV RV RV R2# V P                  '       d    RV RV R\        V P                  4       R2# RV RV R2# VR8X  dq   \	        V\
        4      '       d   Vw  r«pM\        W4      w  r«pV P                  '       d&   RV RV
 RV RV R\        V P                  4       R2# RV RV
 RV RV R2	# VR'9   Ed¥   Vf   \        R4      h\        V,          pTpV\         P                  8X  dm   \        \         P                  ,          p\         P                  p\	        V\        4      '       d,   VP                  '       d   \        V P                  V34      w  pV P                  V4      pV P                  \         P                  4      pRpRpVev   \	        V\        4      '       d!   VP                  '       d   RV 2pMGR pRV 2pM?Vf   \        R!4      hR\!        V4      P#                  4        2pRV P%                  V4       2pV P                  '       d/   V R"V RV RV V R#V RV V R\        V P                  4       R2# V R"V RV RV V R#V RV V R2# VR$8X  d    \	        V\        4      '       dR   VP&                  \         P                  8w  d   \        R%4      h\        \(        P*                  P                  V34      w  pV P                  '       d    R&V RV R\        V P                  4       R2# V RV 2# \,        h)(z>Emit the C++ expression for combining vector reduction values.r`   zmax_masked_reduce(r¡   r›   rÓ  r	  r_   zmin_masked_reduce(rË  r  ra   r™   rš   zsum_masked_reduce(rœ   r[   r�   rb   zprod_masked_reduce(rž   rc   zxor_sum_masked_reduce(rŸ   rg   r¢   ri   r£   z}, r¤   zexpected src_dtype is not NonerS   z, falsez)expected horizontal_reduction is not Nonez_combine_vec<rÀ  rf   z'expected next_value.dtype == torch.boolzany_masked_reduce(r�   )rƒ   ro   rQ  rE   r§   r¨   r©   r…   rH   rm   rG   r«   rO   r°  rR  r•  rp   rÙ  r/  r‡   r5   re  r  )r1  r†   r¬   r­   r®   r“   r3  r¯   r°   r±   r²   r³   r´   rˆ   r`  r*  r+  Út_extraÚ	arg_extras   &&&&&&&&           rX   r7  Ú"CppVecKernel.reduction_combine_vec>  sª  € ð œuŸz™zÑ)ˆØ˜UÔ"Ø�~�~ˆ~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_÷ ð �e˜3˜z˜lÐ+ðð -¨S¨E°°J°<¸qÐAðð
 ˜uÔ$Ø�~�~ˆ~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_÷ ð �e˜3˜z˜lÐ+ðð -¨S¨E°°J°<¸qÐAðð
 ˜uÔ$ßØ—>—>�>Ø1°*°¸RÄÈDÏNÉNÓ@[Ð?\Ð\_Ð`jÐ_kÐklÐmÐmà1°*°¸SÀÀÈAÐNÐNà—>—>�>Ø/°¨u°B°z°lÀ"Ä[ÐQU×Q_ÑQ_ÓE`ÐDaÐabÐcÐcç)0¡#°c�KØ!˜U ! K =°°*°Ð>Ð>Ø˜vÔ%Ø�~�~ˆ~Ø,¨S¨E°°J°<¸rÄ+ÈdÏnÉnÓB]ÐA^Ð^_Ð`Ð`à˜˜c * Ð.Ð.Ø˜yÔ(Ø�~�~ˆ~Ø/°¨u°B°z°lÀ"Ä[ÐQU×Q_ÑQ_ÓE`ÐDaÐabÐcÐcà˜˜c * Ð.Ð.ØÐ/Ô/ßØ—>—>�>Ø-¨c¨U°"°Z°LÀÄ;ÈtÏ~É~ÓC^ÐB_Ð_bÐcmÐbnÐnoÐpÐpà-¨c¨U°"°Z°LÀÀJÀ<ÈqÐQÐQà—>—>�>Ø-¨c¨U°"°Z°LÀÄ;ÈtÏ~É~ÓC^ÐB_Ð_`ÐaÐaà-¨c¨U°"°Z°LÀÐBÐBØÐ0Ô0Ü˜*¤e×,Ò,à#-Ñ �™&ô $5°^Ó#PÑ �˜&Ø�~�~ˆ~Ø)¨#¨¨d°4°&¸¸2¸$¸bÀÀÈÌ[ÐY]×YgÑYgÓMhÐLiÐijÐkÐkà)¨#¨¨d°4°&¸¸2¸$¸bÀÀÈÐLÐLØÐ3Õ3ØÒ Ü$Ð%EÓFÐFÜ! )Õ,ˆFØ%ˆMØœEŸJ™JÔ&ä%¤e§k¡kÕ2�Ü %§¡�ä˜j¬.×9Ò9¸j×>O×>OÐ>OÜ$8¸¿¹È
À}Ó$U‘M�ZØ×)Ñ)¨-Ó8ˆEØ×)Ñ)¬%¯+©+Ó6ˆEØˆGØˆIØÒ Ü˜e¤^×4Ò4Ø—|—|�|Ø&(¨¨ L™	à"+˜Ø&(¨¨ L™	à+Ò3Ü,ØGóð ð !#¤3Ð';Ó#<×#BÑ#BÓ#DÐ"EÐF�GØ"$ T×%AÑ%AÀ%Ó%HÐ$IÐ J�IØ�~�~ˆ~à%Ð& m°F°8¸2¸e¸WÀBÀuÀgÈgÈYð WØ�u˜B˜z˜l¨9¨+°R¼ÀDÇNÁNÓ8SÐ7TÐTUðWðð
 )Ð)¨°v°h¸bÀÀÀrÈ%ÈÐQXÐPYÐY[Ð\_Ð[`Ð`bÐcmÐbnÐoxÐnyÐyzÐ{Ð{Ø˜uÔ$Ü˜*¤n×5Ò5Ø×#Ñ#¤u§z¡zÔ1Ü(Ð)RÓSÐSÜ 4´Q·X±X×5EÑ5EÈ
À}Ó U‘�Ø�~�~ˆ~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_à˜˜c * Ð.Ð.ä%Ð%rW   c           	     óR  <€ \        V\        4      '       g   \        R 4      hVP                  f   \        R4      hVP                  '       g?   \        V\        4      '       d   VP                  '       d   RV R2p\
        S	V `  WW44      # TpTpV'       d"   V P                  VP                  4       RV R2pV'       d"   V P                  VP                  4       RV R2pV'       d#   V'       d   RV RV RV RV R2	pV RV RV 2pM8V'       d   V RV 2pV RV 2pM!V'       g   \        R4      hV RV 2pV RV 2pV P                  VP                  4       RV R2pV'       d=   VP                  '       g"   V P                  VP                  4       R	V R2pRV R
V R2pV P                  '       dT   V P                  VP                  4       RV P                  VP                  4       RV R\        V P                  4       R2pRV R2pV P                   RV RV R2# )z(expected isinstance(var, CppCSEVariable)zexpected var.dtype is not Noner’  z).all_masked()r›   rw  z) & (rÂ   zexpected upperr+  z) | ~(rú  z::from(1), (r¥   z, "index out of bounds: z"))r§   rG   r…   r‡   r«   r/  rà  r,  rf  rQ  rE   r•  )
r1  r¬   rÙ  rÚ  r¯  Úlower_scalarÚupper_scalarrÿ  Ú
cond_printr4  s
   &&&&&    €rX   rà  ÚCppVecKernel.indirect_assert¹  sB  ø€ Ü˜#œ~×.Ò.Ü Ð!KÓLÐLØ�9‰9ÒÜ Ð!AÓBÐBØ�z�zˆzÜ˜$¤×/Ò/°D·K·K°KØ˜4˜& Ð/�Ü‘7Ñ*¨3°uÓCÐCØˆØˆßØ×)Ñ)¨#¯)©)Ó4Ð5°Q°u°g¸QÐ?ˆEßØ×)Ñ)¨#¯)©)Ó4Ð5°Q°u°g¸QÐ?ˆEß—UØ�u�g˜T #  e¨C¨5°°E°7¸!Ð<ˆDØ(˜>¨¨c¨U°#°l°^ÐD‰JßØ�W˜D  Ð&ˆDØ(˜>¨¨c¨UÐ3‰JçÜ$Ð%5Ó6Ð6Ø�U˜#˜e˜WÐ%ˆDØ˜5  L >Ð2ˆJØ×%Ñ% c§i¡iÓ0Ð1°°4°&¸Ð:ˆßØ—;—;�;Ø×-Ñ-¨c¯i©iÓ8Ð9¸ÀÀÀaÐH�à�t�f˜F 4 &¨Ð*ˆDØ�>�>ˆ>à×&Ñ& s§y¡yÓ1Ð2°&¸×9LÑ9LÈSÏYÉYÓ9WÐ8Xð YØ�V˜3œ{¨4¯>©>Ó:Ð;¸1ð>ð ð �4�&˜Ð'ˆØ×&Ñ&Ð' q¨¨Ð.FÀzÀlÐRTÐUÐUrW   c                óH  <€ \        V\        4      '       g   \        R 4      hVP                  '       g   \        S
V `  WW44      # \        V,          pV P                  V4      p\        V,          pV P                  V4      pRV R2p	V\        P                  8w  d5   V\        P                  8X  d    V P                  V4       RV RV RV R2p	V	# V\        P                  8X  d#   V\        P                  8w  d   V RV RV R2p	V	# W28w  d–   Rp	V'       dN   V\        P                  \        P                  39   d)   V\        P                  \        P                  39   d   R	p	MR
p	Yhu;8X  d   ^8X  d   M MV	RV RV R2,           p	V	# V	RV RV RV RV RV R2,           p	V	# )z(expected isinstance(src, CppCSEVariable)r’  r›   z::from<rZ  rÀ  z.to<r€   rS   zat::vec::round_convertzat::vec::convertr   )r§   rG   r…   r«   r/  r­  rH   rR  rƒ   ro   rf  rm   rn   r!  rÅ  )r1  r©  r‡   r¯   r¼  Úsrc_cpp_typerR  Údst_cpp_typeÚdst_num_vectorsr·  r4  s   &&&&&     €rX   r­  ÚCppVecKernel.get_to_dtype_exprá  s®  ø€ Ü˜#œ~×.Ò.Ü Ð!KÓLÐLØ�z�zˆzÜ‘7Ñ,¨S¸ÓMÐMÜ# IÕ.ˆØ×/Ñ/°	Ó:ˆÜ# EÕ*ˆØ×/Ñ/°Ó6ˆØ�3�%�qˆzˆØœŸ
™
Ô" u´·
±
Ô':Ø×)Ñ)¨)Ó4Ð5°W¸\¸NÈ!ÈOÐK\Ð\^Ð_bÐ^cÐcdÐeˆDð( ˆð' œ%Ÿ*™*Ô$¨´%·*±*Ô)<Ø�U˜$˜|˜n¨A¨oÐ->¸cÐBˆDð$ ˆð# ÔØˆDçØ¤%§+¡+¬u¯|©|Ð!<Ô<ØœeŸj™j¬%¯+©+Ð6Ô6à/‘à)�ØÖ6°Q×6Ø  , ¨r°#°°aÐ8Õ8�ð ˆð Ø˜,˜ q¨Ð(9¸¸<¸.ÈÈ/ÐIZÐZ\Ð]`Ð\aÐabÐcõdð ð ˆrW   )r†  rû  rQ  rÀ  r=  rÄ  r  )NNFr½  )"r  r  r  r  r  r3  r0  r>  rR  r¿  r,  rƒ   rm   rf  r  rð  r@  r°  r   rü  r/  r=  r@  r)  r?  r6  r5  r„   r7  rà  r­  r
  r  rl  rm  s   @@rX   r(  r(  /  s÷   ù‡ € Ø€I÷C÷&4ð 4÷"ð ÷Rð R÷
Pð Pð 38·+±+÷ Hò H÷Nð N÷#ò #÷JXò X÷tó ÷*,ò ,ô\+<÷Zð òNò`%P÷Nð ÷*ð ò#òJð6 Ø48Ø,0Ø(-¯©÷y&ò y&÷v&V÷P÷ rW   r(  c                   ó²   a a€ ] tR tRt oRt]tRV 3R lltR tR t	RR lt
V3R lV 3R lltRV 3R	 lltR
 tV 3R ltV3R lR ltV3R lR ltRtVtV ;t# )ro  i  a  
A vector kernel that handles the 2d tiles with the tile size defined in `tiling_factor` on
the inner-most loop level and one of the outer loop level (`outer_tiling_idx`). When the data
tile is accessed in a contiguous way from the outer loop axis, a transposition is applied on the
tile to make the access contiguous from the inner-most loop axis. Then, the same vectorization
logic from its parent `CppVecKernel` is leveraged for load/store/compute. The transposed tile load
and store are generated into kernel.preloads and kernel.poststores buffers.

The loop structure looks like below:
for ...
  for i_outer ...
    for ...
      for inner_most ...
        // generated by CppTile2DKernel
        float tmp0[16*16]; at::vec::transpose_mxn<...>(tmp0, in_ptr0 + ..., ...); // into kernel.preloads
        float tmp1[16*16]; // into kernel.preloads
        for i_inner ... { // the kernel inner loop
          vectorized loads/compute/stores (e.g., load tmp0, store tmp1) // into kernel.loads/compute/stores
        }
        at::vec::transpose_mxn(out_ptr0 + ..., tmp1, ...) // into kernel.poststores
      for inner_most ... (tail)
        // generated by CppVecKernel
        ...
  for i_outer ... (tail)
    for ...
      for ...
        // generated by CppKernel
        ...
c                ó²   <€ \         SV `  VVVV^,          V4       W@n        WPn        W`n        V'       d   TMTV n        V'       d   TMTV n        RV n        R# )r$  TN)r/  r0  Útiling_indicesÚinner_tail_sizeÚouter_tail_sizeÚinner_num_elemsÚouter_num_elemsÚinner_is_tiling_idx)r1  r™  r�  rÀ  rw  rx  ry  r4  s   &&&&&&&€rX   r0  ÚCppTile2DKernel.__init__$  sZ   ø€ ô 	‰ÑØØØØ˜1ÕØô	
ð -ÔØ.ÔØ.Ôß2A™À}ˆÔß2A™À}ˆÔØ#'ˆÖ rW   c                óT   € \        V P                  V P                  ,           R 24      # )rN  )r.   r<  Ú	outer_idxr7  s   &rX   Úinner_itervarÚCppTile2DKernel.inner_itervar;  s"   € Ü! T§]¡]°4·>±>Õ%BÐ$CÀ6Ð"JÓKÐKrW   c                óò  € V P                   V P                  ,          pV P                   V P                  ,          p\        V\        P
                  4      '       d   R # \        WV P                  4      p\        WV P                  4      pV P                  RJ ;'       d[    V^8H  ;'       dN    VP                  V4      ;'       d5    VP                  V4      '       * ;'       d    VP                  V4      '       * # ©FN)
r<  r  r=  r   r   rÞ  r%   rÀ  r±  Úhas)r1  r“   Ú	outer_varÚ	inner_varÚouter_strideÚinner_strides   &&    rX   Úneed_vec_transposeÚ"CppTile2DKernel.need_vec_transpose>  sÓ   € Ø—M‘M $§.¡.Õ1ˆ	Ø—M‘M $§/¡/Õ2ˆ	ô ˜u¤d§h¡h×/Ò/ÙÜ*¨5¸T×=OÑ=OÓPˆÜ*¨5¸T×=OÑ=OÓPˆà�O‰O˜tÐ#÷ 0ð 0Ø Ñ!÷0ð 0à—	‘	˜)Ó$÷0ð 0ð !×$Ñ$ YÓ/Ô/÷0ð 0ð !×$Ñ$ YÓ/Ô/ð	
rW   c                ó  € \         P                  P                  V4      pV P                  pV R \	        V4       2pRp	\	        \        W0P                  V P                  ,          V P                  4      4       p
\	        V P                  4       pV'       d   Y˜r˜YºrºRpV P                  V,          '       d   V P                  V P                  ríMV P                  V P                  ríV'       d
   VR8X  d   RMRp\        V\        P                  4      '       d   VP                  '       d3   \        V\        P                  4      '       dL   VP                  '       g:   R\         V,           RV RV R	V
 R	V	 R	V R	\	        V4       R	\	        V4       R
2pM8R\         V,           R\	        V4       R\	        V4       RV RV R	V
 R	V	 R	V R
2pV'       d   V P"                  P%                  4       pMgV P"                  P'                  V4      '       g*   V P"                  P)                  V P*                  VRR7      pMRpV P"                  P-                  V4      pV'       dC   \         V,          pRV RV R2pV RV RV RV RV R2
pV P*                  P/                  V4       VP1                  R\3        V4      4      pV'       d(   V P4                  P/                  \7        VV4      4       V# V P*                  P/                  V4       V# )ró   Ú__place_holder__Trø  ÚtrueÚfalseztranspose_mxn<rZ  rÀ  r¡   rÞ   Fr0  zalignas(std::max(std::size_t(z), alignof(z)))r�   rê  r\   rà   )r5   rí  rî  rÀ  rE   r%   r<  r=  rû  r|  rz  r{  r§   r”   r•   r&  rH   r®  ra  Úcontainsr¯  rŽ  ÚgetrÉ   rO  rp   r�  r;   )r1  rÎ   r¬   r“   Úis_storeÚ
store_moder‡   Úfactorr©  r¬  Úld_srcÚld_dstÚneed_defineÚMÚNrø  Úload_or_storeÚtile_varÚ	cpp_dtypeÚalignasÚdefine_lines   &&&&&&               rX   Úgen_transposed_tile_load_storeÚ.CppTile2DKernel.gen_transposed_tile_load_storeO  sÈ  € ô —‘×!Ñ! $Ó'ˆØ×#Ñ#ˆØ��Sœ UÓ+Ð,Ð-ˆØ ˆÜÔ 3°E¿=¹=ÈÏÉÕ;YÐ[_×[mÑ[mÓ nÓoÐpˆÜ §¡Ó/Ð0ˆßØ�Ø#�FàˆØ×#Ñ# h×.Ô.Ø×'Ñ'¨×)=Ñ)=‰qð ×$Ñ$Ø×$Ñ$ð ÷ !)¨j¸LÔ.H‘VÐPWˆ
Ü�qœ%Ÿ*™*×%Ò%¨a¯k¯k¨kÜ�qœ%Ÿ*™*×%Ò%¨a¯k¯k¨kð !¤¨eÕ!4Ð 5°Q°z°lð CØ�5˜˜6˜( " S E¨¨F¨8°2´kÀ!³nÐ5EÀRÌÐTUËÐGWÐWYð[ñ ð !¤¨eÕ!4Ð 5°Q´{À1³~Ð6FÀaÌÐTUËÐGWÐWXÐYcÐXdð eØ�5˜˜6˜( " S E¨¨F¨8°2ð7ð ÷ Ø—x‘x—‘Ó(‰HØ—‘×"Ñ" =×1Ò1Ø—x‘x×(Ñ(¨¯©¸ÈUÐ(ÓS‰HàˆKØ—x‘x—|‘| MÓ2ˆHçÜ$ UÕ+ˆIð 6°f°X¸[ÈÈÐSVÐWˆGØ$˜I Q y k°°8°*¸A¸f¸XÀQÀvÀhÈbÐQˆKØ�M‰M×#Ñ# KÔ0à%×-Ñ-Ð.@Ä#ÀhÃ-ÓPˆßØ�O‰O×%Ñ%¤l°4¸Ó&GÔHð ˆð �M‰M×#Ñ# MÔ2àˆrW   c                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# rç  rè  )r–   r  s   "€rX   r—   ÚCppTile2DKernel.__annotate__‹  s   ø€ ÷ 1ñ 1™ð 1¡U§Z¡Zñ 1rW   c                ó‚  <€ V P                   P                  V4      pV P                  V4      pV P                  4       pV P	                  V4      '       dÍ   V P                  WVR R7      pV R\        W@P                  ,          4       2p\        P                  P                  V4      pV P                  V^ V4      pV P                  P                  V P                  W‡R7      p	V	P                  RWV3/ 4       \!        V	\"        4      '       g   \%        R4      hRV	n        V	# V P)                  V4      p
\*        SV `Y  W4      # )F)r‘  ró   rJ  r°  r2  T)r™  rì  r£  r€  r‰  rž  rE   rû  r5   rí  rî  rð  r®  r¯  r«  r±  r§   rG   r…   r«   rp  r/  r°  )r1  rÎ   r“   r¬   rd  rš  rï  r‡   rÏ   r¸  r¹  r4  s   &&&        €rX   r°  ÚCppTile2DKernel.load‹  s  ø€ Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆà×"Ñ"Ó$ˆØ×"Ñ" 5×)Ò)Ø×:Ñ:Ø˜5¨5ð ;ó ˆHð "˜
 #¤k°%¿.¹.Õ2HÓ&IÐ%JÐKˆGÜ—G‘G×%Ñ% dÓ+ˆEØ×*Ñ*¨7°A°uÓ=ˆDØ—X‘X×&Ñ& t§z¡z°4Ð&ÓEˆFØ×!Ñ! &¨4°uÐ*=¸rÔBÜ˜f¤n×5Ò5Ü$Ð%RÓSÐSØ ˆFŒMØˆMà×/Ñ/°Ó6ˆIÜ‘7‘< Ó0Ð0rW   c                óê  <€ R V9  d   \        R4      h\        V\        4      '       g   \        V4      hVP                  '       g   V P	                  V4      pV P
                  P                  V4      pV P                  4       pV P                  V4      pV P                  V4      '       Ed    V P                  WVRVR7      pV R\        W`P                  ,          4       2pV P                  '       gl   \        P                  P!                  V4      \"        \$        P&                  \$        P(                  \$        P*                  \$        P,                  .,           9   d    V RV R\        V P                  4       R2p	MV RV R2p	V P.                  P1                  \3        W4      4       R	# VR8H  ;'       d    V P5                  4       p
V
'       d   TMV P7                  V4      p\8        SV `u  WW44       R	# )
rõ  rö  T)r‘  r’  ró   r  r¡   rÞ   rø  N)r…   r§   rG   r«   r)  r™  rú  r€  r£  r‰  rž  rE   rû  rQ  r5   rí  rî  r‚   rƒ   rÅ  r!  r"  r#  r   rÉ   r;   rò  rp  r/  rü  )r1  rÎ   r“   rù   rû  r¬   rd  rš  ÚstorebufrÏ   Úvec_atomic_addr¹  r4  s   &&&&&       €rX   rü  ÚCppTile2DKernel.store¢  sŸ  ø€ Ø˜ÔÜ Ð!9Ó:Ð:Ü˜%¤×0Ò0Ü  Ó'Ð'Ø�|�|ˆ|à—N‘N 5Ó)ˆEà�i‰i×Ñ˜tÓ$ˆà×"Ñ"Ó$ˆØ×$Ñ$ UÓ+ˆØ×"Ñ" 5×)Ó)Ø×:Ñ:Ø˜5¨4¸Dð ;ó ˆHð #˜ 3¤{°5¿>¹>Õ3IÓ'JÐ&KÐLˆHØ�~�~ˆ~¤§¡×!2Ñ!2°4Ó!8¼MÜ—‘Ü—
‘
Ü×#Ñ#Ü×!Ñ!ð	Mõ =ô "ð  ˜ ¨ z°´KÀÇÁÓ4OÐ3PÐPRÐS‘à˜ ¨ z°Ð4�Ø�K‰K×!Ñ!¤,¨tÓ":Ö;ð " \Ñ1×UÐU°d×6SÑ6SÓ6UˆNß!/™°T×5LÑ5LÈUÓ5SˆIÜ‰G‰M˜$¨5Ö7rW   c                ó  € V P                  4       pV P                  '       d4   VP                  R V RV R\        V P                  4       RV R2	4       R# VP                  R V RV R\        V P
                  4       RV R2	4       R# )r  r  rÂ   rÃ   rÄ   N)r€  r|  rÉ   rE   r{  rz  )r1  r¹   rd  s   && rX   r©  Ú#CppTile2DKernel.codegen_inner_loopsÅ  s’   € Ø×"Ñ"Ó$ˆØ×#×#Ð#Ø�N‰NØ˜U˜G 6¨%¨°´KÀ×@TÑ@TÓ4UÐ3VÐVXÐY^ÐX_Ð_bÐcöð �N‰NØ˜U˜G 6¨%¨°´KÀ×@TÑ@TÓ4UÐ3VÐVXÐY^ÐX_Ð_bÐcörW   c                ó²  <€ \         SV `  W4      pV P                  ^,          V P                  8  d   V P                  M\	        V P                  4      w  V n        V n        V P                  V P                  ^ ,          8X  d,   V P                  V n        V P                  V n
        RV n        V# V P                  V n        V P                  V n
        RV n        V# )r$  FT)r/  rF  rw  r~  Úreversedr  r=  ry  rQ  r{  rû  r|  rx  rz  )r1  ÚgroupÚreduction_groupr7  r4  s   &&& €rX   rF  ÚCppTile2DKernel.set_rangesÐ  s¾   ø€ Ü‰wÑ! %Ó9ˆð ×"Ñ" 1Õ%¨×(<Ñ(<Ô<ð ×Òä˜$×-Ñ-Ó.ñ 	(ˆŒ˜œð
 �?‰?˜d×1Ñ1°!Õ4Ô4Ø!×1Ñ1ˆDŒNØ!×1Ñ1ˆDŒNØ',ˆDÔ$ð
 ˆð "×1Ñ1ˆDŒNØ!×1Ñ1ˆDŒNØ'+ˆDÔ$ØˆrW   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r½  rï  )r–   r  s   "€rX   r—   r¡  â  s#   ø€ ÷ 
ñ 
©¯
©
ð 
±u·z±zñ 
rW   c                óZ   € V P                  VV P                  V P                  4       R 7      # )r  )rº  r  r€  r¿  s   &&rX   rp  Ú"CppTile2DKernel.transform_indexingâ  s0   € Ø×+Ñ+ØØŸ™Ø×%Ñ%Ó'ð ,ó 
ð 	
rW   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r½  rï  )r–   r  s   "€rX   r—   r¡  é  s#   ø€ ÷ .ñ .©U¯Z©Zð .¹E¿J¹Jñ .rW   c                ó$   € V P                  V4      # r  )rp  r¿  s   &&rX   r/  Ú'CppTile2DKernel._adjust_argreduce_indexé  s   € Ø×&Ñ& uÓ-Ð-rW   )
r|  rz  rx  rû  r  r{  ry  rQ  r=  rw  r¾  r  )r  r  r  r  r  rm  r3  r0  r€  r‰  rž  r°  rü  r©  rF  rp  r/  r
  r  rl  rm  s   @@rX   ro  ro    sZ   ù‡ € ñð< #€I÷(ò.Lò
ô":÷x1ó 1÷.!8òF	õ÷$
ð 
÷.÷ .ð .rW   ro  c                ót   € V ^8„  d   QhR\         R\        \        P                  R,          \        3,          /# )r’   Ú_bodyr¾   N)r   r¨   rƒ   r‡   ro   )r–   s   "rX   r—   r—   í  s-   € ÷ -$ñ -$¤ð -$¬e´E·K±KÀ$Õ4FÌÐ4LÕ.Mñ -$rW   c                óò  € V P                   .\        V P                  P                  4       4      ,           pRpRpV EF5  pVP                  P
                   EF  pVP                  R8X  g   VP                  R9   d   K(  VP                  R	9  d   Rp\        VR4      '       dÈ   VP                  '       d¶   \        P                  VP                  9  d   \        R4      hVP                  \        P                  ,          pVP                  '       d   VP                  \        9  d   RpKÕ  Ve-   W&P                  8w  d   \        P                   ! R4       EK  EK  VP                  pEK  RpEK  	  EK8  	  W#3# )
zÎ
Returns the low precision data type (torch.float16/torch.bfloat16) contained in the nodes
and if all the nodes can codegen with this data type without converting to float.
Otherwise returns None and True.
NFÚplaceholderTr{  z.expected OptimizationContext.key in _node.metaz.bf16 and fp16 are mixed in the scheduler node.)Ú	get_indexr¥  r©  )r°  rü  rÊ  rÔ  rú  )Ú
root_blockr$  Ú	subblocksr€  rí  Únodesr–  Útargetrª   r{  rA   rz  r…   r‡   r‚   ÚwarningsÚwarn)r¶  Ú
sub_blocksÚ_lowp_fp_typeÚ	_use_fp32Ú	sub_blockr3  rv  s   &      rX   Úget_loop_body_lowp_fprÄ  í  s2  € ð ×"Ñ"Ð#¤d¨5¯?©?×+AÑ+AÓ+CÓ&DÕD€Jà(,€MØ€IÜˆ	Ø—_‘_×*Õ*ˆEØ�x‰x˜=Ô(¨E¯L©Lð =ô -ñ
 ð �|‰|ð $ô ð !�	ä�u˜f×%Ò%¨%¯*¯*¨*Ü&×*Ñ*°%·*±*Ô<Ü(ØHóð ð 05¯z©zÔ:M×:QÑ:QÕ/R�Ø—}—}�}¨¯©¼]Ô(JØ $’IØ"Ò.Ø$¯©Ô5Ü ŸšÐ&V×Wò 6ð %,§M¡M“Mà “	ôA +ñ  ðF Ð#Ð#rW   c                   ó„   a € ] tR tRt o RtRt]P                  ! ]R7      t	]P                  ! ]R7      t
^ tV 3R ltRtV tR# )ÚKernelOpStatsi  z0Op statistics collected during tiling selection.g¸…ëQ¸¾?)Údefault_factoryc                ó†   <€ V ^8„  d   Qh/ S[ S[,          ;R&   S[S[S[3,          ;R&   S[S[S[3,          ;R&   S[;R&   # )r’   ÚOVERHEAD_RATIO_THRESHOLDÚ
op_counterÚnon_contig_indexing_op_counterÚmask_op_count)r   rm   r  rp   rk   )r–   r  s   "€rX   r—   ÚKernelOpStats.__annotate__  sO   ø‡ ‚ ñ '¡u�oÑ4ñ ñ ‘S™#�X•ÑHñ ñ %)©©c¨¥Nñ ñ ñ Ñò rW   rV   N)r  r  r  r  r  rÉ  ÚdataclassesÚfieldr  rÊ  rË  rÌ  r	  r
  r  r  s   @rX   rÆ  rÆ    sB   ø‡ € á:ð
 15Ðà!,×!2Ò!2À4Ô!H€JØ5@×5FÒ5FØô6Ð"ð €M÷ ƒ rW   rÆ  c                   ó<   a € ] tR tRt o RtV 3R lR ltR tRtV tR# )ÚTilingSelecti-  zŒ
Implement the heuristic to select the tiling factors and tiling indices.
In the future, we can implement advanced heuristic in a subclass.
c                óZ   <€ V ^8„  d   QhRS[ S[S[,          S[S[,          S[3,          /# r�  )r¨   r$  rk   rÆ  )r–   r  s   "€rX   r—   ÚTilingSelect.__annotate__3  s.   ø€ ÷ v'ñ v'ñ 
‰t‘C�y™$™s�)¡]Ð2Õ	3ñ	v'rW   c           	     ó  a!€ \        V4      p\        V4      pV'       g   \        R 4      h\        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       d   . . \	        4       3# \
        P                  p\        V^ ,          4      ^ ,          o!S!'       dQ   \        ;QJ d&    V!3R lVR,           4       F  '       d   K   RM	  RM! V!3R lVR,           4       4      '       d   S!p\        P                  ! 4       P                  VR7      pV P                  WV4      pV'       EdÅ   \        VR R7      w  r‰\        V4      \        V	4      ,           p
\	        4       p\        P                   P"                  '       Ed@   R	 pR
 pR p\%        \'        V
4      4       Uu. uF  p\)        \*        P,                  V4      NK  	  pp\'        V4      pVRV VVR ppV EF  pVP.                  .\1        VP2                  P5                  4       4      ,           pV EFÆ  pVP6                  P8                   EF§  pVP:                  R9   d¯   VP:                  R9   d   ^M^pVP<                  P?                  VV34      VP@                  V,          P@                  ^ ,          ,          pV! VV4      '       dE   V! VVWg4      pVP:                  R9   d   Vf&   MVR9  d   V! VP:                  VPB                  4       \E        VP:                  \F        4      '       g   Kå  VP:                  R9   d   V;PH                  ^,          un$        VP:                  PK                  R4      '       d   EK3  VP:                  R9   d   EKG  VP:                  VPL                  9  d   ^VPL                  VP:                  &   EK~  VPL                  VP:                  ;;,          ^,          uu&   EKª  	  EKÉ  	  EK	  	  \O        VPL                  P5                  4       4      p\O        VPB                  P5                  4       4      p^#pVV8¼  g    V^ 8”  d   VV,          VPP                  8¼  d   . . V3# V	'       g_   V'       dW   \'        V4      ^8X  dG   \S        W‡^ ,          ,          .4      '       g(   W‡^ ,          ,          V^,          8  d   V^
8  d   . . V3# V\T        9   dû   \        P                  ! 4       P                  VR7      pV FÏ  pV^ 8  d   V\'        V
4      ,           pV^ 8  g   V\'        V
4      8¼  d   K5  \S        V
4      '       dq   \V        P6                  PX                  P[                  V
V,          ^ R7      p V V8  d6   \V        P6                  PX                  P]                  V V4       V^,          p MK¶  V
V,          V8  g   KÆ  V^,          p M	  \'        V4      ^8X  d   V.W{3# \'        V4      ^8X  d   Wf.W{3# . . \	        4       3# u upi )zexpected all_dtypesc              3   ó2   "  € T F  q\         9  x € K  	  R # 5ir  )ÚVECTORIZABLE_DTYPES)r  r‡   s   & rX   r  Ú-TilingSelect.select_tiling.<locals>.<genexpr>=  s   é € ÐH¹Z°EÔ/Ö/»Zùs   ‚TFc              3   óN   <"  € T F  p\        V4      ^ ,          S8H  x € K  	  R# 5ir`  )rÄ  )r  Ú	loop_bodyÚ_lowp_fp_dtypes   & €rX   r  r×  A  s'   øé € ð "
á,�	ô # 9Ó-¨aÕ0°NÖBÛ,ùó   ƒ"%r#  rJ  c                 ó&   € \        V ^,          4      # rñ  ©rÛ   ©Úsizess   &rX   r&  Ú,TilingSelect.select_tiling.<locals>.<lambda>N  s   € ´#°e¸Aµh´-rW   ©rz  c                 ód   € W^ ,          ,          p\        WV4      pVP                  '       d   V# R# r`  )r%   r&  )r“   r<  rÀ  rw  rÄ  rB  s   &&&&  rX   Ú_try_get_strideÚ3TilingSelect.select_tiling.<locals>._try_get_strideU  s3   € ð '°aÕ'8Õ9�GÜ0°ÀÓO�FØ%+×%5×%5Ð%5˜6Ð?¸4Ð?rW   c                 óF   € W9  d   ^W&   R# W;;,          ^,          uu&   R# ©r$  NrV   )Ú	node_namerË  s   &&rX   Ú_update_negative_op_countÚ=TilingSelect.select_tiling.<locals>._update_negative_op_count_  s!   € ð !ÔFØDEÐ6ÓAà6×AÀQÕFÕArW   c                 óÒ   € \        V4      ^8H  ;'       dS    \        V 4      ^ 8„  ;'       d=    V^ ,          ^ 8¼  d
   V^ ,          MV^ ,          \        V 4      ,           \        V 4      8  # rñ  rÝ  )r<  rw  s   &&rX   Ú_is_valid_indicesÚ5TilingSelect.select_tiling.<locals>._is_valid_indicesg  sl   € ô
 ˜NÓ+¨qÑ0÷ (ð (Ü ›M¨AÑ-÷(ð (ð  .¨aÕ0°AÔ5ð +¨1Ö-à!/°Õ!2´S¸³]Õ!Bä˜h›-ñ(ð	rW   NÚmasked_subblockr˜  )r¥  r©  r°  rü  )r¥  r©  ©r   r$  )r—  rµ  )r3   rú  rž  r¹  )/rC   rB   r…   rf   rÆ  rƒ   rm   rÄ  r#  r   rÃ  Ú	nelementsÚ_select_tiling_indicesr`   r¨   r   rù  Úenable_tiling_heuristicsrQ  rÛ   r/   r   rC  rº  r$  r»  r€  rí  r¼  r½  r°  Úindexing_from_argsr™  rË  r§   rp   rÌ  Ú
startswithrÊ  ra   rÉ  r)   r‚   r5   r  rI  Úcheck_lt)"r1  Úfn_listÚvar_sizes_listÚloop_bodiesÚ
all_dtypesr‡   rÀ  rw  r¬  r­  r}  Ústatsrã  rè  rë  rv  r<  r~  r7  Úreduction_varsr¶  rÀ  rÃ  r3  Úarg_idxr“   rB  Úop_numÚnon_contig_indexing_op_numÚquantity_thresholdÚfactor_lowpÚtiling_indiceÚ
call_rangerÚ  s"   &&&                              @rX   Úselect_tilingÚTilingSelect.select_tiling3  sG  ø€ ô % WÓ-ˆÜ/°Ó<ˆ
ßÜ Ð!6Ó7Ð7ß‹3ÑH¹ZÓH�3�3Š3ÑH¹ZÓH×HÒHØ�rœ=›?Ð*Ð*Ü—‘ˆÜ.¨{¸1­~Ó>¸qÕAˆßŸc›cô "
à(¨ž_ó"
ŸcŸcšcô "
à(¨ž_ó"
÷ 
ò 
ð #ˆEä#×0Ò0Ó2×<Ñ<À5Ð<ÓIˆØ×4Ñ4Ø ]ó
ˆ÷ ˆ>Ü%(ØÑ$?ô&Ñ"ˆEô   ›,¬¨Ó)?Õ?ˆKä!“OˆEÜ�z‰z×2×2Ñ2ò@òGòô" #¤3 {Ó#3Ô4óá4˜ô 3´4·;±;ÀÖBÙ4ð ð ô #& e£*�àÐ-˜oÐ.Ø˜_Ð-Ð.ð %�ô )�EØ"'×"2Ñ"2Ð!3´d¸5¿?¹?×;QÑ;QÓ;SÓ6TÕ!T�JÜ%/˜	Ø%.§_¡_×%:Õ%:˜EØ$Ÿ|™|ð 0ô  ð (-§|¡|Ð7SÔ'Sñ %&à)*ð !(ð
 )2¯©×(IÑ(IØ%)¨>Ð$:ó)"à"'§*¡*¨WÕ"5×":Ñ":¸1Õ"=õ)? ñ $5°X¸~×#NÒ#NÙ-<Ø(-¨x¸ó.& Fð
 ,1¯<©<Ð;WÔ+Wð )/«à-3¸6Ô-Aá(AØ,1¯L©LØ,1×,PÑ,Pô)*ô  *¨%¯,©,¼×<Ô<ð $)§<¡<Ð3FÔ#FØ$)×$7Ò$7¸1Õ$<Õ$7à$)§L¡L×$;Ñ$;Ð<M×$NÕ$NØ',§|¡|Ø'Q÷(Rð (-§|¡|¸5×;KÑ;KÔ'KØIJ¨×(8Ñ(8¸¿¹Ô(Fà(-×(8Ñ(8¸¿¹×(FÈ!Õ(KÖ(FôU &;ô &0ñ )ô^ ˜U×-Ñ-×4Ñ4Ó6Ó7�Ü-0Ø×8Ñ8×?Ñ?ÓAó.Ð*ð &(Ð"Ø-Ð1CÔCØ˜Q”JØ2°VÕ;Ø×5Ñ5ô6ð ˜r 5˜=Ð(÷ (ßÜ˜NÓ+¨qÔ0Ü,à!°Õ"3Õ4ð÷ò ð
 ¨QÕ/Õ0°=À1Õ3DÔDØ œð ˜r 5˜=Ð(àœÔ%ô *×6Ò6Ó8×BÑBÈÐBÓO�Û%3�MØ$ qÔ(Ø(5¼¸KÓ8HÕ(H˜Ø$ qÔ(¨M¼SÀÓ=MÔ,MÙ Ü'¨×4Ò4Ü%&§W¡W×%5Ñ%5×%GÑ%GØ'¨Õ6Àð &Hó &˜
ð &¨Ô3ÜŸG™G×,Ñ,×5Ñ5°jÀ+ÔNØ,7¸1Õ,<˜MÙ!ñ 4ð % ]Õ3°kÖAØ(3°qÕ(8˜Ùñ &4ô" �>Ó" aÔ'Ø%�¨Ð=Ð=Ü�>Ó" aÔ'Ø%Ð5°~ÐLÐLØ�2”}“Ð&Ð&ùògs   Æ(#Xc                óö  € . p\        W4       Fh  w  rV\        P                  ! V.VO5!  pT\        P                  ! VP
                  VP                  4       Uu. uF  qˆP                  NK  	  up,          pKj  	  \        \        ,          ! 4       p	. p
\        \        ,          ! 4       p\        \        ,          ! 4       pV EF]  pVP                   EFH  p\        P                  ! R VP                  4      '       g   K-  \        WÞV4      pV^ 8X  d   KB  V^8X  dY   V	P                  \        VP                  R,          4      4       V
P!                  \        VP                  R,          4      4       K¡  \"        ;QJ d&    R VP                   4       F  '       d   K   RM	  RM! R VP                   4       4      '       d/   VP                  \        VP                  R,          4      4       EK  VP                  \        VP                  R,          4      4       EKK  	  EK`  	  W›,
          V,
          p\%        VR R7      w  pp\'        V4      \'        V4      ,           p\'        V	4      ^ 8X  d   V^,
          .# V'       d   \)        V4      RR # W›,          V,
          p\)        V	4      p\'        V4      ^8X  d&   VR,          V9   d   VR,          V^,
          8X  d   V# \)        VV
P*                  R7      RR # u upi )	z^d\d+$r#  c              3   óV   "  € T F  p\        V\        P                  4      x € K!  	  R # 5ir  )r   r   ÚSIZE©r  rË  s   & rX   r  Ú6TilingSelect._select_tiling_indices.<locals>.<genexpr>  s    é € ÐSÑ?R¸!œ¨¬4¯9©9×5Ð5Ó?Rùó   ‚')FTc                 ó&   € \        V ^,          4      # rñ  rÝ  rÞ  s   &rX   r&  Ú5TilingSelect._select_tiling_indices.<locals>.<lambda>  ó   € ÄsÈ5ÐQRÍ8Ä}rW   rá  Nr¼  )rc  r   Úextract_read_writesÚ	itertoolsÚchainÚreadsÚwritesr“   r   rk   rÍ  rÐ   rô   rÎ   r%   r¡  r'  r#  r`   rÛ   ÚsortedÚcount)r1  rõ  rö  rÀ  Ú	all_indexÚfnÚ	var_sizesÚrwÚdepÚcontig_varsÚcontig_vars_listÚnon_contig_stride_constÚnon_contig_stride_otherr“   r¬   rB  Úcontig_onlyr¬  r­  Únum_itervarsÚcontig_and_const_strideÚcontig_vars_sorteds   &&&&                  rX   rð  Ú#TilingSelect._select_tiling_indicesë  sM  € ð ˆ	Ü  Ö9‰MˆBÜ×1Ò1°"ÐA°yÓAˆBØ¬y¯ª¸r¿x¹xÈÏÉÔ/SÓTÑ/S¨Ÿ)œ)Ñ/SÑTÕTŠIñ :ô !¤–oÓ'ˆØÐÜ",¬S¦/Ó"3ÐÜ",¬S¦/Ó"3ÐÜˆEØ×)Õ)�Ü—y’y ¨C¯H©H×5Ò5ÙÜ,¨U¸ÓG�Ø˜Q”;ÙØ˜q”[Ø—O‘O¤C¨¯©°­Ó$5Ô6Ø$×+Ñ+¬C°·±¸µÓ,=Ö>ß“SÑS¸v×?RÒ?RÓS—S—S’SÑS¸v×?RÒ?RÓS×SÒSØ+×/Ñ/´°C·H±H¸RµLÓ0A×Bà+×/Ñ/´°C·H±H¸RµLÓ0A×Bô *ñ ð "Õ;Ð>UÕUˆÜ!$ ^Ñ9TÔ!UÑˆˆÜ˜5“z¤C¨Ó$8Õ8ˆÜˆ{Ó˜qÔ à  1Õ$Ð%Ð%ßÜ˜+Ó& r sÐ+Ð+àÕ1Ø#õ#$Ðô $ KÓ0ÐäÐ"Ó# qÔ(Ø" 2Õ&Ð*AÔAØ" 2Õ&¨,¸Õ*:Ô:à%Ð%ÜÐ(Ð.>×.DÑ.DÔEÀbÀcÐJÐJùòK Us   ÁK6rV   N)	r  r  r  r  r  r  rð  r
  r  r  s   @rX   rÑ  rÑ  -  s#   ø‡ € ñ÷
v'ð v'÷p.Kð .KrW   rÑ  c                   óÌ   a a€ ] tR tRt o]t]t]t	V 3R lt
R tV3R lR ltV3R lR ltR tR	 tR
 tV3R lR ltR tR tRV3R lR lltV3R lR ltV3R ltRtVtV ;t# )rZ  i  c                óÎ   <€ \         SV `  VP                  VP                  P                  4       Wn        R V n        R V n        \        P                  ! 4       V n
        . V n        R # r  )r/  r0  r™  Úwsr�  ra  rR  r}  r   rÃ  Úpicked_vec_isaÚkernels©r1  ra  r4  s   &&€rX   r0  ÚCppKernelProxy.__init__'  sO   ø€ Ü‰Ñ˜×*Ñ*¨L¯O©O×,GÑ,GÔHØ(ÔØˆŒØˆÔÜ2=×2JÒ2JÓ2LˆÔØ(*ˆŽrW   c                ó†   € V F:  p\        V\        4      '       g   \        R 4      h\        P                  ! V4       K<  	  R# )ú)expected isinstance(_node, SchedulerNode)N)r§   r$   r…   r:   Úpropagate_scheduler_node)r1  r¼  r3  s   && rX   Údata_type_propagationÚ$CppKernelProxy.data_type_propagation/  s3   € ÛˆEÜ˜e¤]×3Ò3Ü$Ð%PÓQÐQÜ×8Ò8¸Ö?ó rW   c                ó    <€ V ^8„  d   QhRS[ /# )r’   Úscheduler_node)r$   )r–   r  s   "€rX   r—   ÚCppKernelProxy.__annotate__6  s   ø€ ÷ 
ñ 
±=ñ 
rW   c                ó   € \        VP                  \        4      '       g   R # \        P                  ! V4       \        VP                  4      ^ ,          RJ;'       d"    \        VP                  4      ^,          '       * # )TN)r§   r¶  r   r:   r+  rÄ  )r1  r/  s   &&rX   Úis_lowp_fp_schedulerÚ#CppKernelProxy.is_lowp_fp_scheduler6  se   € Ü˜.×.Ñ.´×9Ò9Ùä×4Ò4°^ÔDä! .×"6Ñ"6Ó7¸Õ:À$ÐF÷ Cð CÜ)¨.×*>Ñ*>Ó?ÀÕBÔBð	
rW   c                ó    <€ V ^8„  d   QhRS[ /# )r’   rÙ  r   )r–   r  s   "€rX   r—   r0  @  s   ø€ ÷ n*ñ n*¹ñ n*rW   c                ó²   € R  R lpVP                   .\        VP                  P                  4       4      ,           pV F  pV! VP                  4       K  	  R# )c                óL   € V ^8„  d   QhR\         P                  P                  /# ©r’   Ú	sub_graph©rƒ   ÚfxÚGraph)r–   s   "rX   r—   ÚDCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.__annotate__A  s   € ÷ i	*ñ i	*¤E§H¡H§N¡Nñ i	*rW   c           	      ó˜  aaaaaaaa€ R  R loR R loR V3R lloR V3R lloR VV3R	 llp\        V P                  4      p. oV EF—  pVP                  R9   dú   S! V4      ;o\        9   dç   \        ;QJ d*    VV3R
 lVP
                   4       F  '       d   K   RM	  RM! VV3R
 lVP
                   4       4      '       d   K€  VP                  ^ ,          pV P                  V4      ;_uu_ 4        V P                  RWC\        P                  3R7      oVP                  SV3R l4       \        ;P                  ^,          un        RRR4       EK  VP                  R8X  d¾   S! V4      ;o\        9   d«   VP                  w  rErgpV! VS4      '       d   EKT  \        P                  P!                  V4      oV P#                  V4      ;_uu_ 4        V P                  RWGS3R7      oVP%                  VS4       \        ;P                  ^,          un        RRR4       EKÜ  VP                  R8X  d§   VP                  w  popp	p
V\        9   d‡   S\        P                  \        P&                  \        P(                  \        P*                  39  d   \-        R4      hTS\        9   d   \        P                  MS\        P                  V	V
3Vn        EK�  EK“  VP                  R8X  dŸ   VP                  R,          \        9   dƒ   VP                  w  rJo\        ;QJ d*    VV3R lVP
                   4       F  '       d   K   RM	  RM! VV3R lVP
                   4       4      '       d   EK(  WJ\        P                  3Vn        EKB  VP                  R8X  d°   VP                  R,          \        9   d”   VP                  w  rKo\        ;QJ d*    VV3R lVP
                   4       F  '       d   K   RM	  RM! VV3R lVP
                   4       4      '       d   EK×  SP/                  V4       WK\        P                  3Vn        EK  VP                  R8X  g   EK  VP                  w  rGopV\        9   du   V! Wx4      '       gg   V P#                  V4      ;_uu_ 4        V P                  RWGV3R7      oVP%                  VS4       \        ;P                  ^,          un        RRR4       S\        9   g   EK³  \        ;QJ d*    VV3R lVP
                   4       F  '       d   K   RM	  RM! VV3R lVP
                   4       4      '       d   EK  VP                  ^ ,          pV P                  V4      ;_uu_ 4        V P                  RWC\        P                  3R7      oVP                  SV3R l4       \        ;P                  ^,          un        RRR4       EKš  	  R V3R llpV! V 4       R#   + '       g   i     EKÁ  ; i  + '       g   i     EKÖ  ; i  + '       g   i     ELE; i  + '       g   i     EKÿ  ; i)c                óz   € V ^8„  d   QhR\         P                  P                  R\         P                  R,          /# ©r’   r  r¾   N©rƒ   r:  ÚNoder‡   )r–   s   "rX   r—   ÚZCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.__annotate__B  s+   € ÷  ñ  ¤e§h¡h§m¡mð  ¼¿¹ÀdÕ8Jñ  rW   c                ó„  € V P                   R8X  d1   \        P                  P                  V P                  ^,          4      # V P                   R8X  d   V P                  R,          # V P                   R8X  dJ   \        V P                  4      ^8”  d   V P                  ^,          # V P                  P                  RR4      # R# )z6Get input dtype for nodes that may consumes lowp fp dtrü  rÁ  r«  r¯   Nr¼  )r½  r5   rí  rî  r™  rÛ   r*  r�  ©r  s   &rX   Úget_input_dtypeÚ]CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.get_input_dtypeB  sˆ   € à—;‘; 'Ô)ÜŸ7™7×,Ñ,¨T¯Y©Y°q­\Ó:Ð:Ø—[‘[Ð$6Ô6ØŸ9™9 R�=Ð(Ø—[‘[ JÔ.Ü˜4Ÿ9™9“~¨Ô)Ø#Ÿy™y¨�|Ð+à#Ÿ{™{Ÿ™¨{¸DÓAÐAárW   c                óz   € V ^8„  d   QhR\         P                  P                  R\         P                  R,          /# r?  r@  )r–   s   "rX   r—   rB  P  s+   € ÷  ñ  ¤u§x¡x§}¡}ð  ¼¿¹ÀtÕ9Kñ  rW   c                ób  € V P                   R8X  dV   \        V P                  4      ^8w  d   \        R4      h\        P
                  P                  V P                  ^,          4      # V P                   R9   d   V P                  R,          # V P                   R8X  d   V P                  ^,          # R# )z6Get output dtype for nodes that may produce lowp fp dtr°  zexpected len(node.args) == 3rÁ  N)r«  rž  r¥  r©  r¼  )r½  rÛ   r™  r…   r5   rí  rî  rD  s   &rX   Úget_output_dtypeÚ^CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.get_output_dtypeP  s†   € à—;‘; &Ô(Ü˜4Ÿ9™9“~¨Ô*Ü,Ð-KÓLÐLÜŸ7™7×,Ñ,¨T¯Y©Y°q­\Ó:Ð:Ø—[‘[ð %ô ð  Ÿ9™9 R�=Ð(Ø—[‘[Ð$6Ô6ØŸ9™9 Q�<Ð'árW   c                ól   € V ^8„  d   QhR\         P                  P                  R\         P                  /# ©r’   r  Údtr@  )r–   s   "rX   r—   rB  b  s&   € ÷ 4ñ 4¬¯©¯©ð 4¼5¿;¹;ñ 4rW   c                óF   <€ V\         9  d   \        R4      hS! V 4      V8H  # )z]Check if the given node produces output with expected low precision floating point data type.úexpected dt in DTYPE_LOWP_FP)r‚   r…   )r  rM  rI  s   &&€rX   Úis_lowp_fp_sourceÚ_CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_sourceb  s'   ø€ àœ]Ô*Ü(Ð)GÓHÐHÙ'¨Ó-°Ñ3Ð3rW   c                ól   € V ^8„  d   QhR\         P                  P                  R\         P                  /# rL  r@  )r–   s   "rX   r—   rB  h  s&   € ÷ 
!ñ 
!¤e§h¡h§m¡mð 
!¼¿¹ñ 
!rW   c                ó‚   <€ V\         9  d   \        R4      hS! V 4      ;p'       d   W!8H  # V P                  R8X  d   R# R# )zZCheck if the given node accept input with expected low precision floating point data type.rO  r«  TF)r‚   r…   r½  )r  rM  Úinput_dtyperE  s   && €rX   Úis_lowp_fp_sinkÚ]CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_sinkh  sB   ø€ àœ]Ô*Ü(Ð)GÓHÐHÙ"1°$Ó"7Ð7�;Ö7Ø&Ñ,Ð,Ø—[‘[ JÔ.áá rW   c                ól   € V ^8„  d   QhR\         P                  P                  R\         P                  /# rL  r@  )r–   s   "rX   r—   rB  t  s&   € ÷ ñ ´5·8±8·=±=ð ÄeÇkÁkñ rW   c                óÈ   <a€ S! V S4      ;'       dQ    \         ;QJ d*    VV3R lV P                   4       F  '       d   K   R# 	  R# ! VV3R lV P                   4       4      # )z…Check if the node is a lowp fp sources which are all directly fed to ops that accepts lowp fp input
thus no need to promote to float
c              3   ó6   <"  € T F  pS! VS4      x € K  	  R # 5ir  rV   ©r  ÚuserrM  rU  s   & €€rX   r  Ú}CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_source_no_promote.<locals>.<genexpr>x  s   øé € ð ;Ù:D°$‘O D¨"×-Ð-»*ùó   ƒFT©r#  Úusers)r  rM  rU  rP  s   &f€€rX   Úis_lowp_fp_source_no_promoteÚjCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_source_no_promotet  s[   ù€ ñ )¨¨rÓ2÷ ð ·s³sõ ;Ø:>¿*º*ó;·s´sð ²sð °sõ ;Ø:>¿*º*ó;ó 8ð rW   c              3   ó6   <"  € T F  pS! VS4      x € K  	  R # 5ir  rV   rZ  s   & €€rX   r  ÚWCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<genexpr>…  ó   øé € ÐMÁ¸™?¨4°×4Ð4Ãùr]  FTr«  ©r™  c                 ó   <€ V SJ# r  rV   ©rv  Úto_type_nodes   &€rX   r&  ÚVCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<lambda>�  s
   ø€ °A¸\Ñ4IrW   Nrü  r=  zEexpected dtype in [ torch.float, torch.bfloat16, torch.float16, to...rž  c              3   ó6   <"  € T F  pS! VS4      x € K  	  R # 5ir  rV   rZ  s   & €€rX   r  rc  Â  rd  r]  c              3   ó6   <"  € T F  pS! VS4      x € K  	  R # 5ir  rV   rZ  s   & €€rX   r  rc  È  rd  r]  rÁ  c              3   ó6   <"  € T F  pS! VS4      x € K  	  R # 5ir  rV   )r  r[  r‡   rU  s   & €€rX   r  rc  ñ  s   øé € ÐUÉÀ¡°°e× <Ð <Ëùr]  c                 ó   <€ V SJ# r  rV   rg  s   &€rX   r&  ri  ù  s
   ø€ ¸AÀ\Ñ<QrW   c                óL   € V ^8„  d   QhR\         P                  P                  /# r7  r9  )r–   s   "rX   r—   rB  þ  s   € ÷ *8ñ *8¬e¯h©h¯n©nñ *8rW   c                 ó*   <€ R  V3R llpV! V 4       R# )c                óL   € V ^8„  d   QhR\         P                  P                  /# r7  r9  )r–   s   "rX   r—   ÚvCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>.__annotate__ÿ  s   € ÷ ')ñ ')¼E¿H¹H¿N¹Nñ ')rW   c                 óú  <a€ R  R lpV P                    Uu. uF  q"P                  R8X  g   K  VNK  	  ppV Uu. uF  q!! V4      '       g   K  W"P                  /NK!  	  ppV Fê  pVP                  4        FÓ  w  opSV P                   9   g   K  \        ;QJ d    V3R lV 4       F  '       d   K   RM	  RM! V3R lV 4       4      '       gF   SS9   g   Kb  \        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   Kž  SP
                  R,          pSP                  V4       V P                  S4       KÕ  	  Kì  	  V P                  f   V P                  4        R# R# u upi u upi )	c                óL   € V ^8„  d   QhR\         P                  P                  /# )r’   Úto_node)rƒ   r:  rA  )r–   s   "rX   r—   ÚœCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>.__annotate__  s   € ÷ Vñ V¬U¯X©X¯]©]ñ VrW   c                 ó’   € \         ;QJ d&    R  V P                   4       F  '       d   K   R# 	  R# ! R  V P                   4       4      # )c              3   ó>   "  € T F  qP                   R 8H  x € K  	  R# 5i)r«  N)r½  ©r  Úusrs   & rX   r  Ú®CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>._used_by_to.<locals>.<genexpr>  s   é € Ð"UÁ}À§:¡:°Ö#;Ã}ùs   ‚FTr^  )rt  s   &rX   Ú_used_by_toÚ›CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>._used_by_to  s3   € ß"›sÑ"UÀwÇ}Â}Ó"UŸsœsÐUšsÐU˜sÑ"UÀwÇ}Â}Ó"UÓUÐUrW   r«  c              3   óp   <"  € T F+  qP                   R,          SP                   R,          8H  x € K-  	  R# 5i©r$  Nr¼  re  )r  ry  r  s   & €rX   r  Ú™CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>.<genexpr>  s$   øé € Ð#SÉUÀc§H¡H¨R¥L°D·I±I¸bµMÖ$AËUùs   ƒ36FTc              3   óT   "  € T F  qP                   R,          \        9   x € K   	  R# 5ir~  )r™  r‚   rx  s   & rX   r  r    s   é € ð ,&ÙQVÈ#¯©°­¼Ö(EÓQVùs   ‚&(Nr¼  )
r¼  r½  r_  r8  r#  Úall_input_nodesÚreplace_all_uses_withÚ
erase_nodeÚowning_moduleÚlint)	r8  r{  r  Úall_to_nodesÚall_to_nodes_and_usersÚ
node_usersr_  Úval_nodeÚto_lowp_fp_legalized_nodess	   & `     €rX   Ú_eliminate_duplicate_to_nodeÚ†CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_nodeÿ  sD  ù€ õVð *3¯ªó$Ù)8 ¿K¹KÈ:Ñ<UŸ˜©ð !ð $ñ 8Dó.Ù7C¨tÀ{ÐSW×GXÔ*˜Ÿz™zÓ*±|ð +ð .ó '=˜
Ø+5×+;Ñ+;Ö+=™K˜D %Ø# y§¡Ö6ß #£Ô#SÉUÓ#S§§¢Ô#SÉUÓ#S× SÒ Sà$(Ð,FÖ$Fß(+«ñ ,&ÙQVó,&¯¯ªñ ,&ÙQVó,&÷ )&ô )&ð
 ,0×+?Ñ+?ÀÕ+C Ø $× :Ñ :¸8Ô DØ )× 4Ñ 4°TÖ :ó ,>ñ '=ð, !×.Ñ.Ò6Ø!Ÿ™Ö(ñ 7ùò9$ùò.s   —E3®E3ºE8ÁE8NrV   )r8  r‹  rŠ  s   & €rX   Úeliminate_to_dtypeÚ`CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtypeþ  s   ø€ ÷')ð ')ñR -¨YÖ7rW   )r°  r¥  r©  r¼  )r$  r¼  r½  r‚   r#  r_  r™  Úinserting_afterÚcall_methodrƒ   rm   r‚  r   Úcpp_to_dtype_countr5   rí  rî  Úinserting_beforeÚreplace_input_withr  r   r•  r…   r'  )r8  r`  Úsub_graph_nodesr3  r3   rÎ   rÔ   Ú	value_varr¯   r†   rù   rµ  r�  rM  r‡   rE  rI  rU  rP  rŠ  rh  s   &            @@@@@@@@rX   Úadd_to_dtypeÚDCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtypeA  s¶  ÿ€ õ õ ÷$4ð 4÷
!ð 
!÷ñ ô # 9§?¡?Ó3ˆOØ)+Ð&Ü(�à—L‘LÐ$HÔHÙ/°Ó6Ð6˜¼=ÔH÷ “sÕMÀÇÂÓM—s—s’sÕMÀÇÂÓM×MÒMÙ ØŸ*™* Q�-�CØ"×2Ñ2°5×9Õ9Ø'0×'<Ñ'<Ø&¨c¼%¿+¹+Ð-Fð (=ó (˜ð ×3Ñ3Ø(Ô*Iôô  ×2Ò2°aÕ7Õ2÷ :Ò9ð —L‘L GÔ+Ù.¨uÓ5Ð5˜¼-ÔGà16·±Ñ.�C˜q¨Qá3°I¸r×BÒBÚ ÜŸG™G×-Ñ-¨dÓ3�EØ"×3Ñ3°E×:Õ:Ø'0×'<Ñ'<Ø&¨c¸eÐ-Dð (=ó (˜ð ×0Ñ0°¸LÔIä×2Ò2°aÕ7Õ2÷ ;Ò:ð —\‘\ [Ô0ð Ÿ
™
ñØØØ!Ø&Øà ¤MÔ1ð !Ü!ŸK™KÜ!ŸN™NÜ!ŸM™MÜ!ŸK™Kð	)ô ô #1Ø gó#ð ð  Ø+0´MÔ+AœEŸKšKÀuÜ!ŸK™KØ*Ø!ð&˜Ÿ
ò 2ð, —\‘\ ZÔ/°E·J±J¸rµNÄmÔ4Sà',§z¡zÑ$�S ß“sÕMÀÇÂÓM—s—s’sÕMÀÇÂÓM×MÒMÚ Ø"%¬e¯k©kÐ!:�E—JØ—\‘\ ZÔ/°E·J±J¸rµNÄmÔ4Sà#(§:¡:‘L�S˜Rß“sÕMÀÇÂÓM—s—s’sÕMÀÇÂÓM×MÒMÚ ð /×5Ñ5°eÔ<Ø"%¬%¯+©+Ð!6�E—JØ—\‘\Ð%7×7Ø9>¿¹Ñ6�S U¨Ið !¤MÔ1ñ  <¸I×QÒQØ!*×!;Ñ!;¸E×!BÕ!BØ/8×/DÑ/DØ$.°cÀiÐ5Pð 0Eó 0" ð !&× 8Ñ 8¸ÀLÔ Qä '× :Ò :¸aÕ ?Õ :÷ "Cð ¤×-÷  ›CÕUÈÏÊÓUŸCŸCšCÕUÈÏÊÓU×UÕUà"'§*¡*¨Q¥-˜CØ!*×!:Ñ!:¸5×!AÕ!AØ/8×/DÑ/DØ$.°cÄ%Ç+Á+Ð5Nð 0Eó 0" ð !&× ;Ñ ;Ø$0Ô2Qô!"ô !(× :Ò :¸aÕ ?Õ :÷ "BÒ!Añm )÷@*8ð *8ñX ˜yÖ)÷E :×9Ñ9ú÷$ ;×:Ñ:ú÷R "C×!BÐ!Bú÷" "B×!AÑ!Aús4   Ã6AW9ÇAXÒ"AX#ÖAX7×9XØX Ø#X4Ø7Y	N)rº  r$  r»  r€  rí  )r1  rÙ  r–  rÀ  rÃ  s   &&   rX   Úlegalize_lowp_fp_dtype_loopbodyÚ.CppKernelProxy.legalize_lowp_fp_dtype_loopbody@  sH   € õi	*ðV  ×*Ñ*Ð+¬d°9×3FÑ3F×3MÑ3MÓ3OÓ.PÕPˆ
Û#ˆIÙ˜Ÿ™Ö)ó $rW   c                óÌ  a € \         ;QJ d    V 3R  lV 4       F  '       d   K   RM	  RM! V 3R  lV 4       4      '       Ed   V EF  pVP                  P                  .\        VP                  P                  P                  4       4      ,           pV F»  pVP                  P                   Fž  pVP                  R	9   g   K  VP                  '       g   \        R4      h\        P                  VP                  9  d   \        R4      hVP                  \        P                  ,          pVP                  \        9  g   K•  \        R4      h	  K½  	  EK  	  R# V F„  p\        V\         4      '       g   \        R4      h\        VP                  \"        4      '       g   \        R4      hVP                  pVP%                  4       '       d   Ks  S P'                  V4       K†  	  R# )
c              3   óv   <"  € T F.  p\        V\        4      ;'       d    SP                  V4      x € K0  	  R # 5ir  )r§   r$   r2  )r  r3  r1  s   & €rX   r  Ú8CppKernelProxy.legalize_lowp_fp_dtype.<locals>.<genexpr>1  s5   øé € ð 
á�ô �uœmÓ,×QÐQ°×1JÑ1JÈ5Ó1QÔQÛùs   ƒ9Ÿ9FTzexpected fx_node.metaz0expected OptimizationContext.key in fx_node.metaz'expected opt_ctx.dtype in DTYPE_LOWP_FPNr*  z*expected isinstance(_node._body, LoopBody))r°  rü  )r#  r¶  rº  r$  r»  r€  rí  r¼  r½  r{  r…   rA   rz  r‡   r‚   r§   r$   r   Úis_memory_copyr˜  )r1  r¼  r3  rÀ  rÃ  Úfx_noderv  r°  s   f&      rX   Úlegalize_lowp_fp_dtypeÚ%CppKernelProxy.legalize_lowp_fp_dtype0  sy  ø€ ß‹3ô 
áó
�3�3Š3ô 
áó
÷ 
ó 
ô
 �Ø#Ÿk™k×4Ñ4Ð5¼Ø—K‘K×)Ñ)×0Ñ0Ó2ó9õ �
ó ",�IØ#,§?¡?×#8Ô#8˜Ø"Ÿ>™>Ð->Ö>Ø#*§<§< <Ü&4Ð5LÓ&MÐ MÜ2×6Ñ6¸g¿l¹lÔJÜ&4Ø$Vó'"ð !"ð <C¿<¹<Ü 3× 7Ñ 7õ<˜Gð  'Ÿ}™}´MÖAÜ&4Ø$Mó'"ð !"ó $9ô ",ñ	 ñ, ãˆEÜ˜e¤]×3Ò3Ü$Ð%PÓQÐQÜ˜eŸk™k¬8×4Ò4Ü$Ð%QÓRÐRØ"Ÿ[™[ˆDØ×&Ñ&×(Ô(Ø×4Ñ4°TÖ:ó rW   c           
     óà  aaa a!a"a#a$€ \        S4      \        S4      8w  d   \        R4      hV P                  o!\        SR R7      w  o o"V P	                  S S"4       V!V#3R lpVV V"V3R lo#V! V P
                  4      p\        P                  ;P                  VP                  ,          un        \        P                  ;P                  VP                  ,          un	        \        P                  V4      V n        V P                  '       d   V P                  '       g8   V.V n        V P!                  RR4       V P                  P#                  V 4       R# \$        P&                  P(                  P+                  RR7      ;_uu_ 4        \-        4       pVP/                  SS4      w  rgo$R	 V$3R
 llp\        V4      \        V4      8w  d   \        R4      hRp	Rp
RpV'       Ed	   RpV^ ,          pV^,           p\        V P                  P0                  4      V8”  dÑ   V P                  P0                  V,          P2                  pV P                  P0                  V,          P2                  pT;'       d    V'       * p	V'       * ;'       dc    \4        ;QJ d3    R V P                  P0                  VR  4       F  '       g   K   RM(	  RM$! R V P                  P0                  VR  4       4      p
\        V4      ^8X  EdK   \6        ;P8                  ^,          un        V P                  P;                  V^ ,          V^ ,          R7      pV! V P<                  V^ ,          V^ ,          4      pVP>                  VP@                  ,
          pVPB                  ^ VP@                  3/Vn"        \(        PF                  PH                  '       d;   V! VV^ ,          4      '       d%   V! V P<                  V^ ,          V^ ,          V4      pMTpVPB                  .Vn%        VPB                  VP@                  VP>                  3/Vn"        VV.V n        TpEM@\        V4      ^8X  Ed(   V^,          \        V P                  4      ^,
          8X  d   V^ ,          V^,          8X  g   \        R4      h\6        ;P8                  ^,          un        V P                  P;                  V^ ,          V^ ,          R7      pR^ VP@                  3RVP@                  VP>                  3/pVP>                  VP@                  ,
          pV P                  P;                  V^,          V^ ,          R7      pR^ VP@                  3RVP@                  VP>                  3/pVP>                  VP@                  ,
          pV! V PL                  V^ ,          V4      pVPB                  VR,          VPB                  VR,          /Vn"        . p\(        PF                  PH                  '       d•   V! VV^ ,          4      '       d   R Fw  w  ppVR8X  d   TMRpVR8X  d   TMRpV! V PL                  V^ ,          VVV4      pVPB                  VV,          VPB                  VV,          /Vn"        VPO                  V4       Ky  	  MÒV! V P<                  V^ ,          V^ ,          4      pVPB                  VR,          VPB                  VR,          /Vn"        VPB                  .Vn%        VPO                  V4       VPB                  VR,          VPB                  ^ VP>                  3/Vn"        VPB                  VPB                  .Vn%        VPO                  V4       V.V,           V n        TpMV.V n        T P!                  T	;'       g    \        V4      ^8H  ;'       d    T
V4       V P                  P#                  V 4       RRR4       R#   + '       g   i     R# ; i)zOGenerate scalar and vectorized C++ kernels with tiling for the given functions.z,expected len(fn_list) == len(var_sizes_list)c                 ó&   € \        V ^,          4      # rñ  rÝ  rÞ  s   &rX   r&  Ú2CppKernelProxy.codegen_functions.<locals>.<lambda>\  r  rW   rá  c                 óÈ   <€ SP                   ! V .VO5!  ;_uu_ 4       p\        ;P                  ^,          un        S! V4       VuuRRR4       #   + '       g   i     R# ; iræ  )Ú
new_kernelr   Úgenerated_kernel_count)r&  r™  re  ra  Úruns   &* €€rX   Úcodegen_kernelÚ8CppKernelProxy.codegen_functions.<locals>.codegen_kernel`  sG   ø€ Ø×(Ò(¨Ð4¨t×4Ô4¸ô ×.Ò.°!Õ3Õ.á�F”Ø÷ 5×4×4Ó4ús   ž'AÁA!	c           	      ó¨  <€ V P                  SS4      w  rR p\        SS	4       F–  w  rEVSS3\        \        P                  ! SS4      4      R339   d   V'       d   \        R4      hV! W4       KM  RpVSR38X  g   \        RV RS RS 24      hV P                  4       ;_uu_ 4        V! VR4       RRR4       K˜  	  R#   + '       g   i     K­  ; i)Fzexpected not in_suffixTzunexpected group: rÆ  r¡   NrV   )rF  rc  r¨   r  r  r…   r¢  )
re  r7  rú  Ú	in_suffixr  r  rõ  r¬  r­  rö  s
   &     €€€€rX   r§  Ú-CppKernelProxy.codegen_functions.<locals>.runj  sâ   ø€ Ø#)×#4Ñ#4°U¸OÓ#LÑ ˆDØˆIÜ!$ W¨nÖ!=‘�ØØ˜OÐ,Üœ9Ÿ?š?¨5°/ÓBÓCÀRÐHð!ô ÷ !Ü,Ð-EÓFÐFÙ�tÖ,à $�Ià!à!Øðôô -Ø0°°¸4À¸wÀbÈÐHYÐZóð ð  ×/Ñ/×1Õ1Ù˜4 œ÷ 2Ñ1ó+ ">÷* 2×1Ð1ús   Â(
C Ã CFN©Úinplace_buffersc                ó$   € V ^8„  d   QhR\         /# r�  ©ro   )r–   s   "rX   r—   Ú6CppKernelProxy.codegen_functions.<locals>.__annotate__š  s   € ÷ ñ Ä$ñ rW   c                 óZ  <€ \        SP                  P                  4       4      pV^ 8”  d€   SP                  \        SP                  P                  4       4      ,           V,          SP
                  8”  d6   \        P                  P                  P                  V 4      p^V,          V8„  # R# )r   T)
ra   rÊ  r€  rÌ  rË  rÉ  r5   rí  r  rI  )rQ  rÀ  rü  Úhint_tail_sizeÚtiling_statss   &&  €rX   Ú_tail_vec_worthwhileÚ>CppKernelProxy.codegen_functions.<locals>._tail_vec_worthwhileš  s”   ø€ ô ˜\×4Ñ4×;Ñ;Ó=Ó>�à˜Q”Jà$×2Ñ2Ü˜l×IÑI×PÑPÓRÓSõTð õ	ð
 #×;Ñ;ô<ô &'§W¡W×%5Ñ%5×%GÑ%GÈ	Ó%R�NØ˜~Õ-°Ñ=Ð=ÙrW   z3expected len(tiling_factors) == len(tiling_indices)c              3   ó:   "  € T F  pVP                   x € K  	  R # 5ir  )r†  )r  rj  s   & rX   r  Ú3CppKernelProxy.codegen_functions.<locals>.<genexpr>À  s   é € ð Xá$K˜Dð ×)Ö)Û$Kùs   ‚T)r“  zEexpected tiling_indices[1] == len(self.itervars) - 1 and tiling_fa...ÚmainÚtail))r¹  rº  )rº  r¹  )rº  rº  )(rÛ   r…   ra  r`   rF  Ú
kernel_clsr5   rí  Úremoved_buffersÚinplaced_to_remover@  rŒ  rR  r%  r<  r&  Úaggregate_reduction_buffersÚ
set_kernelrƒ   Ú	_inductorr   ÚpatchrÑ  r  rL  r†  rf   r   Úgenerated_cpp_vec_kernel_countÚtileÚvec_kernel_clsrI  Ú
tiled_sizer¬   r{  rù  Úenable_loop_tail_vecr|  Útile2d_kernel_clsr'  )%r1  rõ  rö  r¨  Úscalar_kernelÚtiling_selectÚtiling_factorsrw  rµ  Ú_inner_loop_reduction_outer_notÚ _tiled_loop_reduction_descendantÚ_outer_loopÚinner_loop_reductionÚouter_loop_levelÚinner_loop_levelÚouter_loop_reductionrj  Ú
vec_kernelrQ  Útail_kernelÚ
outer_loopr\  ry  Ú
inner_loopÚinner_rangesrx  Útile2d_kernelÚouter_rÚinner_rÚ_inner_tail_sizeÚ_outer_tail_sizere  r¬  ra  r­  r§  r´  s%   &ff                             @@@@@rX   Úcodegen_functionsÚ CppKernelProxy.codegen_functionsW  s  þ€ äˆw‹<œ3˜~Ó.Ô.Ü Ð!OÓPÐPØ×(Ñ(ˆÜ!$ ^Ñ9TÔ!UÑˆˆà�‰˜˜Ô/ö	÷	%ð 	%ñ6 ' t§¡Ó7ˆÜ	�‰×Ò =×#@Ñ#@Õ@ÕÜ	�‰×"Ò" m×&FÑ&FÕFÕ"Ü!Ÿ™¨Ó6ˆŒà×"×"Ð"¨$¯-¯-¨-Ø)˜?ˆDŒLØ×,Ñ,¨U°DÔ9Ø�N‰N×%Ñ% dÔ+Ùô �_‰_×#Ñ#×)Ñ)¸%Ð)×@Õ@Ü(›NˆMØ;H×;VÑ;VØ˜ó<Ñ8ˆN¨L÷ð ô" �>Ó"¤c¨.Ó&9Ô9Ü$ØIóð ð /4Ð+Ø/4Ð,ØˆKßˆ~Ø',Ð$Ø#1°!Õ#4Ð Ø#3°aÕ#7Ð Ü�t—~‘~×+Ñ+Ó,Ð/?Ô?Ø+/¯>©>×+?Ñ+?Ø(õ,ç"‘lð )ð ,0¯>©>×+?Ñ+?Ø(õ,ç"‘lð )ð -×IÐIÐ5IÔ1Ið 4ð <PÔ7O÷ 8ð 8×TWÓTWñ Xà$(§N¡N×$8Ñ$8Ð9IÐ9JÑ$KóX×TW×TWÒTWñ Xà$(§N¡N×$8Ñ$8Ð9IÐ9JÑ$KóXó UÐ4ô
 �>Ó" aÕ'ä×6Ò6¸!Õ;Õ6Ø—~‘~×*Ñ*¨>¸!Õ+<À^ÐTUÕEVÐ*ÓW�Ù+Ø×'Ñ'¨¸Õ):¸NÈ1Õ<Mó�
ð !ŸI™I¨¯©Õ7�	Ø,0¯H©H°q¸$¿/¹/Ð6JÐ+K�
Ô(Ü—:‘:×2×2Ð2Ñ7KØ˜~¨aÕ0÷8ò 8ñ #1Ø×+Ñ+Ø& qÕ)Ø& qÕ)Ø!ó	#‘Kð #0�KØ48·H±H°:�MÔ0Ø-1¯X©X¸¿¹ÈÏÉÐ7SÐ,T�Ô)Ø *¨KÐ8�”Ø"’Ü�^Ó$¨Õ)à" 1Õ%¬¨T¯]©]Ó);¸aÕ)?Ô?Ø& qÕ)¨^¸AÕ->Ô>ä(Ø_óð ô
 ×6Ò6¸!Õ;Õ6Ø!Ÿ^™^×0Ñ0Ø" 1Õ%¨n¸QÕ.?ð 1ó �
ð ˜Q 
× 5Ñ 5Ð6Ø˜Z×2Ñ2°J·O±OÐDð �ð #-§/¡/°J×4IÑ4IÕ"I�Ø!Ÿ^™^×0Ñ0Ø" 1Õ%¨n¸QÕ.?ð 1ó �
ð ˜Q 
× 5Ñ 5Ð6Ø˜Z×2Ñ2°J·O±OÐDð �ð #-§/¡/°J×4IÑ4IÕ"I�Ù .Ø×*Ñ*Ø" 1Õ%Ø"ó!�ð —N‘N L°Õ$8Ø—N‘N L°Õ$8ð/�Ô+ð !�Ü—:‘:×2×2Ð2Ñ7KØ# ^°AÕ%6÷8ò 8ó-Ñ(˜ ð 07¸&Ô/@™OÀdð )ð 07¸&Ô/@™OÀdð )ñ "0Ø ×2Ñ2Ø*¨1Õ-Ø*Ø,Ø,ó"˜ð 'ŸN™N¨L¸Õ,AØ&ŸN™N¨L¸Õ,Að0˜Ô,ð $×*Ñ*¨6Ö2ò--ñ0 "0Ø×+Ñ+¨^¸AÕ->ÀÈqÕ@Qó"�Jð #Ÿ™¨°VÕ(<Ø"Ÿ™¨°VÕ(<ð0�JÔ,ð 2<·±Ð0@�JÔ-Ø×&Ñ& zÔ2à"Ÿ™¨°VÕ(<Ø"Ÿ™¨¨J¯O©OÐ(<ð3�MÔ/ð 5?·N±NÀJÇNÁNÐ3S�MÔ0Ø×&Ñ& }Ô5Ø -˜°Õ<�”Ø(‘à -˜�”ð ×,Ñ,Ø/÷ Sð SÜ˜Ó'¨1Ñ,×QÐQÐ1QØôð
 �N‰N×%Ñ% dÔ+÷G A×@×@Ò@úsS   ÆA_Ç"B_É1_Ê_Ê(_Ê>D_Ï_ÏH-_Ø_ØE=_Þ_Þ/#_ß_-	c                óˆ   € V F*  pV P                  V4       \        P                  ! V4       K,  	  V P                  W4       R # r  )r˜  r:   Úpropagate_loopbodyrÜ  )r1  r÷  rö  r°  s   &&& rX   Úcodegen_loop_bodiesÚ"CppKernelProxy.codegen_loop_bodies9  s8   € ÛˆDØ×0Ñ0°Ô6Ü×2Ò2°4Ö8ñ  ð 	×Ñ˜{Ö;rW   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©r’   r¼  ©r$  r$   )r–   r  s   "€rX   r—   r0  ?  s   ø€ ÷  8ñ  8¡4©Õ#6ñ  8rW   c                ó  € V P                  V4       V P                  V4       \        V4      ^8  d   \        R4      hR pV Uu. uF  p\        P
                  ! W#4      NK  	  pp\        \        P                  \        4      '       d:   \        P                  P                  '       d   R pV Uu. uF
  q%! V4      NK  	  ppV Uu. uF  q3P                  ^,          NK  	  ppV P                  WF4       R# u upi u upi u upi )r$  zexpected len(nodes) >= 1c                 óÌ   € V P                  4        V P                  4        \        \        P                  \
        4      '       d   V P                  ! V!  # V P                  V4      # r  )Údecide_inplace_updateÚmark_runr§   r5   re  r2   r¶  Úcodegen)r  Ú
index_varss   &*rX   r  Ú(CppKernelProxy.codegen_nodes.<locals>.fnF  sG   € Ø×&Ñ&Ô(Ø�M‰MŒOÜœ!Ÿ(™(Ô$5×6Ò6Ø—z’z :Ñ.Ð.à—|‘| JÓ/Ð/rW   c                 óP   € \         P                  P                  V 4      pWn        V# r  )r5   r~  Úlocalize_functionÚoriginal_fn)r  Ú
wrapped_fns   & rX   Úwrap_fnÚ-CppKernelProxy.codegen_nodes.<locals>.wrap_fnU  s)   € Ü×3Ñ3×EÑEØó�
ð *,Ô&Ø!Ð!rW   N)rŸ  r,  rÛ   r…   rS  Úpartialr§   r5   r~  rK   r  r¬  rÜ  )r1  r¼  r  r  rõ  rð  rö  s   &&     rX   Úcodegen_nodesÚCppKernelProxy.codegen_nodes?  sÛ   € à×#Ñ# EÔ*Ø×"Ñ" 5Ô)Üˆu‹:˜Œ>Ü Ð!;Ó<Ð<ò	0ñ <AÓA¹5°4”9×$Ò$ RÖ.¹5ˆÐAô ”q×-Ñ-Ô/A×BÒBÜ×&Ñ&×4×4Ð4ò"ñ .5Ó5©W r�w˜r–{©WˆGÐ5á49Ó:±E¨DŸ*™* QŸ-˜-±EˆÐ:Ø×Ñ˜wÖ7ùò# Bùò 6ùâ:s   ÁC7Â1C<ÃDc                ó>   € V P                  V P                  W4       R # r  )r‰  rR  )r1  r¹   ro  s   &&&rX   r�  ÚCppKernelProxy.codegen_loopsa  s   € Ø×Ñ §¡°ÖBrW   c                óL   € V P                    F  pVP                  4        K  	  R # r  )r&  r¢  ©r1  re  s   & rX   r¢  Ú4CppKernelProxy.update_stores_with_parallel_reductiond  s   € Ø—l”lˆFØ×8Ñ8Ö:ó #rW   c                ó.   <€ V ^8„  d   QhRS[ R,          /# r¥  r¦  )r–   r  s   "€rX   r—   r0  h  s   ø€ ÷ 	3ñ 	3™\¨DÕ0ñ 	3rW   c                óp  € Vf   \        R4      hRpV P                   F€  p\        P                  ! 4       ;_uu_ 4       pVP	                  W4      '       dA   RpVP                  VP                  4       4       VP                  VP                  4       4       R R R 4       K‚  	  R #   + '       g   i     K—  ; i)Nzexpected code is not NoneÚ
C10_LIKELYÚC10_UNLIKELY)	r…   r&  rÇ   rÈ   rº  rÊ   rË   rb  r®  )r1  r¹   Ú	if_prefixre  rÓ   s   &&   rX   r®  ÚCppKernelProxy.gen_bodyh  s†   € ØŠ<Ü Ð!<Ó=Ð=Ø ˆ	Ø—l”lˆFÜ×%Ò%×'Ô'¨5Ø×,Ñ,¨T×=Ò=Ø .�IØ×'Ñ'¨¯©«Ô6Ø—K‘K §¡Ó 1Ô2÷	 (Ñ'ó #ß'×'Ð'ús   ¾AB$Â$B5c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r’   Úinner_loop_reduction_outer_notrÔ  Ú	LoopLevel)ro   r
   )r–   r  s   "€rX   r—   r0  s  s&   ø€ ÷ Z
ñ Z
Ù.2ðZ
Ù@HÈÕ@UñZ
rW   c                óø  a € R V 3R llpS P                   ^ ,          pV'       d   V'       g   \        R4      hV! V4       MZVP                  4        S P                  P	                  VP                  4       S P
                  P	                  VP
                  4       S P                  P	                  VP                  4       S P                  P	                  VP                  4       S P                  P	                  VP                  4       S P                  P	                  VP                  4       S P                  P	                  VP                  4       S P                  P	                  VP                  4       R# )z°
CppKernel/CppVecKernel/CppTile2dKernel have reduction buffers themselves.
Here, we decide how to aggregate them together and place new reduction buffers
under CppKernelProxy.
c                ó   € V ^8„  d   QhRR/# )r’   rÔ  r  rV   )r–   s   "rX   r—   Ú@CppKernelProxy.aggregate_reduction_buffers.<locals>.__annotate__|  s   € ÷ =	/ñ =	/¸+ñ =	/rW   c                 ó   <€ \        SP                  4      ^8  d   \        R4      hSP                  ^ ,          pSP                  R
,          p\        VSP                  4      '       g   \        R4      h\        V4      SP                  J dc   VP                  VP                  4       VP                  4        SP                  P                  VP                  VP                  ,           4       M5VP                  4        SP                  P                  VP                  4       \        4       p\        P                  ! 4       ;_uu_ 4       pVP                  VRV P                  4      '       d;   VP!                  VP#                  4       4       VP                  VP$                  4       RRR4       \        P                  ! 4       ;_uu_ 4       pVP                  VRV P                  4      '       Ed&   VP!                  VP#                  4       4       \        V4      SP                  J dÓ   VP&                  pV FW  pV RV P                   R\)        V P*                  4       R2p\-        VP.                  Wg4       \-        VP$                  Wg4       KY  	  \1        VP.                  4       VP                  \3        VP$                  V P                  V P                   R	2V P*                  V P4                  4      4       MVP                  VP$                  4       RRR4       VSn        R#   + '       g   i     EL‰; i  + '       g   i     L-; i)r’   zexpected len(self.kernels) >= 2z:expected isinstance(main_loop_kernel, self.vec_kernel_cls)rü  Nrý  rß   z_tail - rë  rª  r¼  )rÛ   r&  r…   r§   rÄ  r  r»  r  rÀ  r  rb  r7   rÇ   rÈ   rº  r¬   rÊ   rË   r�  r’  rE   rÅ  rð   r   rû   r×   rI  )	rÔ  Úmain_loop_kernelÚtail_loop_kernelÚ
suffix_bufrÓ   rú  rÎ   rí   r1  s	   &       €rX   Ú!aggregate_reduction_prefix_suffixÚUCppKernelProxy.aggregate_reduction_buffers.<locals>.aggregate_reduction_prefix_suffix|  s­  ø€ Ü�4—<‘<Ó  1Ô$Ü$Ð%FÓGÐGØ#Ÿ|™|¨A�ÐØ#Ÿ|™|¨BÕ/ÐÜÐ.°×0CÑ0C×DÒDÜ$ØPóð ô
 Ð$Ó%¨¯©Ó8ð !×:Ñ:Ø$×2Ñ2ôð !×:Ñ:Ô<Ø×%Ñ%×,Ñ,Ø$×5Ñ5Ø&×7Ñ7õ8õð
 !×:Ñ:Ô<Ø×%Ñ%×,Ñ,Ð-=×-NÑ-NÔOô &›ˆJÜ×%Ò%×'Ô'¨5Ø#×6Ñ6Ø ¨j¯n©n÷ò ð ×'Ñ'¨
×(9Ñ(9Ó(;Ô<Ø×%Ñ%Ð&6×&GÑ&GÔH÷ (ô ×%Ò%×'Ô'¨5Ø#×6Ñ6Ø °
·±÷ó ð ×'Ñ'¨
×(9Ñ(9Ó(;Ô<ÜÐ,Ó-°·±Ó@Ø)9×)MÑ)M˜Û$2˜DØ*.¨¨u°Z·^±^Ð4DÀHÌ[ÐYc×YnÑYnÓMoÐLpÐpqÐ'r˜HÜ,Ð-=×-DÑ-DÀdÔUÜ,Ø 0× AÑ AÀ4öñ %3ô 5Ð5E×5LÑ5LÔMØ"×)Ñ)Ü6Ø 0× AÑ AØ *§¡Ø#-§>¡>Ð"2°%Ð 8Ø *× 5Ñ 5Ø *§¡óõð #×)Ñ)Ð*:×*KÑ*KÔL÷5 (ð6 %/ˆDÖ!÷C (×'Ð'ú÷ (×'ús   ÅAL)ÇE
L=Ì)L:	Ì=M	zexpected outer_loopN)r&  r…   r  r  rb  r�  r‚  rƒ  r„  r…  r‡  rˆ  )r1  r  rÔ  r
  Úmain_kernels   f&&  rX   r¾  Ú*CppKernelProxy.aggregate_reduction_bufferss  s  ø€ ÷=	/ð =	/ð~ —l‘l 1•oˆß)ßÜ$Ð%:Ó;Ð;Ù-¨jÕ9à×1Ñ1Ô3Ø×!Ñ!×(Ñ(¨×)EÑ)EÔFØ×!Ñ!×(Ñ(¨×)EÑ)EÔFØ×&Ñ&×-Ñ-¨k×.SÑ.SÔTØ×&Ñ&×-Ñ-¨k×.SÑ.SÔTØ×!Ñ!×(Ñ(¨×)IÑ)IÔJØ×#Ñ#×*Ñ*¨;×+MÑ+MÔNØ×*Ñ*×1Ñ1Ø×5Ñ5ô	
ð 	×*Ñ*×1Ñ1Ø×5Ñ5ö	
rW   c                ón   <€ V ^8„  d   Qh/ S[ S[,          ;R&   S[ S[,          ;R&   S[ S[,          ;R&   # )r’   r»  rÄ  rÇ  )r  ru  r(  ro  )r–   r  s   "€rX   r—   r0    s:   ø‡ ‚ ñ ‘Y•Ñ+ñ ñ ™Õ&Ñ5ñ ñ ™OÕ,Ñ>ò rW   )r}  ra  r&  rR  r%  r�  r  )r  r  r  r  ru  r»  r(  rÄ  ro  rÇ  r0  r,  r2  r˜  rŸ  rÜ  rà  ró  r�  r¢  r®  r¾  r	  r
  r  rl  rm  s   @@rX   rZ  rZ    s   ù‡ € ð #,€JØ)5€NØ/>Ðõ+ò@÷
ð 
÷n*ð n*ò`%;òN`,òD<÷ 8ð  8òDCò;÷	3ò 	3÷Z
ð Z
÷o … rW   rZ  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )rb  iÐ  c                ór   <€ \         SV `  VP                  VP                  P                  4       . V n        R # r  )r/  r0  r™  r$  r�  rd  r'  s   &&€rX   r0  ÚOuterLoopFusedKernel.__init__Ñ  s)   ø€ Ü‰Ñ˜×*Ñ*¨L¯O©O×,GÑ,GÔHØ%'ˆŽ
rW   c           
     óÄ  € . pV P                    Uu. uF  qDP                  4       NK  	  ppV Fx  pVP                  pVf   \        R4      hVP	                  VP                  \        \        V4      VP                  ,
          VP                  R7      V4      P                  4       Kz  	  \        \        VP                  \        V4      4      VP                  R7      # u upi )Nz expected call_ranges is not Nonerš  )rd  rY  r}  r…   r'  rz  rý   rÛ   r   rÿ   r_   r`   )r1  r{  rn  Úkernels_parallel_depthrR  Únested_kernelsre  r}  s   &&&     rX   rz  Ú*OuterLoopFusedKernel.decide_parallel_depthÕ  sß   € Ø!#Ðà48·J²Jó+
Ù4> y× Ñ Ö"±Jð 	ð +
ó %ˆFð !×,Ñ,ˆKØÒ"Ü$Ð%GÓHÐHØ"×)Ñ)Ø×,Ñ,Ü!ä Ó,Ð/A×/MÑ/MÕMà$6×$BÑ$Bô	ð ó÷ !‘.ö
ñ %ô" ÜØ"×1Ñ1´3Ð7MÓ3Nóð +×6Ñ6ô	
ð 	
ùò++
s   ‘C)rd  )	r  r  r  r  r0  rz  r
  r  rl  rm  s   @@rX   rb  rb  Ð  s   ù‡ € õ(÷
ò 
rW   rb  c                   ó"   € ] tR tRtRtRtRtRtR# )ÚReasonFusedNodesiô  Úsame_vars_reduceÚcompatible_reductionÚcompatible_ranges_no_reductionrV   N)r  r  r  r  ÚSAME_VARS_REDUCEÚCOMPATIBLE_REDUCTIONÚCOMPATIBLE_RANGES_NO_REDUCTIONr
  rV   rW   rX   r  r  ô  s   † Ø)ÐØ1ÐØ%EÔ"rW   r  c                   ó´  a a€ ] tR tRt o]tRt]! ]P                  ]P                  .4      t]V3R lR l4       tV 3R ltV3R lR ltR tR	 tV3R
 lR ltR tR tR tV3R lR ltR tR tR tV3R lR ltR tR tR tR tV3R lR ltV3R lR lt V3R lR lt!V3R  lR! lt"V3R" lR# lt#R$ t$R% t%R& t&R,R' lt'R( t(R,R) lt)V3R* lt*R+t+Vt,V ;t-# )-ÚCppSchedulingiú  iô  c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r’   Údevicer¾   )rƒ   r!  r   r6   )r–   r  s   "€rX   r—   ÚCppScheduling.__annotate__  s$   ø€ ÷ $ñ $©%¯,©,ð $¹:ÁnÕ;Uñ $rW   c                ó   € V P                   # r  )Úbackend_features)r&  r!  s   &&rX   Úget_backend_featuresÚ"CppScheduling.get_backend_features
  s   € à×#Ñ#Ð#rW   c                ód   <€ \         SV `  V4       V'       d   V P                  4        R V n        R# rƒ  )r/  r0  Úreset_kernel_groupÚ_ready_to_flush)r1  r"  r4  s   &&€rX   r0  ÚCppScheduling.__init__  s'   ø€ Ü‰Ñ˜Ô#ßØ×#Ñ#Ô%Ø$ˆÖrW   c                ó    <€ V ^8„  d   QhRS[ /# )r’   Ústatusr°  )r–   r  s   "€rX   r—   r"    s   ø€ ÷ &ñ &©ñ &rW   c                ó   € Wn         R # r  ©r)  )r1  r,  s   &&rX   Ú_set_flush_statusÚCppScheduling._set_flush_status  s   € Ø%ÖrW   c                óZ   € \         ;QJ d    . R  V 4       F  NK  	  5# ! R  V 4       4      # )c              3   ó�   "  € T F<  p\        \        \        P                  P                  P
                  V4      4      x € K>  	  R # 5ir  )r¨   r)  r5   rí  r  Úsimplifyr  s   & rX   r  Ú)CppScheduling.group_fn.<locals>.<genexpr>  s/   é € ÐMÁuÀ!”Uœ3œqŸw™w×/Ñ/×8Ñ8¸!Ó<×=Ð=Ãuùs   ‚AA)r¨   )r1  rß  s   &&rX   Úgroup_fnÚCppScheduling.group_fn  s$   € ßŒuÑMÁuÓM�uÐMˆuÑMÁuÓMÓMÐMrW   c                ó$   € \        4       V n        R # r  )ÚKernelGroupra  r7  s   &rX   r(  Ú CppScheduling.reset_kernel_group  s   € Ü'›MˆÖrW   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r�  )r   ÚExtraIndexingConstraints)r–   r  s   "€rX   r—   r"    s   ø€ ÷ 
ñ 
±"×2MÑ2Mñ 
rW   c                ót  € \        V\        4      '       d÷   \        VP                  4      ^ 8:  d   \	        VP                  4      hRp\
        \        ,          ! 4       pVP                   Fp  pV P                  V4      pVf   VP                  pW%P                  8w  d"   \	        W%P                  VP                  34      hVP                  VP                  4       Kr  	  Vf   \	        R4      h\        P                  ! V\        V4      4      # \        V\        4      '       g   \	        R4      hVP                  p\        V\        P                   4      '       g   \	        R4      hVP#                  4       w  rxp\        P                  ! VP$                  \        VP&                  P)                  4       4      4      # )r   Nz-expected at least one snode to set var_rangesz(expected isinstance(node, SchedulerNode)z3expected isinstance(comp_buffer, ir.ComputedBuffer))r§   r"   rÛ   Úsnodesr…   r   r   Ú_get_indexing_ranges_exprsrW  ÚupdateÚexprsr   r;  r$  r$   r  ÚComputedBufferÚget_default_sizes_bodyrÖ  Úindexing_exprsr€  )	r1  r  rÖ  rC  ÚsnodeÚconstraintsÚcomp_bufferrÔ   r°  s	   &&       rX   r>  Ú(CppScheduling._get_indexing_ranges_exprs  sY  € Ü�dÔ.×/Ò/Ü�4—;‘;Ó 1Ô$Ü$ T§[¡[Ó1Ð1ØˆJÜ'¬ž_Ó.ˆNØŸœ�Ø"×=Ñ=¸eÓD�ØÒ%Ø!,×!3Ñ!3�JØ×!3Ñ!3Ô3Ü(¨*×6HÑ6HÈ$Ï+É+Ð)VÓWÐWØ×%Ñ% k×&7Ñ&7Ö8ñ %ð Ò!Ü$Ð%TÓUÐUÜ×.Ò.¨z¼4ÀÓ;OÓPÐPä˜$¤×.Ò.Ü Ð!KÓLÐLØ—i‘iˆÜ˜+¤r×'8Ñ'8×9Ò9Ü Ð!VÓWÐWØ ×7Ñ7Ó9‰
ˆ�Ü×*Ò*Ø�O‰OœT $×"5Ñ"5×"<Ñ"<Ó">Ó?ó
ð 	
rW   c                óD  € \        V\        4      '       d   WP                  4       3.# \        V\        4      '       g   \	        R 4      h. pVP
                   FD  p\        V\        4      '       g   \	        R4      hVP                  W3P                  4       34       KF  	  V# )z-expected isinstance(node, FusedSchedulerNode)ú)expected isinstance(snode, SchedulerNode))r§   r$   Úsnapshot_loop_stater"   r…   r=  r'  )r1  r  Ú	snapshotsrD  s   &&  rX   Ú_snapshot_node_loop_statesÚ(CppScheduling._snapshot_node_loop_states8  sŒ   € Ü�dœM×*Ò*Ø×3Ñ3Ó5Ð6Ð7Ð7ä˜$Ô 2×3Ò3Ü Ð!PÓQÐQØˆ	Ø—[”[ˆEÜ˜e¤]×3Ò3Ü$Ð%PÓQÐQØ×Ñ˜e×%>Ñ%>Ó%@ÐAÖBñ !ð ÐrW   c                óô  € \        V\        \        34      '       g   \        R 4      h\        V\        \        34      '       g   \        R4      hVP                  w  pw  rEVP                  w  pw  rgVR8X  d   VR8X  g   \        WW34      h\        V4      \        V4      8  d   TMTp\        V4      \        V4      8  d   TMTp	\        V\        4      '       g   \        R4      hV P                  V	4      p
VP                  V
R7       VP                  w  pw  rCVP                  w  pw  rcWF8X  d   R# V P                  V4      p\        V	\        4      '       d   V	P                  VR7       Mg\        V	\        4      '       g   \        R4      hV	P                   F6  p\        V\        4      '       g   \        R4      hVP                  VR7       K8  	  VP                  w  pw  rCVP                  w  pw  rcWF8H  # )z?expected isinstance(node1, (SchedulerNode, FusedSchedulerNode))z?expected isinstance(node2, (SchedulerNode, FusedSchedulerNode))ú2expected isinstance(node_to_recomp, SchedulerNode))Úextra_indexing_constraintsTz1expected isinstance(ref_node, FusedSchedulerNode)rI  rV   )	r§   r$   r"   r…   r¬  rÛ   r>  Úrecompute_size_and_bodyr=  )r1  r  r  rÔ   Úvars1Úreduce1Úvars2Úreduce2Únode_to_recompÚref_nodeÚref_indexing_constraintsÚ#node_to_recomp_indexing_constraintsrD  s   &&&          rX   Ú_align_compatible_range_nodesÚ+CppScheduling._align_compatible_range_nodesE  sÝ  € Ü˜%¤-Ô1CÐ!D×EÒEÜ ØQóð ô ˜%¤-Ô1CÐ!D×EÒEÜ ØQóð ð $Ÿk™kÑˆÑˆEØ#Ÿk™kÑˆÑˆEØ˜2” '¨R¤-Ü  'Ð!3Ó4Ð4ä"% e£*¬s°5«zÔ"9™¸uˆÜ ›J¬¨U«Ô3‘5¸ˆÜ˜.¬-×8Ò8Ü Ð!UÓVÐVà#'×#BÑ#BÀ8Ó#LÐ Ø×.Ñ.Ø'?ð 	/ô 	
ð Ÿ™‰ˆ‰:ˆEØŸ™‰ˆ‰:ˆEØŒ>Ùà.2×.MÑ.MØó/
Ð+ô �h¤×.Ò.Ø×,Ñ,Ø+Nð -õ ô ˜hÔ(:×;Ò;Ü$ØGóð ð "Ÿœ�Ü! %¬×7Ò7Ü(Ð)TÓUÐUØ×-Ñ-Ø/Rð .ö ñ )ð Ÿ™‰ˆ‰:ˆEØŸ™‰ˆ‰:ˆEØ‰~ÐrW   c                ó¨  € VP                  4       '       g   VP                  4       '       d   \        P                  ! W4      # VP                  4       '       d8   VP                  4       '       d   \	        R 4      h\
        P                  ! W4      # V P                  W4      \        P                  8X  dO   V P                  W4      '       g"   \	        VP                  VP                  34      h\
        P                  ! W4      # V P                  W4      '       d&   \        P                  WV P                  W4      4      # \
        P                  ! W4      # )z expected not node2.is_template())Ú
is_foreachr!   r(  Úis_templater…   r"   Ú_why_fuse_nodesr  r  rZ  r¬  Úcan_fuse_vertical_outer_loopr  Ú_get_outer_loop_fusion_depth©r1  r  r  s   &&&rX   r(  ÚCppScheduling.fusez  s  € Ø×Ñ×Ò ×!1Ñ!1×!3Ò!3Ü-×2Ò2°5Ó@Ð@Ø×Ñ× Ò Ø× Ñ ×"Ò"Ü$Ð%GÓHÐHÜ%×*Ò*¨5Ó8Ð8ð ×$Ñ$ UÓ2Ü#×BÑBôCð ×:Ñ:¸5×HÒHÜ(à!ŸK™KØ!ŸK™Kðóð ô *×.Ò.¨uÓ<Ð<Ø×2Ñ2°5×@Ò@Ü2×7Ñ7Ø $×"CÑ"CÀEÓ"Qóð ô *×.Ò.¨uÓ<Ð<rW   c                ó.   <€ V ^8„  d   QhRS[ R,          /# )r’   r¾   N)r  )r–   r  s   "€rX   r—   r"  •  s   ø€ ÷ ñ Ñ/?À$Õ/Fñ rW   c                ó  € VP                   w  pw  rEVP                   w  pw  rgWF8X  d   WW8X  d   \        P                  # VR8X  d   WFV,           8X  d   \        P                  # V P	                  W4      '       d   \        P
                  # R # )NrV   )r¬  r  r  r  Ú&_can_fuse_nodes_with_compatible_rangesr  )r1  r  r  rÔ   rR  rS  rT  rU  s   &&&     rX   r_  ÚCppScheduling._why_fuse_nodes•  sv   € Ø#Ÿk™kÑˆÑˆEØ#Ÿk™kÑˆÑˆEàŒ>˜gÔ0Ü#×4Ñ4Ð4Ø�bŒ=˜U¨g¥oÔ5Ü#×8Ñ8Ð8Ø×6Ñ6°u×DÒDÜ#×BÑBÐBárW   c                óÖ  € VP                   w  pw  rEVP                   w  pw  rgVR
8H  ;'       d    VR
8H  p\        P                  ! V4      \        P                  ! V4      8H  p	\        V4      ^8H  ;'       g    \        V4      ^8H  p
V'       d   V	'       d	   V
'       g   R# \        V4      \        V4      8  d   TMTp\        V4      \        V4      8  d   TMTp\	        V\
        4      '       d   R# \	        V\        4      '       g   \        R4      h\	        VP                  \        P                  4      '       d   R# \	        VP                  \        P                  4      '       g   \        R4      hVP                  P                  P                  4       pRp\	        V\
        4      '       Ed%   \        \        \         R3,          ,          ! 4       pVP"                   FÁ  p\	        V\        4      '       g   \        R4      h\	        VP                  \        P                  4      '       d    Mu\	        VP                  \        P                  4      '       g   \        R4      hVP%                  \        VP                  P                  P                  4       4      4       KÃ  	  \        V4      ^8w  d   R# \'        \)        \+        V4      4      4      pMz\	        V\        4      '       g   \        R4      h\	        VP                  \        P                  4      '       g   \        R	4      hVP                  P                  P                  4       pWí8w  d   R# V P-                  V4      pVP/                  V P-                  V4      4        V P1                  W4      \3        V4       F  w  ppVP5                  V4       K  	  #   \3        T4       F  w  ppTP5                  T4       K  	  i ; i)r$  FrO  z;expected isinstance(node_to_recomp.node, ir.ComputedBuffer)N.rI  z2expected isinstance(snode.node, ir.ComputedBuffer)z,expected isinstance(ref_node, SchedulerNode)z5expected isinstance(ref_node.node, ir.ComputedBuffer)rV   )r¬  rÖ  rb   rÛ   r§   r"   r$   r…   r  r   ÚTemplateBufferrA  ÚdataÚget_sizer   r¨   r   r=  r¡  r$  ÚnextÚiterrL  r-  rZ  r«  Úrestore_loop_state)r1  r  r  rÔ   rR  rS  rT  rU  Úc1Úc2Úc3rV  rW  Úranges2Úranges1Ú
ranges_setrD  rK  r  Ústates   &&&                 rX   rf  Ú4CppScheduling._can_fuse_nodes_with_compatible_ranges¢  sô  € ð $Ÿk™kÑˆÑˆEØ#Ÿk™kÑˆÑˆEà˜‰]×,Ð,˜w¨"™}ˆÜ�YŠY�uÓ¤§¢¨5Ó!1Ñ1ˆÜ�‹Z˜1‰_×/Ð/¤ E£
¨a¡ˆß—rŸbÙä"% e£*¬s°5«zÔ"9™¸uˆÜ ›J¬¨U«Ô3‘5¸ˆô �nÔ&8×9Ò9Ùô ˜.¬-×8Ò8Ü Ð!UÓVÐVÜ�n×)Ñ)¬2×+<Ñ+<×=Ò=ÙÜ˜.×-Ñ-¬r×/@Ñ/@×AÒAÜ ØMóð ð
 !×%Ñ%×*Ñ*×3Ñ3Ó5ˆØˆÜ�hÔ 2×3Ó3Ü#¤E¬#¨s¨(¥OÖ4Ó6ˆJØ!Ÿœ�Ü! %¬×7Ò7Ü(Ð)TÓUÐUÜ˜eŸj™j¬"×*;Ñ*;×<Ò<ÙÜ! %§*¡*¬b×.?Ñ.?×@Ò@Ü(ØLóð ð —‘œu U§Z¡Z§_¡_×%=Ñ%=Ó%?Ó@ÖAñ )ô �:‹ !Ô#Ùäœ4¤ ZÓ 0Ó1Ó2‰Gä˜h¬×6Ò6Ü$Ð%SÓTÐTÜ˜hŸm™m¬R×->Ñ->×?Ò?Ü$ØKóð ð —m‘m×(Ñ(×1Ñ1Ó3ˆGàÔÙà×3Ñ3°NÓCˆ	Ø×Ñ˜×8Ñ8¸ÓBÔCð	/Ø×5Ñ5°eÓCä'¨	Ö2‘��eØ×'Ñ'¨Ö.ò  3øœx¨	Ö2‘��eØ×'Ñ'¨Ö.ò  3ús   Î	O  Ï (O(c                óJ  € \        V\        \        \        34      '       g   \	        R 4      h\        V\        \        34      '       g   \	        R4      h\
        ;QJ d    R W3 4       F  '       g   K   RM	  RM! R W3 4       4      '       d   R# V P                  W4      RJ# )zEexpected isinstance( node1, (FusedSchedulerNode, SchedulerNode, Ex...z?expected isinstance(node2, (FusedSchedulerNode, SchedulerNode))c              3   óN   "  € T F  p\        V\        \        34      x € K  	  R # 5ir  )r§   r  r    r  s   & rX   r  Ú:CppScheduling._can_fuse_horizontal_impl.<locals>.<genexpr>ú  s(   é € ð 
á&�ô �tÔ9Ô;TÐU×VÐVÛ&ùs   ‚#%TFN)r§   r"   r$   r    r…   rf   r_  rb  s   &&&rX   Ú_can_fuse_horizontal_implÚ'CppScheduling._can_fuse_horizontal_implí  s¤   € äØÔ*¬MÔ;TÐU÷ò ô !ØWóð ô ˜%Ô"4´mÐ!D×EÒEÜ ØQóð ÷ ‹3ñ 
à™ó
�3�3Š3ñ 
à™ó
÷ 
ò 
ñ Ø×#Ñ# EÓ1¸Ð=Ð=rW   c                ó,  € VP                  4       '       g   VP                  4       '       d   R # \        VP                  4       4      \        VP                  4       4      ,           \        P                  P
                  8”  d   R # V P                  W4      # r½  )r^  rÛ   r.  r   rù  Úmax_horizontal_fusion_sizerz  rb  s   &&&rX   Úcan_fuse_horizontalÚ!CppScheduling.can_fuse_horizontal  sj   € Ø×Ñ×Ò %×"3Ñ"3×"5Ò"5Ùä�—‘Ó!Ó"¤S¨¯©Ó):Ó%;Õ;Ü�j‰j×3Ñ3ô4ñ à×-Ñ-¨eÓ;Ð;rW   c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r’   r  r  r¾   ©r   ro   )r–   r  s   "€rX   r—   r"    s%   ø€ ÷ 
ñ 
Ù&ð
Ù/@ð
á	ñ
rW   c                óª  € VP                  4       ;p'       d»   \        VP                  \        P                  4      ;'       d�    \        VP
                  \        P                  4      ;'       dc    \        VP
                  P                  4      ^8H  ;'       d9    VP
                  P                  ^ ,          P                  4       VP                  8H  # R# )r$  F)Úget_template_noder§   Úlayoutr   ÚMultiOutputLayoutr  ÚMultiOutputrÛ   Úinputsr‚  rÎ   )r1  r  r  Útemplate_bufs   &&& rX   Úcan_fuse_multi_outputs_templateÚ-CppScheduling.can_fuse_multi_outputs_template  s­   € ð !×2Ñ2Ó4Ð4ˆ<Ö4ä˜<×.Ñ.´×0DÑ0DÓE÷ Ið IÜ˜uŸz™z¬2¯>©>Ó:÷Ið Iä˜Ÿ
™
×)Ñ)Ó*¨aÑ/÷Ið Ið —J‘J×%Ñ% aÕ(×1Ñ1Ó3°|×7HÑ7HÑHð	ñ rW   c                ó`  € ^ p\         ;QJ d    R W3 4       F  '       d   K   RM	  RM! R W3 4       4      '       g   V# \        V\        4      '       d   VP                  4       R	,          MTp\        V\        \
        34      '       g   \        R4      h\        V\        4      '       d   VP                  4       ^ ,          MTp\        V\        \
        34      '       g   \        R4      hVP                  w  pw  rxVP                  w  pw  ršVR
8X  d   V	R
8X  d   VR
8w  d
   V
R
8w  d   V# \         ;QJ d    R W3 4       F  '       d   K   RM	  RM! R W3 4       4      '       d*   VP                  VP                  8X  d   VP                  # T# \        \        V4      \        V	4      4      pV^8¼  dx   VRV V	RV 8X  dk   \        ;QJ d    R W3 4       F  '       g   K   RM	  RM! R W3 4       4      '       d-   \        V4      \        J d   TMTpVP                  V8X  d   V# V# V# V# )r   c              3   ó\   "  € T F"  p\        V4      \        \        \        39   x € K$  	  R # 5ir  )r  r  r"   r$   r  s   & rX   r  Ú=CppScheduling._get_outer_loop_fusion_depth.<locals>.<genexpr>  s.   é € ð 
ñ '�ô �‹JÜ+Ô-?ÄÐOöPã&ùr  FTz@expected isinstance(_node1, (FusedSchedulerNode, SchedulerNode))z@expected isinstance(_node2, (FusedSchedulerNode, SchedulerNode))c              3   óD   "  € T F  p\        V4      \        J x € K  	  R # 5ir  r  r  s   & rX   r  r�  9  r   r!  Nc              3   óD   "  € T F  p\        V4      \        J x € K  	  R # 5ir  r  r  s   & rX   r  r�  D  s   é € ð ÙFT¸d”�T“
Ô9Õ9Ãnùr!  r¼  rV   )r#  r§   r  r%  r"   r$   r…   r¬  r'  r_   rÛ   rf   r  )r1  r  r  ÚDISABLE_OUTER_LOOP_FUSIONÚ_node1Ú_node2rÔ   rR  rS  rT  rU  r'  Ú_compare_nodes   &&&          rX   ra  Ú*CppScheduling._get_outer_loop_fusion_depth  s.  € Ø$%Ð!ß‹sñ 
ð ™ó
�s�sŠsñ 
ð ™ó
÷ 
ò 
ð
 -Ð,ô ˜%Ô!<×=Ò=ð ×!Ñ!Ó# BÖ'àð 	ô
 ˜&Ô#5´}Ð"E×FÒFÜ ØRóð ô
 ˜%Ô!<×=Ò=ð ×!Ñ!Ó# AÖ&àð 	ô
 ˜&Ô#5´}Ð"E×FÒFÜ ØRóð ð %Ÿl™lÑˆÑˆEØ$Ÿl™lÑˆÑˆEØ�BŒ;˜5 Bœ;¨7°b¬=¸WÈ¼]à,Ð,ß‹3ÑTÀeÁ^ÓT�3�3Š3ÑTÀeÁ^ÓT×TÒTð ×0Ñ0°E×4QÑ4QÔQð ×-Ñ-ðð /ðô
 #&¤c¨%£j´#°e³*Ó"=Ðà# qÔ(ØÐ.Ð.Ð/°5Ð9QÐ:QÐ3RÔRç‹sñ ØGLÁnó�s�sŠsñ ØGLÁnó÷ ò ô " %›[Ô,GÓG‘EÈUð ð !×8Ñ8Ð<SÔSà2Ð2à4Ð4ð /Ð.Ø(Ð(rW   c                óf  € VP                  4       '       * ;'       d–    VP                  4       '       * ;'       dy    VP                  4       VP                  ,          ;'       dP    V P                  W4      ;'       d    VP	                  4       '       * '       * ;'       d    V P                  W4      ^8¬  # rñ  )r^  Úget_operation_namesÚ	ancestorsrz  r†  ra  rb  s   &&&rX   r`  Ú*CppScheduling.can_fuse_vertical_outer_loopT  s­   € à×!Ñ!Ó#Ô#÷ Eð EØ×%Ñ%Ó'Ô'÷Eð Eà×)Ñ)Ó+¨e¯o©oÕ=÷Eð Eð ×.Ñ.¨uÓ<÷ -ð -Ø×*Ñ*Ó,Ô,ô÷Eð Eð ×1Ñ1°%Ó?À1ÑDð		
rW   c                ó8   € V P                  W4      '       d   ^# ^ # rñ  )r`  rb  s   &&&rX   Úget_fusion_pair_priorityÚ&CppScheduling.get_fusion_pair_priority`  s   € Ø×,Ñ,¨U×:Ò:áárW   c                óF  € VP                  4       '       d   R # VP                  4       '       d.   \        W.4      w  r4VP                  4       '       * ;'       d    T# V P                  W4      ;'       d    VP                  4       '       * ;'       g    V P	                  W4      # r½  )r^  rN   r†  rz  r`  )r1  r  r  Útemplate_fusion_supportedrÔ   s   &&&  rX   Úcan_fuse_verticalÚCppScheduling.can_fuse_verticalg  s’   € Ø×Ñ×ÒáØ×Ñ×ÒÜ+SØ�wó,Ñ(Ð%ð ×)Ñ)Ó+Ô+×IÐIÐ0IÐIà×*Ñ*¨5Ó8×UÐUÀ×ASÑASÓAUÔ=U÷=ð =à×.Ñ.¨uÓ<ð	=rW   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rã  rä  )r–   r  s   "€rX   r—   r"  t  s   ø€ ÷ Wñ W¡D©Õ$7ñ WrW   c                óx	  aaaaa€ \         ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       d   V# RoRo^ pRpRpRp. pV EFr  p\        VP                  \        P                  4      '       g   \        R4      hVP                  P                  4       pVP                  Wx34       Vw  w  ršrºVP                  P                  4        EFç  w  op\        V\        P                  4      '       g   K)  VP                  \        4       EF¤  o\         ;QJ d)    V3R lVP                   4       F  '       g   K   RM	  RM! V3R lVP                   4       4      '       d   SV8w  d   SpV^,          pV^8”  d	   Vu u u # \        SP                  ^,          \        P                   P"                  P$                  4      '       g   KÀ  SP                  ^ ,          VP                  9   g   Kä  Sf   Kê  \&        ;QJ d8    VV3R lVP                  P                  4        4       F  '       d   K   RM-	  RM)! VV3R lVP                  P                  4        4       4      '       g   EK_  SP                  ^,          ^8”  g   EKz  SP                  ^ ,          oSP                  ^,          oRpT	pEK§  	  EKê  	  EKu  	  V'       g   V# Vf   \        R4      h\)        V4      pV F4  w  pw  w  ršrºSVP                  9  d   Vu # \)        V	4      V8w  g   K2  Vu # 	  VV3R	 lpRp\*        \,        ,          ! 4       pV Fp  w  r¨V! V!  w  p
pp
Vf   VP.                  pVVP.                  8w  d   \        VVP.                  34      hVP1                  VP                  P3                  4       4       Kr  	  Vf   \        R
4      h\        P4                  ! V\7        V4      4      pV Uu. uF  qwVP9                  4       3NK  	  ppV F  pVP;                  VVR7       K  	  V^ ,          P<                  ^,          o\         ;QJ d&    V3R lVR,           4       F  '       g   K   RM	  RM! V3R lVR,           4       4      '       d'   \?        V4       F  w  ppVPA                  V4       K  	  V# u upi )aÙ  
Apply loop split optimization.
When one of the indexing_exprs contains a division, we eliminate the division by splitting the loop
to avoid non-contiguous loads, subject to the following conditions:
    1. No reduction and no mudular index for all nodes.
    2. The indexing_exprs of all nodes contain only one (or more, but all the same) division,
       where the divisor is an integer and not too small (the divisor > 8), the dividend is
       one of the iter_vars, and this var, i.e. the dimension that needs to be split, is
       contiguous in all other indexing_exprs.

For example, if the node's var_ranges: {z0: 2, z1: 9216, z2: 960} and indexing_exprs:
{'index0': 8847360*z0 + 960*z1 + z2, 'index1': 32*z0 + (z2//30), 'index2': z2},
we will split z2 -> 30*z2 + z3, then the node's var_ranges will be changed to
{z0: 2, z1: 9216, z2: 32, z3: 30} and indexing_exprs will be changed to
{'index0': 8847360*z0 + 960*z1 + 30*z2 + z3, 'index1': 32*z0 + z2, 'index2': 30*z2 + z3}.
c              3   ón  "  € T F«  p\        VP                  ^,          ^,          4      ^ 8g  ;'       gy    \        ;QJ d>    R VP                  P                  P                  4        4       F  '       g   K   RM3	  RM/! R VP                  P                  P                  4        4       4      x € K­  	  R# 5i)r$  c              3   óJ   "  € T F  qP                  \        4      x € K  	  R # 5ir  )r„  r   )r  r·  s   & rX   r  Ú9CppScheduling.try_loop_split.<locals>.<genexpr>.<genexpr>‰  s   é € ð Ù6X¨d—‘œ×)Ð)Ó6Xùs   ‚!#TFN)rÛ   r¬  rf   r¶  rC  r€  r  s   & rX   r  Ú/CppScheduling.try_loop_split.<locals>.<genexpr>‡  s‘   é € ð 
ñ
 �ô	 �—
‘
˜1•˜aÕ Ó! QÑ&÷ ð ß‹sñ Ø6:·j±j×6OÑ6O×6VÑ6VÔ6Xó�s�sŠsñ Ø6:·j±j×6OÑ6O×6VÑ6VÔ6Xóó ôó ùs   ‚1B5´B5Á 3B5Á6?B5TFNz1expected isinstance(node.node, ir.ComputedBuffer)c              3   óF   <"  € T F  pSP                  V4      x € K  	  R # 5ir  )r„  )r  r¬   Údiv_exprs   & €rX   r  r¥  §  s   øé € ÐQÑ9P°#˜HŸL™L¨×-Ð-Ó9Pùs   ƒ!c              3   óz   <"  € T F0  w  rVS8w  g   K  \        VSP                  ^ ,          4      R9   x € K2  	  R# 5i)r   Nrî  )r%   r™  )r  Úname_Úexpr_r§  rÎ   s   &  €€rX   r  r¥  ²  s:   øé € ð  á0T¡ Ø$¨™}ô SÔ/°°x·}±}ÀQÕ7GÓHÈFÖRÛ0Tùs   ƒ;“(;z'expected matched_index_size is not Nonec                 ó¦  <€ V w  r4Vw  rVVP                  S4      pVP                  4       pW7,          S,          W‡&   VP                  V^,           S4       \        P                  ! W„RR7      w  w  ršpV	P                  4       pVP                  V^,           4      pSWÇ,          ,          V,           WÇ&   \        P                  ! WV.W¹V4      pW„3VW–33# )r$  rD  )r`  )r“   ÚcopyÚinsertr   Úindex_vars_no_squeezeÚpopr   r   )rß  r°  r7  Ú
index_sizeÚreduce_sizerê  Úreduce_varsÚ	split_idxÚnew_index_sizeÚnew_index_varsrÔ   rÖ  Ú	iter_varsÚdivisor_varÚsplit_numberÚ	split_vars   &&&           €€rX   Ú
loop_splitÚ0CppScheduling.try_loop_split.<locals>.loop_splitÐ  sÜ   ø€ Ø&+Ñ#ˆJØ&*Ñ#ˆJØ"×(Ñ(¨Ó3ˆIØ'Ÿ_™_Ó.ˆNØ(2Õ(=ÀÕ(MˆNÑ%Ø×!Ñ! )¨a¥-°Ô>Ü.:×.PÒ.PØ°Cô/Ñ+Ñˆ^ ð '×+Ñ+Ó-ˆIØ#Ÿ-™-¨	°A­Ó6ˆKØ#/°)Õ2FÕ#FÈÕ#TˆIÑ Ü—;’;Ø +Ð.°
ÈKóˆDð  Ð-ØØÐ-ðð rW   zextra_indexing_ranges is None)rP  Úrecompute_sizes_body_funcc              3   óN   <"  € T F  qP                   ^,          S8g  x € K  	  R# 5iræ  )r¬  )r  r  r¬  s   & €rX   r  r¥    s   øé € Ð<±)¨$�z‰z˜!�} Ö%³)ùrÛ  r#  )!rf   r§   r  r   rA  r…   rB  r'  rC  r8  r”   r•   Úfindr   r¶  r™  ÚcoreÚnumbersrX  r#  rÛ   r   r   rÖ  r?  r€  r;  r$  rJ  rQ  r¬  r«  rn  )r1  r¼  Únum_divÚ	div_expr_Ú	match_divÚmatched_index_sizeÚnode_bodiesr  Ú
sizes_bodyr°  rÔ   Úoriginal_bodyr·  Úmatched_num_dimsrº  Úextra_indexing_rangesÚextra_indexing_exprsÚ
split_bodyrP  rK  ru  r§  r¬  rÎ   r¸  r¹  s   &&                   @@@@@rX   Útry_loop_splitÚCppScheduling.try_loop_splitt  s  ü€ ÷& ‹3ñ 
ñ
 ó
�3�3Š3ñ 
ñ
 ó
÷ 
ò 
ð ˆLàˆ	ØˆØˆØˆ	Øˆ	Ø!Ðð .0ˆäˆDÜ˜dŸi™i¬×):Ñ):×;Ò;Ü$ØGóð ð Ÿ™×9Ñ9Ó;ˆJØ×Ñ Ð1Ô2Ø0:Ñ-‰OˆZ˜]Ø+×:Ñ:×@Ñ@×B‘
��dÜ! $¬¯
©
×3Ò3ÙØ $§	¡	¬(× 3�Hç›ÔQ¸×9PÒ9PÓQŸŸšÔQ¸×9PÒ9PÓQ×QÒQØ$¨	Ô1à$,˜	Ø 1�˜Ø ”{Ø$žä" 8§=¡=°Õ#3´U·Z±Z×5GÑ5G×5OÑ5O×PÔPØ$ŸM™M¨!Õ,°×0GÑ0GÖGØ Ô,ß›Cõ  à0=×0LÑ0L×0RÑ0RÔ0Tó ŸCŸCšCõ  à0=×0LÑ0L×0RÑ0RÔ0Tó ÷ õ ð
 %ŸM™M¨!Õ,¨q×0à$,§M¡M°!Õ$4˜	Ø'/§}¡}°QÕ'7˜Ø$(˜	Ø-7Ó*ô/ !4ô Cñ ÷J ØˆLð Ò%Ü Ð!JÓKÐKÜÐ1Ó2Ðã9DÑ5ˆDÑ5‘?�J MØ × 7Ñ 7Ô7Ø’Ü�:‹Ð"2Ö2Ø’ñ	 :Eö	ð, !%ÐÜ)¬#žÓ0ÐÛ(‰MˆAÙ)¨:Ñ6ÑˆAˆz˜1Ø$Ò,Ø(2×(=Ñ(=Ð%Ø$¨
×(=Ñ(=Ô=Ü$à-Ø"×-Ñ-ðóð ð !×'Ñ'¨
×(AÑ(A×(HÑ(HÓ(JÖKñ )ð !Ò(Ü Ð!@ÓAÐAÜ%'×%@Ò%@Ø!ÜÐ%Ó&ó&
Ð"ñ
 EJÓJÁE¸D˜D×4Ñ4Ó6Ó7ÁEˆ	ÐJÛˆDØ×(Ñ(Ø+EØ*4ð )ö ñ ð �a•—‘˜qÕ!ˆß‹3Ô<°%¸¶)Ó<�3�3Š3Ô<°%¸¶)Ó<×<Ò<Ü'¨	Ö2‘��eØ×'Ñ'¨Ö.ñ  3ð ˆùò Ks   Ï/R7c                ó    <€ V ^8„  d   QhRS[ /# ©r’   r  ©r  )r–   r  s   "€rX   r—   r"    s   ø€ ÷ wKñ wKá)ñwKrW   c                ó¾  a aaa	€ S P                   o\        P                  p. o. o	\        V\        4      '       g   \        R4      hR VVV	V 3R llpV! V4      '       gê   V\        n        SP                  4        S	P                  4        \        P                  P                  P                  RR7      ;_uu_ 4        VP                  4        Fm  p\        V\        \        34      '       g   \        R4      hVP                  4       pS P                  S4      pVP!                  V4       SP#                  We4       Ko  	  RRR4       R# R#   + '       g   i     R# ; i)zð
Generate the code for the outer loop fused scheduler node.
1. Codegen with fused outer loop: depends on the analysis of
    the outer loop fused scheduler node, with or without the local buffer.
2. If failed, fallback to standard codegen.
ú6expected isinstance(node, OuterLoopFusedSchedulerNode)c                ó$   € V ^8„  d   QhR\         /# rÏ  rÐ  )r–   s   "rX   r—   Ú;CppScheduling.codegen_outer_loop_node.<locals>.__annotate__   s   € ÷ Q	ñ Q	Ô7Rñ Q	rW   c           
     óþ
  <a aaaaaa€ \        S \        4      '       g   \        R4      hSP                  4        SP                  4        R R lo. p/ o\        ;QJ d.    VV 3R lS P                  4        4       F  '       d   K   RM#	  RM! VV 3R lS P                  4        4       4      '       Ed—   \        4       oS P                  4        EFw  o\        S\        4      '       g   \        R4      hSP                  SP                  4       4       SP                  4       '       g   \        SP                  4       4      ^8w  d   Kz  SP                  4       ^ ,          o\        ;QJ d)    V 3R lSP                   4       F  '       d   K   RM	  RM! V 3R lSP                   4       4      '       g   Kç  SP                  p\        V\         P"                  4      '       g   \        R	4      hVP%                  4       pS P&                  \        S! S4      4      ,
          pVV3R
 lpVP)                  4       '       d   V! 4       '       g   EK}  . pVP*                  R,          pS! S4      VR p\-        V4       F  p	VP/                  ^ V4       Wy,          pK  	  \         P0                  ! VP2                  VP4                  VV4      p
VV3R lpRpV! W¡4      pV'       gG   \         P6                  ! V R\        V4       2V
R7      pVP9                  V4       . SVP:                  &   SVP:                  ,          P9                  V4       EKz  	  \=        SP>                  4      ;_uu_ 4       o\        V4      ^ 8”  dF   V F?  pVP:                  f   \        R4      hSPA                  VSVP:                  ,          4       KA  	  S P                  4        FŠ  p\        V\B        \        34      '       g   \        R4      hSPE                  S4      pVPG                  VP                  4       4       SP9                  V4       SP9                  VP                  4       4       KŒ  	  R V3R llpR VV 3R llp\        V4      ^ 8”  d   V! 4       '       d   V! 4       uuRRR4       # S PI                  SS P&                  4      '       g   V! 4       uuRRR4       # \J        PL                  P9                  \J        PN                  ! \        S4      \        SPP                  4      R7      4       S PS                  S4      pSPU                  V. \V        PX                  P[                  S4      O4       RRR4       R#   + '       g   i     R# ; i)z6
Codegen code with fused outer loop and local Buffer.
rÒ  c                ó$   € V ^8„  d   QhR\         /# rÏ  r  )r–   s   "rX   r—   ÚiCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.__annotate__+  s   € ÷ 
#ñ 
#Ô&7ñ 
#rW   c                 óä   € \        V \        \        34      '       g   \        R 4      hV P	                  4       p\        VR R7      P                  w  pw  r4\        V4      \        V4      ,           pV# )z>expected isinstance(node, (SchedulerNode, FusedSchedulerNode))c                 ó4   € \        V P                  4       4      # r  )rk   r†  rÉ  s   &rX   r&  Ú~CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.get_call_ranges.<locals>.<lambda>2  s   € ¬¨Q¯^©^Ó-=Ô)>rW   rá  )r§   r$   r"   r…   r.  r`   r¬  r¨   )r  r¼  rÔ   r¬  r­  r}  s   &     rX   Úget_call_rangesÚlCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.get_call_ranges+  sm   € Ü! $¬Ô8JÐ(K×LÒLÜ(ØXóð ð .2¯^©^Ó-=�Ü.1ØÑ>ô/ç‘%ñ ,�Ñ+�Eô $ E›l¬U°?Ó-CÕC�Ø"Ð"rW   c              3   ón   <"  € T F*  p\        S! V4      4      SP                  ^,           8H  x € K,  	  R# 5iræ  )rÛ   r'  )r  r3  rÛ  r  s   & €€rX   r  ÚfCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.<genexpr>:  s2   øé € ð á3�Eô ‘O EÓ*Ó+¨t×/KÑ/KÈaÕ/OÖOÛ3ùs   ƒ25FTz2expected isinstance(scheduler_node, SchedulerNode)c              3   ó\   <"  € T F!  qP                   SP                  4       9   x € K#  	  R # 5ir  )r  r.  )r  r[  r  s   & €rX   r  rÞ  Q  s#   øé € ð ÙBX¸$Ÿ	™	 T§^¡^Ó%5Ö5ÓBXùs   ƒ),z5expected isinstance(global_buffer, ir.ComputedBuffer)c                  óÈ  <aa€ ^ o^p \        SP                  P                  P                  4       4       F  w  rSW,          ,          oW,          p K  	  SP                  P	                  SP                  4       4      pV3R loS! V4      ;'       dQ    \        ;QJ d*    VV3R lSP                   4       F  '       d   K   R# 	  R# ! VV3R lSP                   4       4      # )r   c                 ó   <€ V S8H  # r  rV   )rµ  Úcontiguous_index_exprs   &€rX   Úis_contiguous_indexÚ–CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous.<locals>.is_contiguous_indexl  s   ø€ Ø'(Ð,AÑ'AÐ ArW   c              3   óÚ   <"  € T F`  p\        VP                  \        4      ;'       d:    S! VP                  P                  P	                  SP                  4       4      4      x € Kb  	  R # 5ir  )r§   r  r$   r¶  Úget_read_exprr‚  )r  r[  rã  Úscheduler_buffers   & €€rX   r  ÚŒCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous.<locals>.<genexpr>o  sc   øé € ð Qñ -C Dô !+¨4¯9©9´mÓ D÷ !"ð !"Ù$7Ø$(§I¡I§O¡O×$AÑ$AØ(8×(AÑ(AÓ(Có%&ó%"ô!"ó -Cùs   ƒ%A+©AA+FT)r«  r¶  rÖ  r8  Úget_write_exprr‚  r#  r_  )rB  r¬   rQ  Úwrite_index_exprrâ  rã  rç  r/  s       @@€€rX   Úis_all_write_read_contiguousÚyCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous^  sÜ   ú€ Ø45Ð1Ø%&˜FÜ.6à .× 4Ñ 4× ?Ñ ?× EÑ EÓ Gö/¡
 ð !6¸½Õ EÐ 5Ø &¥¢ñ/ð 0>×/CÑ/C×/RÑ/RØ 0× 9Ñ 9Ó ;ó0Ð,õBñ $7Ð7GÓ#H÷ $ð $ÏSËSõ Qð -=×,BÒ,BóQÏSÌSð ÊSð ÈSõ Qð -=×,BÒ,BóQó Nð rW   Nc                 ó
  <€ V F{  pWP                   8X  g   K  \        ;QJ d0    V3R  lSVP                  ,           4       F  '       d   K   RM%	  RM!! V3R  lSVP                  ,           4       4      '       g   Ky  Vu # 	  R# )c              3   ó¬  <"  € T FÉ  pVP                   f   K  \        ;QJ d\    V3R l\        P                  P                  P
                  VP                   ,          P                   4       F  '       d   K   RMQ	  RMM! V3R l\        P                  P                  P
                  VP                   ,          P                   4       4      x € KË  	  R # 5i)Nc              3   ó^   <"  € T F"  pVP                   P                  4       S9   x € K$  	  R # 5ir  )r  r‚  )r  r[  Úvisited_scheduler_nodess   & €rX   r  Ú�CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_buffer.<locals>.<genexpr>.<genexpr>‘  s/   øé € ð (&ñ50¨Dð )-¯	©	×(:Ñ(:Ó(<Ð@WÖ(Wó50ùs   ƒ*-FT)rÎ   r#  r5   rí  r"  Úname_to_bufr_  )r  Úglobal_bufferrð  s   & €rX   r  Ú†CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_buffer.<locals>.<genexpr>�  s¦   øé € ð S"ñ:&¨ð (5×'9Ñ'9ô%&§C£Cô (&ä45·G±G×4EÑ4E×4QÑ4QØ,9×,>Ñ,>õ5*ç*/©%ð50ó(&§C§C¢Cô (&ä45·G±G×4EÑ4E×4QÑ4QØ,9×,>Ñ,>õ5*ç*/©%ð50ó(&÷ %&ð %&ó:&ùs   ƒC˜
C£ACÁ7ACFTN)r„  r#  rÎ   )Úlocal_buffer_layoutr  Ú	local_bufÚlocal_to_global_buffersrð  s   && €€rX   Útry_share_local_bufferÚsCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_bufferŽ  sx   ø€ Û-: 	Ø#6×:JÑ:JÖ#JÏsËsô S"ð :QØ(1¯©ö:&óS"ÏsÏsÊsô S"ð :QØ(1¯©ö:&óS"÷ P"ô P"ð ,5Ò$4ñ .;ñ $(rW   Úlocal_buffer_datarÔ   )rÎ   r„  z&expected local_buffer.name is not Noneú?expected isinstance(_node, (FusedSchedulerNode, SchedulerNode))c                ó2   € V ^8„  d   QhR\         R,          /# )r’   r¾   F)r	   )r–   s   "rX   r—   r×  Å  s   € ÷ !ñ !¼À½ñ !rW   c                  ó€   <€ SP                    F,  p \        P                  P                   P                  V 4       K.  	  R # r½  )r¼  r5   rí  Úremove)Úremoved_bufferÚscopes    €rX   Úfallback_without_local_buffersÚ{CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.fallback_without_local_buffersÅ  s3   ø€ Ø*/×*?Ô*?˜ô Ÿ™×/Ñ/×6Ñ6°~ÖFñ +@ñ !rW   c                ó$   € V ^8„  d   QhR\         /# r�  r°  )r–   s   "rX   r—   r×  Ï  s   € ÷ !ñ !´Dñ !rW   c                  ó  <€ S F|  p V P                   pVf   \        R4      hVP                  pVf   \        R4      hVR SP                    F0  pVP                  \
        P                  P                  8w  g   K.    R# 	  K~  	  R# )Nzexpected loop_nest is not Nonezexpected loops is not NoneTF)rR  r…   rL  r'  rK  r”   ÚSÚOne)r[  rR  rL  rj  rY  r  s       €€rX   Úhas_tiled_fused_outer_loopÚwCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.has_tiled_fused_outer_loopÏ  s{   ø€ Û,AÐ(Ø$4×$>Ñ$>˜	Ø$Ò,Ü"0Ð1QÓ"RÐRØ )§¡˜Ø š=Ü"0Ð1MÓ"NÐNØ$)Ð*H¨D×,HÑ,HÓ$I˜DØ#Ÿz™z¬U¯W©W¯[©[Ö8Û'+ó %Jñ -Bñ !rW   )Úlocal_buffer_numberr¼  ).r§   r  r…   Úclearr#  r%  r   r.  r$   r¡  r‚  r†  rÛ   Úget_outputsr_  r  r   rA  r�  r'  Úis_contiguousrB  r«  r­  ÚFixedLayoutr!  r‡   ÚBufferr'  rÎ   rK   r™  Úadd_local_bufferr"   Úkernel_proxy_clsró  r]  r   Ú!cpp_outer_loop_fused_inner_countsÚCppOuterLoopFusedCountr  ri  Úfinalize_kernelr  r  Úfrom_iterable)r  r  ró  Úglobal_buffer_layoutÚsize_offsetrë  Úlocal_buffer_striderB  Úlocal_buffer_sizeÚszrõ  rø  Úlocal_buf_prefixÚlocal_buffer_usedrƒ  r3  r[  r	  r	  Úouter_fusion_cpp_kernel_proxyrÛ  r÷  rç  r/  r 	  rð  rY  ra  Ú
nodes_listr1  s   f                   @@@@@@€€€€rX   Ú$try_outer_loop_fusion_with_local_bufÚSCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf   sÔ  ÿ€ ô ˜dÔ$?×@Ò@Ü$ØLóð ð "×'Ñ'Ô)Ø×ÑÔõ
#ð .0ˆMàBDÐ#ß‹sõ à!×1Ñ1Ô3ó�s�sŠsõ à!×1Ñ1Ô3ó÷ ó ô <F»<Ð'Ø&*§n¡n×&6�Nä% n´m×DÒDÜ,ØPóð ð ,×/Ñ/°×0GÑ0GÓ0IÔJà&×3Ñ3×5Ò5Ü˜~×9Ñ9Ó;Ó<ÀÔAá à'5×'AÑ'AÓ'CÀAÕ'FÐ$ß“sô ØBR×BXÒBXó—s—s’sô ØBR×BXÒBXó÷ ô ð )9×(=Ñ(=˜Ü)¨-¼×9JÑ9J×KÒKÜ"0Ø Wó#ð ð 0=×/GÑ/GÓ/IÐ,Ø&*×&BÑ&BÄSÙ+¨NÓ;óFõ '˜öð8 1×>Ñ>×@Ò@Ù <× >Ò >â$à9;Ð+Ø!5×!<Ñ!<¸RÕ!@˜Ù,;¸NÓ,KØ'˜Lð-Ð)ô #+Ð+<Ö"=˜BØ/×6Ñ6°q¸&ÔAØ"�LšFñ #>ô /1¯nªnØ0×7Ñ7Ø0×6Ñ6Ø-Ø/ó	/Ð+ö(ð" ,?Ð(á,BØ/ó-Ð)÷  1ä02·	²	Ø(8Ð'9¸¼3¸}Ó;MÐ:NÐ%OØ':ô1Ð-ð *×0Ñ0Ð1BÔCØNPÐ3Ð4E×4JÑ4JÑKà/Ð0A×0FÑ0FÕG×NÑNØ)÷ñU '7ô\ $ L×$5Ñ$5×6Ô6¸%Ü�}Ó%¨Ô)Û(5˜Ø'×,Ñ,Ò4Ü"0Ø Hó#ð ð ×.Ñ.Ø(Ð*AÀ,×BSÑBSÕ*Töñ )6ð "×1Ñ1Ö3�EÜ% eÔ.@Ä-Ð-P×QÒQÜ,Ø]óð ð (,×'<Ñ'<¸\Ó'JÐ$Ø$×2Ñ2°5·?±?Ó3DÔEØ)×0Ñ0Ð1AÔBØ×%Ñ% e§o¡oÓ&7Ö8ñ 4÷!ð !÷!ñ !ô �}Ó%¨Ô)Ñ.H×.JÒ.JÙ9Ó;÷Y 7Ò6ð\ ×>Ñ>Ø)¨4×+GÑ+G÷ò ñ :Ó;÷c 7Ò6ôd ×9Ñ9×@Ñ@Ü×2Ò2ÜÐ1Ó2Ü,/°×0CÑ0CÓ,Dôôð 15×0OÑ0OØ)ó1Ð-ð ×,Ñ,Ø1Ø@”i—o‘o×3Ñ3°JÓ?Ð@ô÷w 7ñ@ ÷A 7Ö6ñ@ ús&   Í&D#U+Ò
U+Ò!U+Ò=U+ÓBU+Õ+U<	Fr­  rû  N)ra  r   rÂ  r§   r  r…   r
	  rƒ   rÀ  r   rÁ  r%  r"   r$   r.  r	  ró  r	  )
r1  r  rÂ  r	  r3  Ú_nodesr[  rY  ra  r	  s
   f&     @@@rX   Úcodegen_outer_loop_nodeÚ%CppScheduling.codegen_outer_loop_node  s,  û€ ð ×(Ñ(ˆÜ)0×)OÑ)OÐ&Ø=?ÐØ02ˆ
Ü˜$Ô ;×<Ò<Ü ØHóð ÷Q	ó Q	ñf 4°D×9Ò9à5SŒGÔ2Ø!×'Ñ'Ô)Ø×ÑÔô —‘×'Ñ'×-Ñ-¸eÐ-×DÕDØ!×1Ñ1Ö3�EÜ% eÔ.@Ä-Ð-P×QÒQÜ,Ø]óð ð 38·/±/Ó2C�FØ'+×'<Ñ'<¸\Ó'JÐ$Ø$×2Ñ2°6Ô:Ø ×0Ñ0Ð1AÖJñ 4÷ EÑDñ :÷ E×DÐDús   Â=BEÅE	c                ó@   <€ V ^8„  d   QhRS[ S[,          S[,          /# rÏ  )r  r"   r$   )r–   r  s   "€rX   r—   r"    s    ø€ ÷ )ñ )á)Ñ,>Õ>ÁÕNñ)rW   c                ó†  € V P                   p\        V\        4      '       d   V P                  V4       MTVP	                  4       pV P                  V4      pV P                  V4      pVP                  V4       VP                  WC4       V P                  4       pV\        P                  8”  d   V P                  R4       R# R# )z2
Turn a set of pre-fused nodes into a C++ kernel.
TN)ra  r§   r  r!	  r.  rÌ  r	  ró  r	  Ú_get_scheduled_num_argsr  ÚMAX_FUSED_KERNEL_ARGS_NUMr/  )r1  r  ra  r¼  r[  Úargs_nums   &&    rX   Úcodegen_nodeÚCppScheduling.codegen_node  s¤   € ð ×(Ñ(ˆä�dÔ7×8Ò8Ø×(Ñ(¨Õ.à)-¯©Ó)9ˆEØ×'Ñ'¨Ó.ˆEØ#×4Ñ4°\ÓBÐØ×*Ñ*¨5Ô1Ø×(Ñ(Ð)9ÔAà×/Ñ/Ó1ˆØ”m×=Ñ=Ô=Ø×"Ñ" 4Ö(ñ >rW   c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r’   r  r¾   r�  )r–   r  s   "€rX   r—   r"    s   ø€ ÷ 
ñ 
Ñ$5ð 
¹$ñ 
rW   c                óz   € \        V\        4      ;'       d%    \        VP                  \        P                  4      # r  )r§   r$   r  r   ÚCppTemplateBuffer)r1  r  s   &&rX   Úis_cpp_templateÚCppScheduling.is_cpp_template  s1   € Ü˜$¤Ó.÷ 
ð 
´:Ø�I‰I”r×+Ñ+ó4
ð 	
rW   c                óL   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[S[ ,          /# )r’   Útemplate_nodeÚepilogue_nodesÚprologue_nodes)r   r   )r–   r  s   "€rX   r—   r"  !  s:   ø€ ÷ ^)ñ ^)á(ð^)ñ !Ñ!2Õ3ð^)ñ !Ñ!2Õ3ñ	^)rW   c                óp  € V'       d   \        R4      hV Uu. uF#  p\        V\        \        34      '       g   K!  VNK%  	  pp\        R,          R;;,          ^,          uu&   \        R,          R;;,          \        V4      ,          uu&   V P                  V4      '       g   \        R4      h\        \        V4      pVP                  w  pw  rVVR8w  d   \        R4      h\        \        P                  VP                  4      pV Uu. uF  qˆP                  NK  	  p	p\        ;QJ d    R V	 4       F  '       d   K   RM	  R	M! R V	 4       4      '       g   \        R
4      hR p
V
! WqP                  V	4      pVP                  VVV	R7      w  rÍT;_uu_ 4        \        VP                  4      '       g   VP!                  4        V F  pVP!                  4        K  	  V! 4       pRRR4       \"        P$                  ! V4      ;_uu_ 4        V.VOpV P'                  XVVP(                  4      pRRR4       \        VP                  4      '       dÎ   \        VP*                  4      ^8w  d   \        R4      hVP*                  ^ ,          P,                   F‡  p\        VP                  \.        4      '       g   \        R4      h\        VP                  P                  \        P0                  4      '       g   \        R4      hVP                  P!                  4        K‰  	  V P3                  XX4       VP5                  VV4       \"        P6                  ;P8                  VP8                  ,          un        V P;                  4        R# u upi u upi   + '       g   i     EL³; i  + '       g   i     EL~; i)z7
Codegen a CPP template, possibly with fused epilogues
zexpected not prologue_nodesÚinductorÚcpp_templated_kernel_counterÚcpp_epilogue_fusion_counterzlTemplate node passed to CppScheduler.codegen_template must be a SchedulerNode that wraps a CppTemplateBufferzexpected rnumel == ()c              3   óV   "  € T F  p\        V\        P                  4      x € K!  	  R # 5ir  )r§   r   rA  )r  rv  s   & rX   r  Ú1CppScheduling.codegen_template.<locals>.<genexpr>A  s"   é € ÐPÑ>O¸”J˜q¤"×"3Ñ"3×4Ð4Ó>Oùr	  FTz9Epilogue nodes must all be instances of ir.ComputedBufferc                 ó*  a€ S'       g   R # V P                  4       V9  d   \        R4      hWP                  4       ,          P                  p\        ;QJ d)    V3R lV 4       F  '       d   K   R '       * # 	  R'       * # ! V3R lV 4       4      '       * # )Fz6expected template_buffer.get_name() in outputs_by_namec              3   óš   <"  € T F@  p\        VP                  \        4      ;'       d    VP                  P                  S9   x € KB  	  R # 5ir  )r§   r  r   )r  r[  r1	  s   & €rX   r  ÚZCppScheduling.codegen_template.<locals>.template_buffer_has_other_users.<locals>.<genexpr>Q  sE   øé € ð ñ "�Dô ˜4Ÿ9™9Ô&7Ó8÷ 5ð 5Ø—I‘I—N‘N nÑ4ô5ã!ùs
   ƒ%A©"AT)r‚  r…   r_  r#  )Útemplate_bufferÚoutputs_by_namer1	  r_  s   &&f rX   Útemplate_buffer_has_other_usersÚGCppScheduling.codegen_template.<locals>.template_buffer_has_other_usersF  s“   ø€ ÷ "Ùà×'Ñ'Ó)°Ô@Ü$ØLóð ð $×$<Ñ$<Ó$>Õ?×EÑEˆEß“sô ñ "ó—s”sô ð ’sô ð �sô ñ "óó ô ð rW   )Ú$flag_template_buffer_has_other_usersr1	  NzSMulti outputs template should be with 1 output template buffer of MultiOutputLayoutz?Multi outputs template should be with ExternKernelSchedulerNodez7Multi outputs template has multi users with MultiOutputrV   )r…   r§   r$   r"   r   rÛ   r-	  r   r¬  r   r,	  r  r#  r=	  Úmake_kernel_renderr*   rè  r5   Úset_kernel_handlerÚdefine_kernelr™  Úoutputsr_  r    r†  Úcodegen_commentÚcall_kernelrí  r¼  Úfree_buffers_in_scheduler)r1  r0	  r1	  r2	  Úepilogue_noderÔ   ÚrnumelÚctbrv  Úepilogue_ir_nodesr>	  r@	  re  Úrenderr  Úsrc_codeÚnode_scheduleÚkernel_namer[  s   &&&&               rX   Úcodegen_templateÚCppScheduling.codegen_template!  s  € ÷ Ü Ð!>Ó?Ð?ñ
 "0ó
á!/�Ü˜-¬-Ô9KÐ)L×M÷ ˆMÙ!/ð 	ð 
ô 	�ÕÐ;×<ÀÕAÓ<Ü�ÕÐ:×;¼sÀ>Ó?RÕRÓ;Ø×$Ñ$ ]×3Ò3Ü Ø~óð ô œ]¨MÓ:ˆØ&×,Ñ,‰ˆ‰;ˆAØ�RŒ<Ü Ð!8Ó9Ð9Ü$(¬×)=Ñ)=¸}×?QÑ?QÓ$RˆÙHVÓ7WÉÀ1¿¼ÉÐÐ7Wß“ÑPÑ>OÓP——’ÑPÑ>OÓP×PÒPÜ ØKóð ò	ñ" 0OØ×.Ñ.Ð0Aó0
Ð,ð ×/Ñ/ØØ1UØ,ð 0ó 
‰ˆ÷
 ŠVÜ,¨]×-?Ñ-?×@Ò@Ø×&Ñ&Ô(Û&�Ø—‘–ñ 'á“xˆH÷ ô ×!Ò! &×)Õ)Ø*Ð<¨^Ð<ˆMØ×,Ñ,¨X°}ÀfÇkÁkÓRˆK÷ *ô % ]×%7Ñ%7×8Ò8ô �=×(Ñ(Ó)¨QÔ.Ü$Øióð ð &×-Ñ-¨aÕ0×6Ô6�Ü! $§)¡)Ô-F×GÒGÜ(ØYóð ô " $§)¡)§.¡.´"·.±.×AÒAÜ(ØQóð ð —	‘	×"Ñ"Ö$ñ 7ð 	×Ñ˜]¨KÔ8Ø×Ñ˜;¨Ô,Ü	�‰×Ò 6×#9Ñ#9Õ9ÕØ×&Ñ&Ö(ùòc
ùò$ 8X÷> �VˆVú÷ *×)Ð)ús)   ˜N»NÄ	NÆANÈ#N$ÎN!	Î$N5	c                ó6   € V P                   P                  4       # r  )ra  Úget_num_argsr7  s   &rX   r%	  Ú%CppScheduling._get_scheduled_num_args�  s   € Ø× Ñ ×-Ñ-Ó/Ð/rW   c                ó   € V P                   # r  r.  r7  s   &rX   Úready_to_flushÚCppScheduling.ready_to_flush„  s   € Ø×#Ñ#Ð#rW   c                ó   € R # r  rV   r7  s   &rX   Úcodegen_syncÚCppScheduling.codegen_sync‡  s   € ÙrW   c                óÖ  € \         P                  P                  pWP                  9   d   VP                  V,          pV# \        P
                  P                  '       d%   \        V\        P
                  P                  4      MR pRP                  RWdP                  4       .4      pWTP                  V&   \         P                  P                  '       d   TMRpVP                  \        \        P                  4      V4      pVP                  \        \        P                  4      V4      pVP                  RR4      pVP!                  R4      pVP#                  RV4      p	\$        '       d   VP#                  RV	^,           4      p	WV	^,             R2p
\'        4       pVf   V P(                  P*                  MTpVP-                  4       w   rÞ\         P                  P                  '       g   VP/                  R	V: R
24       VP1                  VRR7       \         P                  P                  '       g   VP/                  R4       VP3                  VVP5                  4       RV
R7       V# )rS   rÔ   rù  re  z#pragma CMTz//z
extern "C"r›   z;
zasync_compile.cpp_pybinding(z, r'''T)Ústripz''')F)ÚgpuÚcpp_definition)r5   rí  Úwrapper_codeÚsrc_to_kernelr   rù  Údescriptive_namesr(   r·  Únext_kernel_suffixÚcpp_wrapperrO  rp   r-   ÚKERNEL_NAMEÚDESCRIPTIVE_NAMEÚrfindr¾  rU   r=   ra  r™  Úcpp_argdefsrÉ   rb  rC	  Úgetvalue)r1  rM	  r¼  Úkernel_argsr2  rO	  Ú
fused_nameÚkernel_decl_nameÚ
first_charÚ	last_charÚkernel_definitionÚcompile_wrapperr™  rÔ   Ú	arg_typess   &&&&           rX   rC	  ÚCppScheduling.define_kernelŠ  s  € Ü—'‘'×&Ñ&ˆØ×,Ñ,Ô,Ø!×/Ñ/°Õ9ˆKðR ÐôK —:‘:×/×/Ð/ô & e¬V¯Z©Z×-IÑ-IÔJàð ð
 Ÿ(™( E¨:×7QÑ7QÓ7SÐ#TÓUˆKØ.9×!Ñ! (Ñ+Ü./¯g©g×.A×.AÐ.A™{ÀxÐØ×'Ñ'¬¬K×,CÑ,CÓ(DÐFVÓWˆHØ×'Ñ'¬¬K×,HÑ,HÓ(IÈ;ÓWˆHð  ×'Ñ'¨°tÓ<ˆHð "Ÿ™¨Ó5ˆJØ Ÿ™ c¨:Ó6ˆIß‹{à$ŸM™M¨#¨y¸1­}Ó=�	Ø#+¸ÀQ½Ð#GÐ"HÈÐ LÐä,Ó.ˆOØ-8Ò-@�4×$Ñ$×)Ò)ÀkˆDØ"×.Ñ.Ó0‰OˆAˆqÜ—7‘7×&×&Ð&Ø×)Ñ)Ø2°9±-¸vÐFôð ×"Ñ" 8°4Ð"Ô8Ü—7‘7×&×&Ð&Ø×)Ñ)¨&Ô1Ø×!Ñ!ØØ×(Ñ(Ó*ØØ0ð	 "ô ð ÐrW   c                ó¨  € V P                   P                  4       pV'       EdŽ   V P                  WP                   P                  4      pV P	                  V P                   P                  V4       \
        P                  P                  '       d)   \        P                  P                  P                  4        \
        P                  P                  '       d_   \
        P                  P                  '       d?   \        P                  P                  P                  VV P                   P                  4       V P                   P                  \        P                  P                  V4       \
        P                  P                  '       d)   \        P                  P                  P                  4        V P!                  4        V P#                  R 4       R# rƒ  )ra  Úcodegen_grouprC	  Úscheduled_nodesrE	  r   rù  Úenable_kernel_profiler5   rí  r_	  Ú write_kernel_context_guard_beginÚenable_kernel_context_guardÚwrite_kernel_context_guardrF	  Úwrite_kernel_context_guard_endr(  r/  )r1  rM	  rO	  s   &  rX   ÚflushÚCppScheduling.flush¸  s*  € Ø×$Ñ$×2Ñ2Ó4ˆßˆ8Ø×,Ñ,Ø×+Ñ+×;Ñ;óˆKð × Ñ  ×!2Ñ!2×!BÑ!BÀKÔPÜ�z‰z×/×/Ð/Ü—‘×$Ñ$×EÑEÔGä—
‘
×0×0Ð0Ü—J‘J×:×:Ð:ä—‘×$Ñ$×?Ñ?ØØ×%Ñ%×5Ñ5ôð ×Ñ×)Ñ)¬!¯'©'×*>Ñ*>ÀÔLÜ�z‰z×/×/Ð/Ü—‘×$Ñ$×CÑCÔEà×ÑÔ!Ø×Ñ˜uÖ%rW   c                ót   € \         P                  P                  p\        VV4      pVP	                  W$4       R # r  )r5   rí  r_	  r   Úwrite_provenance_debug_handle)r1  rN	  rO	  r2  Údebug_handles   &&&  rX   rE	  ÚCppScheduling.codegen_commentÐ  s1   € ä—'‘'×&Ñ&ˆÜ>Øàó
ˆð
 	×-Ñ-¨kÖHrW   c                ó6   <€ V ^8„  d   Qh/ S[ S[,          ;R&   # )r’   r	  )r  rZ  )r–   r  s   "€rX   r—   r"  ú  s   ø‡ ‚ ñ ™>Õ*Ñ;ò	 rW   )r)  ra  r  ).r  r  r  r  rZ  r	  r&	  r   r6   ÚINPLACE_BUFFERSÚREDUCE_TO_SINGLE_ELEMENTr$  rk  r%  r0  r/  r5  r(  r>  rL  rZ  r(  r_  rf  rz  r~  r‰  ra  r`  rš  rž  rÌ  r!	  r(	  r-	  rP	  r%	  rV	  rY	  rC	  rz	  rE	  r	  r
  r  rl  rm  s   @@rX   r  r  ú  s  ù‡ € ð .<Ðð !$ÐÙ!à×*Ñ*Ø×3Ñ3ð	
óÐð ÷$ó ð$õ%÷&ð &òNò*÷
ð 
ò6ò3òj=÷6ð òI/òV>ò(	<÷
ð 
ò:)òx

òò=÷Wð W÷rwKð wK÷r)ð )÷,
ð 
÷
^)ð ^)ò@0ò$òô,ò\&ô0I÷m … rW   r  c                   ó`   a a€ ] tR tRt oV 3R ltR tR tR tR
V3R lR lltR t	R	t
VtV ;t# )r8  iÛ  c                ó   <€ \         SV `  4        \        4       V n        \	        4       V n        \        V P
                  4      V n        \        P                  ! 4       V n
        V P                  P                  V P                  4       . V n        R # r  )r/  r0  r?   r™  r7   Ú
loops_codeÚWorkSharingr$  rÇ   rÈ   rÓ   rÊ   rt	  )r1  r4  s   &€rX   r0  ÚKernelGroup.__init__Ü  s^   ø€ Ü‰ÑÔÜ“LˆŒ	Ü&›.ˆŒÜ˜dŸo™oÓ.ˆŒÜ×)Ò)Ó+ˆŒ
Ø�
‰
× Ñ  §¡Ô)Ø!ˆÖrW   c                ó<   € V! V P                   \        4       .VO5!  # r  )r™  r,   )r1  r&  r™  s   &&*rX   r¥  ÚKernelGroup.new_kernelå  s   € Ù�4—9‘9Ô2Ó4Ð<°tÓ<Ð<rW   c                óŠ   € V ;P                   V,          un         V P                  pV P                  pVP                  W44       R # r  )rt	  r…	  r$  r�  )r1  r¥  r¼  r¹   r$  s   &&&  rX   r	  ÚKernelGroup.finalize_kernelè  s3   € Ø×Ò Õ%ÕØ�‰ˆØ�W‰WˆØ× Ñ  Ö*rW   c                óV   € V P                   P                  4       w  rp\        V4      pV# r  )r™  rg	  rÛ   )r1  Úarg_defsÚ
_call_argsÚ
_arg_typesr'	  s   &    rX   rS	  ÚKernelGroup.get_num_argsî  s'   € Ø+/¯9©9×+@Ñ+@Ó+BÑ(ˆ˜jÜ�x“=ˆØˆrW   c                ó    <€ V ^8„  d   QhRS[ /# r�  rr  )r–   r  s   "€rX   r—   ÚKernelGroup.__annotate__ó  s   ø€ ÷ 4ñ 4©#ñ 4rW   c                óh  € V P                   P                  4        V P                  '       g   R # \        4       p\        P
                  P                  ;'       d    \        P                  R9   pV'       d   VP                  R.4       VP                  R4       Vf   \        \        P                  4      MTpVf   \        \        P                  4      MTpV P                  P!                  4       w  p pRP#                  ^4      P%                  V4      p\'        4       p\        P
                  P(                  '       d   RMR p	VP                  RV RV	 RV R	V R
2	4       VP+                  4       ;_uu_ 4        VP                  R4       VP                  R4       V'       d_   \,        P.                  P0                  p
V
e   R\        V
4      ,           R,           MR pVP                  RWµ,            RWµ,            R2.4       V P                  P3                  4        F  w  rÍVP                  RV RV R24       K  	  VP5                  V P6                  4       VP                  R4       VP                  R4       RRR4       VP9                  4       #   + '       g   i     TP9                  4       # ; i)rS   z3#include <torch/csrc/inductor/aoti_runtime/utils.h>z+#include <torch/csrc/inductor/cpp_prefix.h>Nz,
ÚC10_ALWAYS_INLINE_ATTRIBUTEzextern "C" z void r�   r’  r›   z3std::atomic<int> inductor_cpu_integer_div_error{0};zFinductor_cpu_integer_div_error_flag = &inductor_cpu_integer_div_error;Úgraph_rÔ   z9torch::aot_inductor::RAIIAtenRecordFunctionHandle record_z_("z", nullptr);rÝ   rÀ   rÁ   z.inductor_cpu_integer_div_error_flag = nullptr;zHinductor_cpu_throw_if_integer_div_error(inductor_cpu_integer_div_error);)ÚlinuxrQ   )rÓ   rh  rt	  r7   r   rù  ru	  ÚsysÚplatformræ   rÉ   rp   r-   rd	  re	  r™  rg	  Úljustr·  rY   Úforce_inline_kernelrË   r5   rí  Úgraph_idÚaliasesrb  r…	  rh	  )r1  rÎ   r¹   ru	  rk	  rO	  r�	  rÔ   Úfunc_export_declÚinline_attrr›	  r`  ÚoldÚnews   &&            rX   rs	  ÚKernelGroup.codegen_groupó  sE  € Ø�
‰
×ÑÔØ×#×#Ð#ÙÜ‹~ˆô !'§
¡
× @Ñ @÷ !
ð !
ÄSÇ\Á\ð V
ñ F
Ð÷ !Ø�O‰OÐRÐSÔTØ�‰ÐDÔEð <@º<œ3œ{×6Ñ6Ô7ÈTÐØ;?º<”cœ+×6Ñ6Ô7ÈTˆØŸ™×.Ñ.Ó0‰ˆ�!�QØ—;‘;˜r“?×'Ñ'¨Ó1ˆÜ1Ó3Ðä-3¯Z©Z×-K×-KÐ-KÑ)ÐQSð 	ð 	�‰ØÐ*Ð+¨6°+°¸aÐ@PÐ?QÐQRÐS[ÐR\Ð\]Ð^ô	
ð
 �[‰[�]�]Ø�N‰NÐPÔQØ�N‰NØXô÷ %ÜŸ7™7×+Ñ+�Ø;CÒ;O˜¤C¨£MÕ1°CÖ7ÐUW�Ø—‘ð&Ø&,Õ&:Ð%;¸3¸vÕ?SÐ>TÐT`ðbðôð !ŸI™I×-Ñ-Ö/‘�Ø—‘  s e¨3¨s¨e°1Ð5Ö6ñ 0à�K‰K˜Ÿ™Ô(Ø�N‰NÐKÔLØ�N‰NØZô÷) ð. �}‰}‹Ð÷/ Ž]ð. �}‰}‹Ðús   Å9DJÊJ1	c                ój   € V P                   P                  4       w  r4pVP                  VVR VR7       R# )F)Útritonrp	  N)r™  rg	  Úgenerate_kernel_call)r1  r2  rO	  rÔ   Ú	call_argsrp	  s   &&&   rX   rF	  ÚKernelGroup.call_kernel)  s8   € Ø"&§)¡)×"7Ñ"7Ó"9Ñˆ�iØ×$Ñ$ØØØØð	 	%ö 	
rW   )r™  r…	  rt	  rÓ   r$  r  )r  r  r  r  r0  r¥  r	  rS	  rs	  rF	  r
  r  rl  rm  s   @@rX   r8  r8  Û  s.   ù‡ € õ"ò=ò+ò÷
4ò 4÷l
ò 
rW   r8  c                   óD   a € ] tR tRt o R tR tR tR tR tR t	Rt
V tR	# )
r†	  i3  c                ób   € Wn         R V n        RV n        \        P                  ! 4       V n        R# rƒ  )r¹   Úin_parallelr�  rÇ   rÈ   rÓ   )r1  r¹   s   &&rX   r0  ÚWorkSharing.__init__4  s'   € ØŒ	Ø ˆÔØˆÔÜ×)Ò)Ó+ˆŽ
rW   c                ó–  € V P                   '       d!   WP                  8w  d   V P                  4        V P                   '       Eg   Wn        R V n         Rp\        P                  P
                  ^8¼  d   RpMV\        P                  ! 4       8X  d   R pMRpV'       g!   \        P                  P                  '       d   V P                  P                  R4       MV P                  P                  RV R24       V P                  P                  V P                  P                  4       4       V P                  P                  R4       R# R# )TFz#pragma omp parallelz!#pragma omp parallel num_threads(r›   zint tid = omp_get_thread_num();N)r©	  r�  rh  r   rù  rn  ÚosÚ	cpu_countr˜  r¹   rÉ   rÓ   rÊ   rË   )r1  rn  Úuse_dynamics   && rX   rT  ÚWorkSharing.parallel:  sí   € Ø××Ð ×+;Ñ+;Ô ;à�J‰JŒLØ××ÑØ&ÔØ#ˆDÔàˆKÜ�z‰z×!Ñ! QÔ&à#‘ØœBŸLšL›NÔ*à"‘ð $�çœfŸj™j×8×8Ð8Ø—	‘	×#Ñ#Ð$:Õ;à—	‘	×#Ñ#Ð&GÈÀyÐPQÐ$RÔSØ�J‰J×$Ñ$ T§Y¡Y×%5Ñ%5Ó%7Ô8Ø�I‰I×ÑØ1öñ)  rW   c                ót   € V P                   '       d   V P                  P                  R 4       V P                   # )z#pragma omp single)r©	  r¹   rÉ   r7  s   &rX   r}  ÚWorkSharing.singleV  s-   € Ø××ÐØ�I‰I×ÑÐ 4Ô5Ø×ÑÐrW   c                óH   € V P                   P                  4        R V n        R# rƒ  )rÓ   rh  r©	  r7  s   &rX   rh  ÚWorkSharing.close[  s   € Ø�
‰
×ÑÔØ ˆÖrW   c                ó:   € V P                   P                  4        V # r  )rÓ   r}  r7  s   &rX   r}  ÚWorkSharing.__enter___  s   € Ø�
‰
×ÑÔØˆrW   c                ó>   € V P                   P                  WV4       R # r  )rÓ   r…  r�  s   &&&&rX   r…  ÚWorkSharing.__exit__c  s   € Ø�
‰
×Ñ˜H¨vÖ6rW   )r¹   r©	  r�  rÓ   N)r  r  r  r  r0  rT  r}  rh  r}  r…  r
  r  r  s   @rX   r†	  r†	  3  s(   ø‡ € ò,òò8 ò
!ò÷7ð 7rW   r†	  c                   óÞ   a € ] tR tRt o RtRt]P                  P                  t	]P                  P                  t
]P                  P                  t^ tRtRtRtRtR tR tR tV 3R ltRtV tR# )	r  ig  NFc                ó|   € \         P                  ! 4       pV'       d   VP                  4       V n        R# ^ V n        R# r`  )r   rÃ  rï  Úsimd_nelements)r1  r%  s   & rX   Ú__post_init__ÚLoopLevel.__post_init__|  s-   € ô .9×-EÒ-EÓ-GˆßAO >×#;Ñ#;Ó#=ˆÖÐUVˆÖrW   c                ó"  € \         P                  ! V4      p\        V P                  V P                  4      pW#n        R Vn        \        VP                  V4      V,          Vn        V P                  Vn	        RVn
        V P                  Vn        V# )TF)r”   rX  r  r¬   rI  rK  Úsimd_vecr   rÅ  rT  Ú	collapsedr†  )r1  r“  Úsympy_factorrj  s   &&  rX   rÃ  ÚLoopLevel.tileˆ  sl   € Ü—}’} VÓ,ˆÜ˜Ÿ™ 4§9¡9Ó-ˆØ!Œ
ØˆŒÜ" 4§9¡9¨lÓ;¸lÕJˆŒØŸ™ˆŒØˆŒØ ×-Ñ-ˆÔØˆrW   c                ó8  € \        V P                  4      p\        V P                  4      p\        P                  P
                  '       d	   W8X  d   R # V P                  '       d"   V P                  ^8”  d   RV P                   R2MRpV P                  '       dS   RpV P                  ^8”  d   VRV P                   R2,          pV P                  '       d   VP                  RRV 24      pM_V P                  '       d   RpMJV P                  '       d   RV 2pM2V P                  '       g   \        P                  ! 4       '       d   R	pMRp\         R
V P                   RV 2pV P                   RV 2pV P                   P"                  '       d&   V P                   R\        V P                   4       2pM;V P                   R\        V P                   4       R\        V P                   4       R2pRV RV RV R2pV P$                  '       g	   V'       g   V.# WH.# )Nzsimd simdlen(r“  rS   z#pragma omp forz
 collapse(r›   z for z#pragma omp z#pragma GCC ivdepr�   r  r   r  z+=(z == 0 ? 1 : zfor(rÃ   )rE   rJ  rI  r   rù  Úno_redundant_loopsÚsimd_omprº	  rT  rO  r¾	  r†  r   r  rJ   r¬   rK  r&  r¿	  )	r1  Úoffset_exprÚ	size_exprÚsimdÚline1Ú
offset_strrã  Ú	steps_strÚline2s	   &        rX   rt  ÚLoopLevel.lines“  sÒ  € Ü! $§+¡+Ó.ˆÜ §	¡	Ó*ˆ	Ü�:‰:×(×(Ð(¨[Ô-EÙð �}�}ˆ} ×!4Ñ!4°qÔ!8ð ˜D×/Ñ/Ð0°Ñ3àð 	ð
 �=�=ˆ=à%ˆEØ�}‰}˜qÔ Ø˜: d§m¡m _°AÐ6Õ6�Ø�}�}ˆ}ØŸ™ g°°t°f¨~Ó>�øØ�]�]ˆ]Ø‰EØ�]�]ˆ]Ø" 4 &Ð)‰EØ×"×"Ð"¤{×'9Ò'9×';Ò';Ø'‰EàˆEÜ"�| 1 T§X¡X J¨a°¨}Ð=ˆ
Ø—h‘h�Z˜q  Ð,ˆØ�:‰:××ÐØŸ8™8˜* B¤{°4·:±:Ó'>Ð&?Ð@‰Ið
 —8‘8�*˜C¤¨D¯J©JÓ 7Ð8ð 9Ü" 4§:¡:Ó.Ð/¨qð2ð ð �z�l " X J¨b°°¸1Ð=ˆØ�>�>ˆ>§Ø�7ˆNØˆ~ÐrW   c                ó  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  ;R&   S[ P                  ;R&   S[ P                  ;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   # )r’   Nr¬   rI  rJ  rÅ  rK  rT  rÄ	  r¾	  r¿	  r†  )r”   r•   rk   ro   )r–   r  s   "€rX   r—   ÚLoopLevel.__annotate__g  s£   ø‡ ‚ á	�‰�dÕ	Ñ!ñ ñ �*‰*�tÕ
Ñ"ñ ñ �J‰JÑ%ñ	 ñ —
‘
Ñ)ñ ñ �:‰:Ñ#ñ ñ Ññ ñ  Ññ! ñ" Ññ# ñ$ Ññ% ñ& Ñò' rW   )rº	  )r  r  r  r  r¬   rI  r”   r	  ÚZerorJ  rÅ  r	  rK  rT  rÄ	  r¾	  r¿	  r†  r»	  rÃ  rt  r	  r
  r  r  s   @rX   r  r  g  si   ø‡ € à!€CØ"€DØŸ™Ÿ™€Fð #ŸW™WŸ\™\€JØŸ™Ÿ™€EØ€HØ€HØ€HØ€IØ€Lò
Wò	ò'÷Y ƒ rW   r  c                   ó    a € ] tR tRt o RtRtRt]V 3R lR l4       tR t	]
R 4       tR tR	 tV 3R
 lR ltR tV 3R lR ltV 3R ltRtV tR# )r@  i½  a>  
A loop-nest-like structure. It is built with the `build` method
as a loop nest and then will perform loop-tiling at some depth.

A typical case is for vectorization, where we typically do loop-tiling
at the innermost loop level. A more complicated case is when we do
2D tiling at both the innermost and outer levels.
Nc                ó    <€ V ^8„  d   QhRS[ /# )r’   re  ©ru  )r–   r  s   "€rX   r—   ÚLoopNest.__annotate__Ì  s   ø€ ÷ ñ ‘iñ rW   c                óH  € V P                   pV P                  pV P                  pVf   \        R4      hRp\	        \        W4      4       FI  w  pw  rg\        Wg4      pV'       g   V.pMVP                  V4       WS8¼  g   K8  V P                  Vn        KK  	  \        V4      p	V	# )z4Build a LoopNest with the given `kernel` as the leafNz$expected reduction_depth is not None)
r<  rW  r~  r…   rÌ   rc  r  r'  r†  r@  )
re  r<  rW  r~  rL  Úloop_idxr¬   rI  rj  rR  s
   &         rX   rŒ  ÚLoopNest.buildË  s™   € ð —?‘?ˆØ—‘ˆØ ×0Ñ0ˆØÒ"Ü Ð!GÓHÐHà(,ˆÜ%.¬s°8Ó/DÖ%EÑ!ˆH‘k�sÜ˜SÓ'ˆDßØ˜‘à—‘˜TÔ"ØÖ*Ø$*×$7Ñ$7�Ö!ñ &Fô ˜U“Oˆ	ØÐrW   c                ó,   € \        V P                  4      # r  )ro   rL  r7  s   &rX   Ú__bool__ÚLoopNest.__bool__á  s   € Ü�D—J‘JÓÐrW   c                óˆ  € V P                   f   \        ^ ^ R7      # ^ p^ pV P                   ^ ,          P                  p\        P                  ! ^4      pV P                    FF  pVP                  V8w  d    M4V\        VP                  VP                  4      ,          pV^,          pKH  	  R pV! V P                   4      pR R lpV\        V P                   4      8  EdZ   \        V\        P                  4      '       Ed9   \        V P                   V,          P                  \        P                  4      '       dþ   VR,          \        V P                   V,          P                  V P                   V,          P                  4      8  d°   Ve8   W'8”  d2   V P                   V,          P                  '       d   V! V 4      '       gu   Tp^ pV P                   V,          P                  p\        V\        V P                   4      4       F0  p	V P                   V	,          P                  V8w  d    MV^,          pK2  	  \        W!R7      # )aY  
Maximal allowed depth for parallelism: All reduction or non-reduction levels.
When the range of the first inner loop beyond the maximum parallel depth is much
larger than the range of all outer loops within the maximum parallel depth,
change the starting depth of parallelism to the first inner loop and recalculate
the maximum parallel depth.
rš  c                 óZ   € \        V 4       F  w  rVP                  '       g   K  Vu # 	  R # r  )rÌ   r¾	  )rL  rï   rj  s   &  rX   Úget_simd_vec_depthÚ7LoopNest.max_parallel_depth.<locals>.get_simd_vec_depthú  s&   € ä$ UÖ+‘�Ø—=—=’=Ø’Hñ ,ñ rW   c                ó$   € V ^8„  d   QhR\         /# )r’   rR  rO  )r–   s   "rX   r—   Ú1LoopNest.max_parallel_depth.<locals>.__annotate__  s   € ÷ 	ñ 	¬ñ 	rW   c                 ó  € \        V P                  \        4      '       g   \        R 4      h\        ;QJ d0    R V P                  P
                   4       F  '       g   K   R# 	  R# ! R V P                  P
                   4       4      # )z5expected isinstance(loop_nest.kernel, CppKernelProxy)c              3   óL   "  € T F  p\        V\        4      '       * x € K  	  R # 5ir  )r§   r(  )r  re  s   & rX   r  ÚILoopNest.max_parallel_depth.<locals>.has_scalar_kernel.<locals>.<genexpr>  s$   é € ð á6�Fô ˜v¤|Ó4×4Ñ4Û6ùrù  TF)r§   re  rZ  r…   rf   r&  )rR  s   &rX   Úhas_scalar_kernelÚ6LoopNest.max_parallel_depth.<locals>.has_scalar_kernel  sz   € Ü˜i×.Ñ.´×?Ò?Ü$ØKóð ÷ “3ñ à'×.Ñ.×6Ò6ó—3”3ð ’3ð �3ñ à'×.Ñ.×6Ò6óó ð rW   rP  )rL  rý   r†  r”   rX  r   rI  rK  rÛ   r§   rQ  )
r1  r   Ú	max_depthr†  Ú	num_stepsrj  rÜ	  Úsimd_vec_depthrã	  rï   s
   &         rX   r{  ÚLoopNest.max_parallel_depthä  s¯  € ð �:‰:ÒÜ °¸qÔAÐAàˆØˆ	Ø—z‘z !•}×1Ñ1ˆÜ—M’M !Ó$ˆ	Ø—J”JˆDØ× Ñ  LÔ0ÙØ!¤H¨T¯Y©Y¸¿
¹
Ó$CÕCˆIØ˜�NŠIñ	 ò	ñ ,¨D¯J©JÓ7ˆõ	ð œ˜DŸJ™J›Õ'Ü˜9¤e§m¡m×4Ó4Ü˜4Ÿ:™: iÕ0×5Ñ5´u·}±}×EÒEØ˜C•Ü�t—z‘z )Õ,×1Ñ1°4·:±:¸iÕ3H×3NÑ3NÓOôPð Ò*ØÔ.Ø—J‘J˜yÕ)×6×6Ð6Ù% d×+Ò+ð $ˆKØˆIØŸ:™: kÕ2×?Ñ?ˆLÜ˜;¬¨D¯J©J«Ö8�Ø—:‘:˜a•=×-Ñ-°Ô=ÙØ˜Q•’	ñ 9ô ¨IÔOÐOrW   c                óD  € VP                   V P                  4       P                   8”  d   \        R 4      hV P                  f   \        R4      h\	        V P                  4      VP                   8  d   \        R4      hV P                  VP
                  ,          pVP                   Vn        VP                  '       d   \        ;P                  ^,          un	        \        VP
                  ^,           VP                   4       F  pRV P                  V,          n        K  	  R# )z?Parallel depth cannot exceed the maximal allowed parallel depthNzexpected self.loops is not Nonez4expected len(self.loops) >= par_depth.parallel_depthT)rÿ   r{  r…   rL  rÛ   r   rT  r†  r   Úparallel_reduction_countrQ  r¿	  )r1  rV  rj  rï   s   &&  rX   r|  ÚLoopNest.mark_parallel&  sÚ   € Ø×#Ñ# d×&=Ñ&=Ó&?×&NÑ&NÔNÜ ØQóð ð �:‰:ÒÜ Ð!BÓCÐCÜˆt�z‰z‹?˜Y×5Ñ5Ô5Ü Ð!WÓXÐXØ�z‰z˜)×/Ñ/Õ0ˆØ!×0Ñ0ˆŒØ××Ðä×,Ò,°Õ1Õ,Ü�y×,Ñ,¨qÕ0°)×2JÑ2JÖKˆAØ&*ˆD�J‰J�q�MÖ#ó LrW   c                óÀ   € V P                   '       g   \        R4      hV P                   V,          P                  V4      V P                   V&   V P                   V,          # )z­
Do loop-tiling at the `depth` level with `factor`.
    for (x0 = 0; x0 < x0_end; x0++)
    ->
    for (x0 = 0; x0 < x0_end; x0 += factor)
See details in Note [tiled_size].
úexpected self.loops)rL  r…   rÃ  )r1  re  r“  s   &&&rX   rÃ  ÚLoopNest.tile7  sJ   € ð �z�zˆzÜ Ð!6Ó7Ð7Ø ŸJ™J uÕ-×2Ñ2°6Ó:ˆ�
‰
�5ÑØ�z‰z˜%Õ Ð rW   c                ó    <€ V ^8„  d   QhRS[ /# r�  rÒ	  )r–   r  s   "€rX   r—   rÓ	  D  s   ø€ ÷ ñ ™Iñ rW   c                óT   € V P                   '       g   \        R 4      hV P                   # )zexpected self.kernel)re  r…   r7  s   &rX   rY  ÚLoopNest.get_kernelD  s!   € Ø�{�{ˆ{Ü Ð!7Ó8Ð8Ø�{‰{ÐrW   c                ó   € Wn         R # r  ©re  rø  s   &&rX   r¿  ÚLoopNest.set_kernelI  s   € ØŽrW   c                ó    <€ V ^8„  d   QhRS[ /# )r’   Úlevelr  )r–   r  s   "€rX   r—   rÓ	  L  s   ø€ ÷ ,ñ ,¡Sñ ,rW   c                ó  € V P                   '       g   \        R 4      h\        V P                   4      V8  d   \        R4      hV\        V P                   4      8X  d   RMV P                   VR p\        W P                  4      # )rí	  z!expected len(self.loops) >= levelN)rL  r…   rÛ   r@  re  )r1  rö	  rL  s   && rX   rc  ÚLoopNest.from_loop_levelL  sc   € Ø�z�zˆzÜ Ð!6Ó7Ð7Üˆt�z‰z‹?˜UÔ"Ü Ð!DÓEÐEØ¤ T§Z¡Z£Ô0‘°d·j±jÀÀÐ6HˆÜ˜Ÿ{™{Ó+Ð+rW   c                ó^   <€ V ^8„  d   Qh/ S[ S[,          R,          ;R&   S[R,          ;R&   # )r’   NrL  re  )r$  r  ru  )r–   r  s   "€rX   r—   rÓ	  ½  s-   ø‡ ‚ ñ ‘	�?˜TÕ!Ñ(ñ ñ ˜ÕÑ#ò rW   ró	  )r  r  r  r  r  rL  re  r  rŒ  rØ	  r&   r{  r|  rÃ  rY  r¿  rc  r	  r
  r  r  s   @rX   r@  r@  ½  so   ø‡ € ñð %)€EØ#€Fà÷ó ðò* ð ñ?Pó ð?PòB+ò"!÷ð ò
÷,ð ,÷_ ƒ rW   r@  c                ój   € V ^8„  d   Qh/ ^ \         9   d   \        \        P                  ,          ;R&   # )r’   rÖ  )Ú__conditional_annotations__r$  rƒ   r‡   )r–   s   "rX   r—   r—      s)   € × Ñ ÷fò ”Tœ%Ÿ+™+Õ&ñ òg rW   )r[   r\   r]   r^   r_   r`   )
r`   r_   ra   rb   rc   rg   ri   rd   re   rf   )NNN)¨rû	  rÇ   rÎ  rS  r  rÖ  rU  r¬	  rÐ   r—	  r¾  Úcollections.abcr   r   Úenumr   Útypingr   r   r   r	   r
   r”   rƒ   Útorch.fxÚtorch._inductorr   Útorch._prims_commonr   r   Útorch.utils._ordered_setr   Útorch.utils._sympy.functionsr   r   r   Útorch.utils._sympy.symbolr   r   r   Ú_dynamo.utilsr   rS   r   r   r   r   r   Údebugr   rÙ  r   r"  r   r   r    r!   r"   r#   r$   r  r%   Úutilsr&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   Úvirtualizedr2   r3   r4   r5   Úcommonr6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   Ú	cpp_utilsrB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   r˜	  rU   ÚcacherY   Ú_loggingÚgetArtifactLoggerr  Úschedule_logÚNATIVE_OMP_RTYPESÚRTYPE_TO_CPPr4  ÚPYTHON_TO_CPPÚCONTAINER_PYTHON_TO_CPPr  r   r‚   r  rm   ro   rÅ  r!  r”  r•  r"  r#  rÖ  rŒ   r�   rµ   r©   r×   rê   rð   rû   Ú	dataclassrý   r  ro  r›  r�  Ú_initialize_pointwise_overridesr  ri  rm  ru  r(  ro  rÄ  rÆ  rÑ  rZ  rb  r  r  r8  r†	  r  r@  r—   )rû	  s   @rX   Ú<module>r
     s×  øðç Ð Û Û Û Û Û Û 	Û 	Û 
Û ß .Ý ß 9Õ 9ã ã Û Ý (ß @Ý /ß KÑ Kß OÑ Oå %ß <Õ <Ý ;Ý  ÷÷ ñ õ +÷÷ ÷ ó ÷ >Ó =÷÷ ÷ ó ÷÷ ÷ ÷ ñ ð& �l‰l˜gÑ%€ð ‡�ñ:ó ð:ð �~‰~×/Ñ/°¸*ÓE€áÒBÓCÐ à	ˆ3Ø
ˆCØˆsØ	ˆ5Ø	ˆ5ØˆhØˆhØ	ˆ4Ø�iØ�yð€ñ !òóÐ ð  ˆlØ	ˆ6ØˆXØ
ˆFØ	ˆ=ØÐ#ØÐ&ØˆlØˆlØˆlð€ð ˆMØ�ðÐ ð 
‡N�NØ	‡M�Mð€ð 
‡M�MØ	‡K�KØ	‡N�NØ	‡M�MØ	‡J�JØ	‡K‚KØ	‡J‚JØ	‡K‚KØ	‡K‚KØ	×ÒØ	×Òð*Ð ó ò)òD÷2)òjõ2õjõBVõ$/ð8 ×Ò÷ð ó ðôq!Ð"4ô q!÷h%!ñ %!òP'ô^"�;ô ^"ðB × ,Ò ,¨UÔ 3ôk7�lô k7ð\ × /Ò /°Ô 9Ø × %Ò %Ô 'ô:˜ô :ô@�ô @ôFQ�9ô Qôhg.�lô g.õT-$ð` ×Ò÷ð ó ð÷lKñ lKô^q

�Yô q

ôh!
˜9ô !
ôHF�tô Fô^I�Nô ^I÷BU
ñ U
÷p17ñ 17ðh ×Ò÷Rð Ró ðRðj ×Ò÷T,ð T,ó òT,rW   