+
    –°³j_.  ã                   óÚ  € ^ RI t ^ RIHt ^ RIt^ RIHt ^ RIHt ^ RIH	t	 ^RI
HtHtHt ^RIHt ^RIHtHtHtHt ^R	IHt ^R
IHtHtHtHt ^RIHtHtH t H!t!H"t"H#t#H$t$ ^RI%H&t&H't' ^RI(H)t)H*t*H+t+H,t, ]'       d   ^RIH-t- ^RIH.t. ] P^                  ! ]04      t1]PL                  Pd                  t2]R 4       t3]! R]3]	! R4      RR7      t4]! ]Pj                  R]2Pj                  Pl                  R7      t7]! ]Pj                  ]Pp                  Ps                  4       '       d   RMRR]2Pj                  Pt                  R7      t;]! ]Px                  R]2Px                  Pl                  R7      t=^ t>]P~                  ]P€                  ]P‚                  ]P„                  3tC]Pˆ                  ! ]2Pj                  4      R!RR/R ll4       tE]Pˆ                  ! ]2Px                  4      R^R^RR/R  l4       tFR# )"é    N)ÚTYPE_CHECKING)Úcounters)ÚCKGemmTemplate)Úload_kernel_template)ÚconfigÚirÚlowering)ÚMMKernelInputs)Ú	loweringsÚmake_pointwiseÚmake_reductionÚtransform_args©Úget_max_y_grid)Úautotune_select_algorithmÚExternKernelChoiceÚSymbolicGridFnÚTritonTemplate)Ú_use_cutlass_for_opÚuse_aten_gemm_kernelsÚuse_ck_gemm_templateÚuse_cpp_bmm_templateÚuse_cutlass_templateÚuse_nv_universal_gemm_templateÚuse_triton_template)ÚopsÚV)Ú_is_static_problemÚis_batch_stride_largest_or_zeroÚmm_argsÚuse_native_matmul)ÚChoiceCaller)ÚKernelTemplatec                ó’   € V! WR ,          4      V! W#R,          4      ,          p\        4       pV! V! W4      ^4      pV! W4      p	WiV3# )ÚBLOCK_MÚBLOCK_Nr   )
ÚbÚmÚnÚmetaÚcdivÚmaxÚtilesÚ
max_y_gridÚgrid_zÚgrid_ys
   &&&&$$    Ú[/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/_inductor/kernel/bmm.pyÚbmm_gridr2   .   sM   € á�˜•OÓ$¡t¨A°I­Ó'?Õ?€Eô  Ó!€JÙ‘�aÓ$ aÓ(€FÙ�!‹_€FØ˜6Ð"Ð"ó    ÚbmmÚ
triton_bmmT)ÚnameÚgridÚsourceÚ"cache_codegen_enabled_for_templatezat::bmm_out)Úop_overloadzat::_bmm_out_dtype_xpuzat::_bmm_out_dtype_cudaÚ	bmm_dtype)r6   r:   zat::baddbmm_outÚlayoutc          
     ó  a a$a%€ \         P                  P                  o%S P                  4       pS P	                  4       P
                  pV%3R lpV%3R lpVEf   VR9   Ed   WQP	                  4       P
                  8X  dé   WAP                  4       8X  dÕ   V\        9   dÊ   V! S P                  4       ^,          4      '       d§   V! VP                  4       ^,          4      '       d„   V! S P                  4       ^,          \        4      '       d\   \        P                  ! S R4      o \        P                  ! V^4      p\        P                  ! \        P                  ! S V4      ^R7      # \        ;QJ d    R S V3 4       F  '       d   K   RM	  RM! R S V3 4       4      '       Ed#   S P                  4       ^,          ^8X  g   VP                  4       ^,          ^8X  d\   \        P                  ! S R4      o \        P                  ! V^4      p\        P                  ! \        P                  ! S V4      ^R7      # R pR	 o$V$3R
 lp	V! S 4      '       d5   \         P                  P                  P                  ^ ,          p
V	! S V
4      o V! V4      '       d4   \         P                  P                  P                  ^,          pV	! W4      p\!        S V4      '       dÿ   \"        \$        P                  ,          ! S R4      o \"        \$        P                  ,          ! V^4      p\'        S V./ RRRR7      w  rÍ\(        P*                  P,                  '       dV   S P.                  \0        P2                  \0        P4                  39   d'   V 3R lpV Uu. uF  p\7        V4      ! V4      NK  	  pp\7        \8        P:                  4      ! V!  p\=        R4      ! V^4      pV# \?        S WVR7      w  ppppo pRp\A        S V.VR7      pS P                  4       ^ ,          p\B        R,          RV RV RV RV 2;;,          ^,          uu&   \D        PG                  RVVVVS P                  4       VP                  4       V4       \H        p/ pV'       d5   S P	                  4       P
                  R9  d   \K        R4      h\L        pRV/p. p. p/ p\O        4       '       d!   VPQ                  V4       VVVPR                  &   \U        VRR7      '       d   VPQ                  \V        4       VPY                  \         PZ                  P]                  VVVVR7      4       \_        V4      w  pp\a        S W4      pV'       dU   V'       dM   \c        VVVV4      '       d9   \e        V4      '       d(   ^RI3H4p  V Pk                  VVVPm                  4       4       \o        VS V4      '       d(   ^RI8H9p! V!Pu                  VVVPm                  4       4       \w        VVVV4      '       d'   \x        Pz                  ! VVVPm                  4       4       V'       d'   \}        VVVVS V4      '       d   ^RI?H@p" V"! VVV4       \ƒ        VVVPm                  4       V4      w  p#pV## u upi )zX
Lowering for autotuning aten.bmm with different backends (Aten, Triton, CUTLASS, etc.)
c                 ó0   <€ SP                  V ^R7      ^8H  # )é   )Úfallback)Úoptimization_hint)ÚdimÚsizevarss   &€r1   Údim_is_one_or_hintÚ%tuned_bmm.<locals>.dim_is_one_or_hintd   s    ø€ ð ×)Ñ)¨#¸Ð)Ó:¸aÑ?Ð?r3   c                 ó0   <€ SP                  W4      '       * # ©N)Ústatically_known_gt)rB   Ú	thresholdrC   s   &&€r1   Údim_is_not_known_gtÚ&tuned_bmm.<locals>.dim_is_not_known_gti   s   ø€ ð ×/Ñ/°Ó?Ô?Ð?r3   N)Úaxisc              3   óZ   "  € T F!  qP                  4       P                  R 8H  x € K#  	  R# 5i)ÚcpuN)Ú
get_deviceÚtype)Ú.0Úxs   & r1   Ú	<genexpr>Útuned_bmm.<locals>.<genexpr>~   s   é € Ð
>±¨A�<‰<‹>×Ñ %Ö'³ùs   ‚)+FTc                 ó¦   € \         P                  ! V 4      '       g   R # \         P                  ! V RR7      w  r\        V\         P                  4      # )TF)Úfreeze)r   Úis_storage_and_layoutÚas_storage_and_layoutÚ
isinstanceÚFlexibleLayout)ÚtÚ_r<   s   &  r1   Úis_valid_to_require_contiguousÚ1tuned_bmm.<locals>.is_valid_to_require_contiguous…   s=   € Ü×+Ò+¨A×.Ò.ÙÜ×0Ò0°¸5ÔA‰IˆAÜ˜f¤b×&7Ñ&7Ó8Ð8r3   c                 óþ   € VR,          ^8H  ;'       d0    V R,          ^8H  ;'       gW    VR,          V R,          8¬  ;'       g<    VR,          ^8H  ;'       d(    V R,          ^8H  ;'       g    VR,          V R,          8¬  # )é   éÿÿÿÿéþÿÿÿ© )ÚsizesÚstridess   &&r1   Ú is_preferred_layout_as_bmm_inputÚ3tuned_bmm.<locals>.is_preferred_layout_as_bmm_input‹   s{   € ð ˜•˜qÑ ×QÐQ e¨B¥i°1¡n÷UØ&P¸À½ÀuÈRÅyÑ8P÷Uð Uà˜"•+ Ñ"×SÐS¨¨b­	°Q©×(RÐ(R¸'À"½+ÈÈrÍÑ:RðUr3   c                 óæ   <€ VP                   R ,          P                  4       pVP                   R ,          P                  4       pS! W#4      '       g    \        P                  P                  V 4      p V # )Úval)r*   ÚsizeÚstrider   ÚExternKernelÚrequire_contiguous)r[   Úmeta_trd   re   rf   s   &&  €r1   Úmay_require_contiguousÚ)tuned_bmm.<locals>.may_require_contiguous•   sU   ø€ Ø—K‘K Õ&×+Ñ+Ó-ˆEØ—k‘k %Õ(×/Ñ/Ó1ˆGÙ3°E×CÒCÜ—O‘O×6Ñ6°qÓ9�ØˆHr3   )ÚargsÚkwargsÚ	broadcastÚtype_promotion_kindÚconvert_input_to_boolc                 óJ   <€ \         P                  ! V SP                  R R7      # )F)Úuse_compute_types)r   Úto_dtypeÚdtype)rR   Úmat1s   &€r1   Ú	_to_dtypeÚtuned_bmm.<locals>._to_dtype³   s   ø€ Ü—|’| A t§z¡zÀUÔKÐKr3   Údot)r<   Ú	out_dtyper4   )r~   Úaten_mm_infoz	aten.bmm_r\   zZTuned aten.bmm: batch=%s, m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, output_layout=%sz+out_dtype is only supported for CUDA or XPUr~   ©Úcheck_max_autotune)Úkwarg_overrides)ÚCUTLASS3xGemmTemplate)ÚCppBmmTemplate)Úadd_nv_universal_gemm_choices)ÚcudaÚxpura   )Br   ÚgraphrC   Ú	get_dtyperO   rP   Ú_BMM_DOT_DECOMPOSE_DTYPESÚget_sizeÚ_BMM_DOT_K_DECOMPOSE_THRESHOLDÚLÚ	unsqueezeÚsum_ÚmulÚallÚcurrent_noderq   r!   r   Úatenr   Úinductor_configÚtritonÚcodegen_upcast_to_fp32ry   ÚtorchÚfloat16Úbfloat16r   r   r}   r   r    r
   r   ÚlogÚinfoÚaten_bmmÚAssertionErrorÚaten_bmm_dtyper   ÚappendÚuidr   Úbmm_templateÚextendÚchoicesÚget_template_configsr   r   r   r   Úcodegen.cutlass.gemm_templaterƒ   Úadd_cutlass_gemm_choicesÚnodesr   Úcodegen.cpp_bmm_templater„   Úadd_choicesr   r   Úadd_ck_gemm_choicesr   Úcodegen.nv_universal_gemmr…   r   )&rz   Úmat2r~   r<   ry   Údevice_typerD   rJ   r]   ro   Ú	meta_mat1Ú	meta_mat2rq   rr   r{   rR   Úmul_pointwiseÚdot_reductionr(   r)   Úkr6   Úkernel_inputsÚ
batch_sizeÚaten_handlerÚaten_extra_kwargsr£   Útemplates_to_user‚   r\   Ú
is_nonzeroÚbatch_stride_largest_or_zerorƒ   r„   r…   Únoderf   rC   s&   f&&$                                @@r1   Ú	tuned_bmmr»   [   sT  ú€ ô
 �w‰w×Ñ€HØ�N‰NÓ€EØ—/‘/Ó#×(Ñ(€Kõ@õ
@ð 	ÓØ˜?Õ*ØŸ?™?Ó,×1Ñ1Ô1Ø—^‘^Ó%Ô%ØÔ.Ô.Ù˜tŸ}™}›¨qÕ1×2Ò2Ù˜tŸ}™}›¨qÕ1×2Ò2Ù §¡£°Õ 2Ô4R×SÒSô �{Š{˜4 Ó$ˆÜ�{Š{˜4 Ó#ˆÜ�vŠv”a—e’e˜D $Ó'¨aÔ0Ð0ç
ƒsÑ
>°$¸±Ó
>‡s‡s‚sÑ
>°$¸±Ó
>×>Ó>à�=‰=‹?˜1Õ Ô" d§m¡m£o°aÕ&8¸AÔ&=Ü—;’;˜t RÓ(ˆDÜ—;’;˜t QÓ'ˆDÜ—6’6œ!Ÿ%š%  dÓ+°!Ô4Ð4ò	9ò	Uõ	ñ *¨$×/Ò/ÜŸ™×,Ñ,×1Ñ1°!Õ4ˆIÙ)¨$°	Ó:ˆDÙ)¨$×/Ò/ÜŸ™×,Ñ,×1Ñ1°!Õ4ˆIÙ)¨$Ó:ˆDä˜˜t×$Ò$ÜœŸ™Ö(¨¨rÓ2ˆÜœŸ™Ö(¨¨qÓ1ˆÜ%Ø˜�ØØØ $Ø"'ô
‰ˆô ×!Ñ!×8×8Ð8¸T¿Z¹ZÜ�M‰MÜ�N‰NðL
ô >
õ
Lñ ;?Ó?¹$°Q”N 9Ô-¨aÖ0¹$ˆDÐ?ä&¤s§w¡wÔ/°Ñ6ˆÜ& uÔ-¨m¸QÓ?ˆàÐô #*Øˆd¨Yô#Ñ€A€qˆ!ˆV�T˜4ð €Dô # D¨$ <¸9ÔE€Mð —‘“ Õ#€JÜˆ^Õ˜y¨¨°A°a°S¸¸!¸¸A¸a¸SÐA×BÀaÕGÓBÜ‡H�HØdØØ	Ø	Ø	Ø�‰ÓØ�‰ÓØô	ô (0€LØÐßØ�?‰?Ó×!Ñ!¨Ô8Ü Ð!NÓOÐOÜ%ˆØ(¨)Ð4Ðà"$€Gð CEÐØ€Oä×ÒØ×Ñ Ô-Ø,=ˆ˜×(Ñ(Ñ)ä˜6°e×<Ó<Ø×Ñ¤Ô-ð ‡N�NÜ	�	‰	×&Ñ&ØØØØ+ð	 	'ó 	
ôô ' vÓ.�M€A€zÜ#BÀ4ÈÓ#VÐ ç$ßÜ  ¨¨A¨q×1Ò1Ü ×%Ò%åIà×6Ñ6Ø�V˜]×0Ñ0Ó2ô	
ô ˜F D¨$×/Ò/Ý=à×"Ñ"ØØØ×ÑÓ!ô	
ô ˜F A q¨!×,Ò,Ü×*Ò*¨7°F¸M×<OÑ<OÓ<QÔRçÔ4°V¸QÀÀ1ÀdÈD×QÒQÝMá% g¨v°}ÔEä'¨¨g°}×7JÑ7JÓ7LÈfÓU�G€Dˆ!Ø€Kùòy @s   ÎZÚalphaÚbetac               ó   € \        W4      '       d˜   V^ 8X  d   ^ pM!\        \        P                  ,          ! W@4      pV^ 8X  d   ^ pMA\        \        P                  ,          ! V\        \        P                  ,          ! W4      4      p\        \        P
                  ,          ! Wg4      # \        WWR7      w  r‰r¥rp \        WV.\        W4R7      R7      pVP                  4       ^ ,          p\        R,          RV RV RV	 RV
 2;;,          ^,          uu&   \        P                  RVVV	V
VP                  4       VP                  4       V P                  4       V4	       Rp. p. p\        4       '       d   VP                  \         4       \#        VR	R
7      '       d   VP                  \$        4       VP'                  \(        P*                  P-                  W¿V4      4       \/        WÞVP1                  4       V4      w  ppV# )zW
Lowering for autotuning aten.mm with different backends (Aten, Triton, CUTLASS, etc.)
)r<   )r¼   r½   )Úscalarsr   zaten.baddbmm_r\   zkTuned aten.baddbmm: batch_size=%s, m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, inp=%s, output_layout=%sÚbaddbmmFr€   )r!   r   r“   r�   r4   Úaddr    r
   Údictr‹   r   rš   r›   r‰   r   rŸ   Úaten_baddbmmr   r¡   r¢   r   r£   r¤   r   r§   )Úinprz   r¬   r¼   r½   r<   Úarg1Úarg2r(   r)   r²   r³   r´   r6   r£   r·   rº   r\   s   &&&$$$            r1   Útuned_baddbmmrÇ     sª  € ô
 ˜×$Ò$Ø�1Œ9Ø‰DäœTŸX™XÖ& tÓ1ˆDà�AŒ:Ø‰DäœTŸX™XÖ& u¬i¼¿¹Ö.AÀ$Ó.MÓNˆDäœŸ™Ö" 4Ó.Ð.ô (/¨t¸3Ô'NÑ$€Aˆ!�T ô #Ø	�DÐ¤4¨eÔ#?ô€Mð
 —‘“ Õ#€JÜˆ^Õ˜}¨Z¨L¸¸!¸¸A¸a¸SÀÀ!ÀÐE×FÈ!ÕKÓFÜ‡H�HØuØØ	Ø	Ø	Ø�‰ÓØ�‰ÓØ�‰‹Øô
ð €Dà"$€Gð CEÐÜ×ÒØ×Ñ¤Ô-ä˜6°e×<Ó<Ø×Ñ¤Ô-ð ‡N�NÜ	�	‰	×&Ñ& }ÈÓMôô (¨°}×7JÑ7JÓ7LÈfÓU�G€Dˆ!Ø€Kr3   rG   )GÚloggingÚtypingr   r—   Útorch._dynamo.utilsr   Ú7torch._inductor.codegen.rocm.ck_universal_gemm_templater   Ú torch._inductor.kernel.mm_commonr   Ú r   r”   r   r	   r�   r³   r
   r   r   r   r   Úruntime.runtime_utilsr   Úselect_algorithmr   r   r   r   Úutilsr   r   r   r   r   r   r   Úvirtualizedr   r   Ú	mm_commonr   r   r    r!   r"   r#   Ú	getLoggerÚ__name__rš   r“   r2   r¡   r4   Úoutrœ   r‡   Ú_is_compiledÚ	dtype_outrž   rÀ   rÃ   rŒ   r˜   r™   Úfloat32Úfloat64rŠ   Úregister_loweringr»   rÇ   rc   r3   r1   Ú<module>rÛ      s´  ðã Ý  ã Ý (Ý RÝ Aç ;Ñ ;Ý *ß PÓ PÝ 2÷ó ÷÷ ñ ÷ !÷ó ÷ Ý!Ý1à×Ò˜Ó!€Ø‡y�y‡~�~€ð ñ#ó ð#ñ Ø	Ø	Ù Ó-Ø'+ô	€ñ ˜eŸi™i¨ÀDÇHÁHÇLÁLÔQ€Ù#Ø	‡I�IØ %§	¡	× 6Ñ 6× 8Ò 8ÑÐ>WØ	Ø—‘×"Ñ"ô	€ñ "Ø	‡M�MÐ$°$·,±,×2BÑ2Bô€ð "$Ð à	‡M�MØ	‡N�NØ	‡M�MØ	‡M�Mð	Ð ð ×Ò�T—X‘XÓñv°Dô vó ðvðr ×Ò�T—\‘\Ó"ð9¨Að 9°Að 9¸dó 9ó #ò9r3   