+
    –°³jæ  ã                   óä  € ^ RI t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIH	t	H
t
 ^RIHt ^RIHt ^RIHtHtHt ^R	IHtHt ^R
IHt ] P0                  ! ]4      t]! ]P6                  RRR7      t]! ]P:                  P<                  P>                  RR]P@                  R7      t!]P:                  P<                  t]P:                  PD                  t"]P:                  PF                  t#R R lt$R R lt%R# )é    N)ÚAny)Úmm_args)Úconfig)ÚCppGemmTemplateÚCppWoqInt4GemmTemplate)Úcreate_epilogue_with_attr)ÚWeightInt4PackMatmul)Úautotune_select_algorithmÚExternKernelChoiceÚrealize_inputs)Úuse_aten_gemm_kernelsÚuse_cpp_gemm_template)ÚVzat::_weight_int8pack_mmF)Úhas_out_variantz*at::native::_weight_int4pack_mm_cpu_tensor)r   Úkernel_creatorc                ó   € V ^8„  d   QhRR/# ©é   ÚreturnN© )Úformats   "Úd/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/_inductor/quantized_lowerings.pyÚ__annotate__r   &   s   € ÷ Iñ I ñ Ió    c                  óH  € ^RI Hp  V P                  \        P                  \
        P                  \
        P                  .4       V P                  \        P                  4       V P                  \
        P                  4       V P                  \
        P                  4       R# )é   ©ÚloweringN)	Ú r   Úadd_needs_realized_inputsÚ	quantizedÚ
max_pool2dÚ
_quantizedÚ$wrapped_fbgemm_pack_gemm_matrix_fp16Ú!wrapped_fbgemm_linear_fp16_weightÚmake_fallbackr   s    r   Úregister_quantized_opsr'   &   sq   € Ýà×&Ñ&ä× Ñ Ü×;Ñ;Ü×8Ñ8ð	
ôð ×Ñœ9×/Ñ/Ô0Ø×Ñœ:×JÑJÔKØ×Ñœ:×GÑGÖHr   c                ó   € V ^8„  d   QhRR/# r   r   )r   s   "r   r   r   5   s   € ÷ D=ñ D=˜Tñ D=r   c                 ó:  a€ ^RI Hp  ^RIHoHp V! \        P
                  RR7      RR/R V3R lll4       pV! \        P                  RR7      RR/R R	 ll4       pV P                  \        P                  4       V P                  \        P                  4       R# )
z0Register weight-only quantized matmul lowerings.r   )ÚexpandÚregister_loweringN)Útype_promotion_kindÚlayoutc          
      ó�   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        /# )r   ÚinputÚweightÚscaler-   r   ©ÚtorchÚTensorr   )r   s   "r   r   Ú)register_woq_mm_ops.<locals>.__annotate__;   sF   € ÷ .ñ .Ü�|‰|ð.ä—‘ð.ô �|‰|ð.ô
 ð.ô 
ñ.r   c                óN  <aa€ \        WSR R7      w    porVVP                  4       \        P                  \        P                  \        P
                  39   d$   VP                  4       \        P                  8X  g.   \        RVP                  4        RVP                  4        24      hSp\        4       '       d   \        P                  WVS3V4      .M. pR VVV3R llp	\        WuVR R7      '       d   \        P                  ! VVWVS.R V	R7       \        RW…VS.V4      w  r¤V
# )	T)r-   Úmat2_transposedzKexpected mat1 dtype in [bfloat16, float16, float] and mat2 dtype int8, got ú and c                óD   € V ^8„  d   QhR\         P                  R\        /# )r   Úbufr   r2   )r   s   "r   r   Ú>register_woq_mm_ops.<locals>.int8pack_mm.<locals>.__annotate__X   s   € ÷ 	ñ 	œuŸ|™|ð 	´ñ 	r   c           
      óT   <€ \        V R \        S! SSP                  4      4      R7      # )Úmul)Úother)r   r   Úsize)r:   r*   r-   r1   s   &€€€r   Ú_mul_epilogueÚ?register_woq_mm_ops.<locals>.int8pack_mm.<locals>._mul_epilogueX   s'   ø€ Ü,Ø�U¤.±¸¸v¿{¹{Ó1KÓ"Lôð r   )r7   )Útrans_wÚepilogue_creatorÚ_weight_int8pack_mm)r   Ú	get_dtyper3   Úbfloat16Úfloat16ÚfloatÚint8ÚAssertionErrorr   Úaten__weight_int8pack_mmÚbindr   r   Úadd_choicesr
   )r/   r0   r1   r-   Ú_Úmat1Úmat2Úaten_layoutÚchoicesr@   Únoder*   s   &&fd       €r   Úint8pack_mmÚ(register_woq_mm_ops.<locals>.int8pack_mm:   s  ú€ ô '.Ø &¸$ô'
Ñ#ˆˆ1ˆa�˜ð �N‰NÓ¤§¡´·±ÄÇÁÐ LÔLØ—‘Ó ¤E§J¡JÔ.ä ðØ!Ÿ^™^Ó-Ð.¨e°D·N±NÓ4DÐ3EðGóð ð ˆô
 %×&Ò&ô &×*Ñ*¨D¸Ð+>ÀÓLÑMàð 	÷	ò 	ô
 ! °DÈ$×OÓOÜ×'Ò'ØØØ˜UÐ#ØØ!.õô ,Ø! 7°4¸Ð,?Àó
‰ˆð ˆr   c                óœ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  R\        R\        /# )r   r/   r0   Ú
qGroupSizeÚqScaleAndZerosr-   r   )r3   r4   Úintr   )r   s   "r   r   r5   l   sX   € ÷ Jñ JÜ�|‰|ðJä—‘ðJô ðJô Ÿ™ð	Jô ðJô 
ñJr   c          	      ó†  € \        WVR R R7      w    rTrgVP                  4       \        P                  \        P                  \        P
                  39   d$   VP                  4       \        P                  8X  g.   \        RVP                  4        RVP                  4        24      h\        P                  P                  \        P                  ! V\        P                  R7      RR7      pTp	\        4       '       d   \        P                  WgWƒ3V	4      .M. p
\         P"                  '       g   \         P$                  '       d\   \'        V	VVR R VR7      '       dE   VP)                  4       P+                  4       '       d!   \,        V,          P/                  V
V	WgWƒ.4       R R	 lp^V^R
 /p\1        RV
WgWƒ.V	VR7      w  rÕV# )T)r-   Úuse_4x2_dimr7   zLexpected mat1 dtype in [bfloat16, float16, float] and mat2 dtype uint8, got r8   )ÚdtypeN)Úname)r7   Úis_woq_int4Úq_group_sizec                ó€   € V ^8„  d   QhR\         P                  P                  P                  R\         P                  /# )r   Úxr   )r3   Ú	_inductorÚirÚIRNoder4   )r   s   "r   r   ÚBregister_woq_mm_ops.<locals>.int4pack_mm_cpu.<locals>.__annotate__£   s2   € ÷ 	Rñ 	R¤%§/¡/×"4Ñ"4×";Ñ";ð 	RÄÇÁñ 	Rr   c                 óò   € V P                  4       P                  4       '       g   \        R 4      hV P                  4       pV P	                  4       p\
        P                  ! ^ ^ÿV\
        P                  VR7      # )z&expected x to have a contiguous layout)r\   Údevice)Ú
get_layoutÚis_contiguousrJ   Úget_sizeÚ
get_devicer3   ÚrandintÚuint8)ra   Úshaperg   s   &  r   Úget_example_weightÚHregister_woq_mm_ops.<locals>.int4pack_mm_cpu.<locals>.get_example_weight£   sT   € Ø—<‘<“>×/Ñ/×1Ò1Ü$Ð%MÓNÐNØ—J‘J“LˆEØ—\‘\“^ˆFÜ—=’=  C¨´e·k±kÈ&ÔQÐQr   c                 ó`   € \         P                  P                  V P                  4       ,          # )N)r   ÚgraphÚ	constantsÚget_name)ra   s   &r   Ú<lambda>Ú>register_woq_mm_ops.<locals>.int4pack_mm_cpu.<locals>.<lambda>¬   s   € œŸ™×*Ñ*¨1¯:©:«<Ö8r   Ú_weight_int4pack_mm_for_cpu)Úinput_gen_fns)r   rE   r3   rF   rG   rH   rm   rJ   r   rr   Úadd_tensor_constantÚtensorÚint64r   Úaten__weight_int4pack_mm_cpurL   r   Úmax_autotuneÚmax_autotune_gemmr   rh   ri   r   rM   r
   )r/   r0   rW   rX   r-   rN   rO   rP   Ú
group_sizerQ   rR   ro   rx   rS   s   &&&&$         r   Úint4pack_mm_cpuÚ,register_woq_mm_ops.<locals>.int4pack_mm_cpuk   s«  € ô '.Ø &°dÈDô'
Ñ#ˆˆ1ˆa˜ð �N‰NÓ¤§¡´·±ÄÇÁÐ LÔLØ—‘Ó ¤E§K¡KÔ/ä ðØ"Ÿn™nÓ.Ð/¨u°T·^±^Ó5EÐ4FðHóð ô —W‘W×0Ñ0Ü�LŠL˜¬5¯;©;Ô7¸dð 1ó 
ˆ
ð ˆô %×&Ò&ô	 -×1Ñ1Ø Ð<¸kóñð ð 	ô × × Ð ¤F×$<×$<Ð$<Ü%ØØØØ $Ø Ø'÷ó ð —‘Ó!×/Ñ/×1Ò1ô # :Õ.×:Ñ:ØØØ˜ZÐ8ôõ	Rð Ð!ØÑ8ð
ˆô
 ,Ø)ØØ˜Ð4ØØ'ô
‰ˆð ˆr   )
r   r   r*   r+   ÚatenrD   rw   r&   Ú_dyn_quant_matmul_4bitÚ_dyn_quant_pack_4bit_weight)r   r+   rT   r€   r*   s       @r   Úregister_woq_mm_opsr…   5   s“   ø€ åß3á”t×/Ñ/ÀTÔJð.ð
 ÷.ñ .ó Kð.ñ` ”t×7Ñ7ÈTÔRðJð öJó SðJðX ×Ñœ4×6Ñ6Ô7Ø×Ñœ4×;Ñ;Ö<r   )&ÚloggingÚtypingr   r3   Ú torch._inductor.kernel.mm_commonr   r   r   Úcodegen.cpp_gemm_templater   r   Úcodegen.cpp_utilsr   Ú	mkldnn_irr	   Úselect_algorithmr
   r   r   Úutilsr   r   Úvirtualizedr   Ú	getLoggerÚ__name__ÚlogrD   rK   Úopsr!   Úint4mm_packed_weight_cpuÚcreater|   r#   r‚   r'   r…   r   r   r   Ú<module>r•      sÃ   ðÛ Ý ã Ý 4å ß NÝ 8Ý +÷ñ ÷
 @Ý ð ×Ò˜Ó!€á-Ø	×ÑÐ8È%ôÐ ñ  2Ø	‡I�I×Ñ×0Ñ0Ø0ØØ'×.Ñ.ô	 Ð ð �I‰I×Ñ€	Ø�Y‰Y×!Ñ!€
Ø‡y�y‡~�~€õI÷D=r   