+
    ÒÜ³jg  ã                   óÚ   € ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RIH	t	H
t
Ht ^ RIHt ^ RIHt ^ RIHtHt ^ RIHtHt ^ RIHtHtHtHt ^ R	IHtHtHtHtH t   ! R
 R4      t!R# )é    N)Útqdm)ÚDictÚListÚOptional)Údefaultdict)Úget_calib_dataset)Úapply_scaleÚ
apply_clip)Úclear_memoryÚget_best_device)ÚWQLinear_GEMMÚWQLinear_GEMVÚWQLinear_MarlinÚWQLinear_GEMVFast)Úappend_str_prefixÚget_op_nameÚget_named_linearsÚset_op_by_nameÚexclude_layers_to_not_quantizec                   óÌ  a € ] tR t^t o RV 3R lR lltV 3R lR ltRV 3R lR lltR tR	 tV 3R
 lR lt	]
P                  ! 4       V 3R lR l4       t]
P                  ! 4       R/ 3V 3R lR ll4       t/ 3V 3R lR llt]
P                  ! 4       V 3R lR l4       t]
P                  ! 4       R 4       t]
P                  ! 4       RV 3R lR ll4       tRR ltR tR tRtV tR# )ÚAwqQuantizerNc                ó   <€ V ^8„  d   QhRR/# )é   ÚreturnN© )ÚformatÚ__classdict__s   "€ÚW/home/luca/screen-checking/.venv/lib/python3.14/site-packages/awq/quantize/quantizer.pyÚ__annotate__ÚAwqQuantizer.__annotate__   s   ø€ ÷ +
ñ +
ð( 
ñ)+
ó    c                óf  € Wn         W n        W0n        W@n        WPn        W`n        Wpn        W€n        W�n        W n	        W°n
        WÐn        Wàn        Wðn        VV n        VV n        VV n        Ve   TM. V n        V P%                  V P                  V P                  R7      w  V n        V n        V n        R # )N)Ú	n_samplesÚmax_seq_len)Ú	awq_modelÚmodelÚ	tokenizerÚw_bitÚ
group_sizeÚ
zero_pointÚversionÚ
calib_dataÚsplitÚtext_columnÚduo_scalingÚexport_compatibler
   Ún_parallel_calib_samplesÚmax_calib_samplesÚmax_calib_seq_lenÚmax_chunk_memoryÚmodules_to_not_convertÚ
init_quantÚmodulesÚmodule_kwargsÚinps)Úselfr%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r5   r0   r
   r1   r2   r3   r4   s   &&&&&&&&&&&&&&&&&&&r   Ú__init__ÚAwqQuantizer.__init__   s®   € ð* #ŒØŒ
Ø"ŒØŒ
Ø$ŒØ$ŒØŒØ$ŒØŒ
Ø&ÔØ&ÔØ!2ÔØ$ŒØ(@Ô%Ø!2ˆÔØ!2ˆÔØ 0ˆÔà&<Ò&HÑ"Èbð 	Ô#ð 7;·o±oØ×,Ñ,¸$×:PÑ:Pð 7Fó 7
Ñ3ˆŒ�dÔ(¨$®)r!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   Úw©ÚtorchÚTensor)r   r   s   "€r   r   r    J   s   ø€ ÷ # ñ # ©¯©ñ # r!   c                óÜ  € VP                   pV P                  ^ 8”  d[   VR	,          V P                  ,          ^ 8X  g    Q RVR	,           RV P                   R24       hVP                  R	V P                  4      pVP                  4       ^8X  g   Q h\        P
                  ! V4      P                  4       ^ 8X  g   Q hV P                  '       dî   VP                  ^RR7      pVP                  ^RR7      p^V P                  ,          ^,
          p^ pW4,
          P                  RR7      V,          p\        P                  ! WG,          4      ) P                  We4      p\        P                  ! \        P                  ! W,          4      V,           We4      V,
          V,          pVP                  V^ ,          R	4      pM±VP                  4       P                  ^RR7      pVP                  RR7      p^V P                  ^,
          ,          ^,
          p^V P                  ^,
          ,          ) pW5,          pRp\        P                  ! \        P                  ! W,          4      We4      V,          p\        P
                  ! V4      P                  4       ^ 8X  g   Q h\        P
                  ! V4      P                  4       ^ 8X  g   Q hVP                  V^ ,          R	4      pVP                  V4      pWV3# )
r   zorg_w_shape (z$) must be a multiple of group_size (z)!T©ÚdimÚkeepdimgñhãˆµøä>©ÚminNéÿÿÿÿ)Úshaper)   ÚreshaperD   r@   ÚisnanÚsumr*   ÚamaxÚaminr(   ÚclampÚroundÚclamp_ÚviewÚabs)	r:   r>   Úorg_w_shapeÚmax_valÚmin_valÚmax_intÚmin_intÚscalesÚzeross	   &&       r   Úpseudo_quantize_tensorÚ#AwqQuantizer.pseudo_quantize_tensorJ   si  € Ø—g‘gˆØ�?‰?˜QÔØ˜r•? T§_¡_Õ4¸Ô9ð  T¸]È;ÐWYÍ?ÐJ[Ð[ð  AE÷  APñ  APð  @Qð  QSð  <Tó  TÐ9Ø—	‘	˜"˜dŸo™oÓ.ˆAØ�u‰u‹w˜!Œ|Ðˆ|Ü�{Š{˜1‹~×!Ñ!Ó# qÔ(Ð(Ð(ð �?�?ˆ?Ø—f‘f ¨D�fÓ1ˆGØ—f‘f ¨D�fÓ1ˆGØ˜Ÿ™•m aÕ'ˆGØˆGØÕ'×.Ñ.°4Ð.Ó8¸7ÕBˆFÜ—k’k 'Õ"2Ó3Ð3×;Ñ;¸GÓMˆEä—’œEŸKšK¨­
Ó3°eÕ;¸WÓNÐQVÕVØõˆAð —J‘J˜{¨1�~¨rÓ2‰Eà—e‘e“g—l‘l q°$�lÓ7ˆGØ—m‘m¨�mÓ-ˆGØ˜DŸJ™J¨�NÕ+¨aÕ/ˆGØ˜dŸj™j¨1�nÕ-Ð.ˆGØÕ&ˆFØˆEÜ—’œEŸKšK¨­
Ó3°WÓFÈÕOˆAä�{Š{˜6Ó"×&Ñ&Ó(¨AÔ-Ð-Ð-Ü�{Š{˜1‹~×!Ñ!Ó# qÔ(Ð(Ð(à—‘˜[¨�^¨RÓ0ˆØ�I‰I�kÓ"ˆà˜%ÐÐr!   c                óx   <€ V ^8„  d   QhRS[ P                  RS[P                  RS[S[P                  ,          /# )r   r>   rY   rZ   )ÚnnÚLinearr@   rA   r   )r   r   s   "€r   r   r    o   s6   ø€ ÷ ñ Ù—‘ðÙ$)§L¡LðÙ9AÁ%Ç,Á,Õ9Oñr!   c                ó*  € VP                   P                  P                  R,          VP                  R,          ,          pVP                  ^V4      P	                  VP                   P                  P                  4      pV P
                  '       df   VP                  ^V4      P	                  VP                   P                  P                  4      pVP                   P                  V,
          V,          pV# VP                   P                  V,          pV# )é   rH   )ÚweightÚdatarI   ÚrepeatrJ   r*   )r:   r>   rY   rZ   Úrepeat_counts   &&&& r   Úpseudo_dequantize_tensorÚ%AwqQuantizer.pseudo_dequantize_tensoro   s¾   € ð —x‘x—}‘}×*Ñ*¨2Õ.°&·,±,¸rÕ2BÕBˆØ—‘˜q ,Ó/×7Ñ7¸¿¹¿¹×8KÑ8KÓLˆð �?�?ˆ?Ø—L‘L  LÓ1×9Ñ9¸!¿(¹(¿-¹-×:MÑ:MÓNˆEØ—‘—‘ Õ&¨&Õ0ˆAð ˆð —‘—‘ Õ&ˆAàˆr!   c           	     ó”
  a€ \        \        \        V P                  4      4      R R7       EF  p\	        V P                  V,          P                  4       4      P                  oSe   \        S4      R8X  dÈ   \        P                  P                  4       '       d7   R\        V\        P                  P                  4       ,          4      ,           pM
\        4       pV P                  V,          P                  V4      V P                  V&   \	        V P                  V,          P                  4       4      P                  oV P                  P                  R4      e0   V P                  R,          P                  S4      V P                  R&   V P                  P                  R4      e0   V P                  R,          P                  S4      V P                  R&   V P                   P                  S4      V n        V P"                  P%                  V P&                  S4       \(        P*                  R8¼  dl   V P                  P                  R4      fO   V P&                  P&                  P-                  V P                   V P                  R,          4      V P                  R&   \(        P*                  R8¼  d-   V P                  P                  R4      f   RV P                  R&   V P                  P/                  4        F\  w  r4\1        V\2        4      '       g   K  \2        ;QJ d    . V3R	 lV 4       F  NK  	  5M! V3R	 lV 4       4      V P                  V&   K^  	  \5        V P                  V,          4      p\7        WPP8                  4      pV P;                  V P                  V,          V4      p\=        4        V P"                  P?                  V P                  V,          W`P                  4      pV Uu. uF'  pV P@                  ! V P                  V,          3/ VB NK)  	  p	p\C        V P                  V,          W–R
7       \E        V	\G        V P&                  V P                  V,          4      R,           4      p	V PH                  '       dy   V PK                  V P                  V,          WV4      p
\I        V P                  V,          V
4       \E        V
\G        V P&                  V P                  V,          4      R,           4      p
V PL                  '       g$   V PO                  V P                  V,          V4       \=        4        EK  	  R# u upi )ÚAWQ©ÚdescNÚcpuzcuda:Úposition_idsÚattention_maskz4.48.0Úposition_embeddingsc              3   óª   <"  € T FH  p\        V\        P                  \        P                  34      '       d   VP                  S4      MTx € KJ  	  R # 5i©N)Ú
isinstancer@   rA   r^   ÚModuleÚto)Ú.0ÚitemÚcommon_devices   & €r   Ú	<genexpr>Ú(AwqQuantizer.quantize.<locals>.<genexpr>­   sE   øé € ð 2á./ dô 3=¸TÄEÇLÁLÔRT×R[ÑR[ÐC\×2]Ò2]˜Ÿ™ Ô.Ø!ô"Û./ùs   ƒAA)Úinput_feat_dictÚ.)(r   ÚrangeÚlenr7   ÚnextÚ
parametersÚdeviceÚstrr@   ÚcudaÚis_availableÚdevice_countr   rt   r8   Úgetr9   r%   Ú
move_embedr&   ÚtransformersÚ__version__Ú
rotary_embÚitemsrr   Útupler   r   r5   Ú_get_input_featr   Úget_layers_for_scalingÚ_search_best_scaler	   r   r   r
   Ú_search_best_clipr0   Ú_apply_quant)r:   ÚiÚbest_deviceÚkÚvÚnamed_linearsÚ
input_featÚmodule_configÚlayerÚscales_listÚ	clip_listrw   s   &          @r   ÚquantizeÚAwqQuantizer.quantize   s  ø€ Ü”eœC §¡Ó-Ó.°U×;Ð;ˆAä  §¡¨a¥×!;Ñ!;Ó!=Ó>×EÑEˆMØÒ$¬¨MÓ(:¸eÔ(CÜ—:‘:×*Ñ*×,Ò,Ø")¬C°´E·J±J×4KÑ4KÓ4MÕ0MÓ,NÕ"N‘Kä"1Ó"3�Kà"&§,¡,¨q¥/×"4Ñ"4°[Ó"A�—‘˜Q‘Ü $ T§\¡\°!¥_×%?Ñ%?Ó%AÓ B× IÑ I�à×!Ñ!×%Ñ% nÓ5ÒAØ59×5GÑ5GØ"õ6ç‘"�]Ó#ð ×"Ñ" >Ñ2ð ×!Ñ!×%Ñ%Ð&6Ó7ÒCØ7;×7IÑ7IØ$õ8ç‘"�]Ó#ð ×"Ñ"Ð#3Ñ4ð Ÿ	™	Ÿ™ ]Ó3ˆDŒIð
 �N‰N×%Ñ% d§j¡j°-Ô@ô ×(Ñ(¨HÔ4Ø×&Ñ&×*Ñ*Ð+@ÓAÒIà<@¿J¹J×<LÑ<L×<WÑ<WØ—I‘I˜t×1Ñ1°.ÕAó=�×"Ñ"Ð#8Ñ9ô ×(Ñ(¨HÔ4Ø×&Ñ&×*Ñ*Ð+;Ó<ÒDØ7;�×"Ñ"Ð#3Ñ4à×*Ñ*×0Ñ0Ö2‘�ä˜a¤×'Ô'ß,1¬Eô 2á./ó2¯E©Eô 2á./ó2ó -�D×&Ñ& qÓ)ñ 3ô .¨d¯l©l¸1­oÓ>ˆMô ;Ø×:Ñ:óˆMð ×-Ñ-¨d¯l©l¸1­o¸}ÓMˆJÜŒNð )-¯©×(MÑ(MØ—‘˜Q• ×-?Ñ-?ó)ˆMñ
 +óá*�Eð ×'Ò'¨¯©°Q­ÑA¸5ÔAÙ*ð ð ô ˜Ÿ™ Q�¨ÕQÜ+Øœ[¨¯©°T·\±\À!µ_ÓEÈÕKóˆKð
 ��ˆØ ×2Ñ2Ø—L‘L •O ]ó�	ô ˜4Ÿ<™<¨�?¨IÔ6Ü-Øœ{¨4¯:©:°t·|±|ÀAµÓGÈ#ÕMó�	ð
 ×)×)Ð)Ø×!Ñ! $§,¡,¨q¥/°=ÔAä�Nóq <ùòBs   Ï0-Uc                ó"  € \        \        \        V P                  4      4      R R7       Fa  p\	        V P                  V,          4      p\        W P                  4      pV P                  V P                  V,          V4       \        4        Kc  	  R# )ÚPackingrj   N)	r   r|   r}   r7   r   r   r5   r�   r   )r:   r‘   r•   s   &  r   ÚpackÚAwqQuantizer.packÚ   sf   € Ü”eœC §¡Ó-Ó.°Y×?ˆAÜ-¨d¯l©l¸1­oÓ>ˆMÜ:Ø×:Ñ:óˆMð ×Ñ˜dŸl™l¨1�o¨}Ô=ÜŽNó @r!   c                óJ   <€ V ^8„  d   QhRS[ S[S[P                  3,          /# )r   r•   )r   r�   r^   r_   )r   r   s   "€r   r   r    ã   s!   ø€ ÷ 'ñ '±$±s¹B¿I¹I°~Õ2Fñ 'r!   c           
     ón  € VP                  4        EFŸ  w  r4VP                  \        4       4      P                  4       pV P	                  VP
                  P                  4      w  VP
                  n        rVV P                  R 8X  dH   VP                  4       P                  4       pVe   VP                  4       P                  4       p\        pM`V P                  R8X  d   \        pMHV P                  R8X  d   \        pM0V P                  R8X  d   \        pM\        RV P                   24      hVP                  VV P                   V P"                  RVVR7      pVP%                  4        VP                  \'        VP)                  4       4      P*                  4       \-        WV4       \/        4        EK¢  	  R# )ÚgemmNÚgemvÚmarlinÚ	gemv_fastzUnknown version F)Úlinearr(   r)   Ú	init_onlyrY   rZ   )rŠ   rt   r   Úhalfr[   rb   rc   r+   ÚtÚ
contiguousr   r   r   r   Ú
ValueErrorÚfrom_linearr(   r)   rl   r~   r   r€   r   r   )	r:   Úmoduler•   ÚnameÚlinear_layerrY   rZ   Úq_linear_moduleÚq_linears	   &&&      r   r�   ÚAwqQuantizer._apply_quantã   sY  € Ø"/×"5Ñ"5×"7ÑˆDà'Ÿ?™?¬?Ó+<Ó=×BÑBÓDˆLà6:×6QÑ6QØ×#Ñ#×(Ñ(ó7Ñ3ˆL×ÑÔ$ fð �|‰|˜vÔ%ØŸ™›×.Ñ.Ó0�ØÒ$Ø!ŸG™G›I×0Ñ0Ó2�EÜ"/‘à—‘ Ô'Ü"/‘à—‘ Ô)Ü"1‘à—‘ Ô,Ü"3‘ô !Ð#3°D·L±L°>Ð!BÓCÐCà&×2Ñ2Ø#Ø—j‘jØŸ?™?ØØØð 3ó ˆHð ×ÑÔØ�K‰Kœ˜V×.Ñ.Ó0Ó1×8Ñ8Ô9Ü˜6¨Ô2Ü�NóM #8r!   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  P                  RS[RS[ P                  /# )r   Úxr®   r8   r   )r@   rA   r^   rs   r   )r   r   s   "€r   r   r      s=   ø€ ÷ ñ Ù—‘ðÙ',§x¡x§¡ðÙGKðá	�‰ñr!   c                óŠ  € V P                   f+   V! V3/ VB p\        V\        4      '       d
   V^ ,          pV# . p\        P                  ! WP                   4      pV FJ  pV! V3/ VB p\        V\        4      '       d
   V^ ,          pVP                  VP                  4       4       KL  	  \        P                  ! V^ R7      pV# )N©rD   )r1   rr   r‹   r@   r-   Úappendrl   Úcat)r:   rµ   r®   r8   Úmodule_outputÚpartitioned_inputsÚ	x_partialÚpartial_outputs   &&&&    r   Ú_module_forwardÚAwqQuantizer._module_forward  s¹   € ð ×(Ñ(Ò0á" 1Ñ6¨Ñ6ˆMÜ˜-¬×/Ò/Ø -¨aÕ 0�ð  Ðð ˆMÜ!&§¢¨Q×0MÑ0MÓ!NÐÛ/�	Ù!'¨	Ñ!C°]Ñ!C�ä˜n¬e×4Ò4Ø%3°AÕ%6�Nà×$Ñ$ ^×%7Ñ%7Ó%9Ö:ñ 0ô "ŸIšI m¸Ô;ˆMàÐr!   c                ó^   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  /# )r   ÚlayersÚinp)r   r^   r_   r@   rA   )r   r   s   "€r   r   r    '  s1   ø€ ÷ G
ñ G
ñ ‘R—Y‘Y•ð	G
ñ
 �\‰\ñG
r!   c           
     ót  € Vf   \        V4      ^8X  g   Q hV^ ,          pRV9   d   VP                  R4       VP                  \        VP	                  4       4      P
                  4      p\        P                  ! V Uu. uF  qwP                  NK  	  up^ R7      pVP                  p	VP                  RV P                  4      pVP                  4       VP                  4       P                  ^RR7      R,           ,          p
V
P                  V	4      p
V
P                  ^ 4      p\        V4       VP!                  4       P                  4       P                  RVP                  R,          4      pVP#                  ^ 4      pVP#                  ^4      pVP%                  4       ^,          p\'        V P(                  Wþ,          ,          4      p\+        VV4      p\        P,                  ! V\        P.                  VP
                  R7      p\1        ^ VV4       Fj  p\+        VV,           V4      pVVV P                  \        P.                  4      P3                  ^ R7      pVVP                  VP
                  4      ,          pKl  	  VV,          P                  VP4                  4      p\        V4       \        P6                  ! 4       ;_uu_ 4        V P9                  We4      pV P;                  WEV4      pVP=                  \        P>                  ! VP4                  4      P*                  \        P>                  ! VP4                  4      P@                  4      pR R R 4       V PC                  WKVWSXX4      p\E        W4      \G        V Uu. uF  p\E        VV4      NK  	  up4      V3# u upi   + '       g   i     L[; iu upi )NÚ	use_cacher·   TrC   g�íµ ÷Æ°>)Údtyper€   rH   )$r}   Úpoprt   r~   r   r€   r@   r¹   rb   rI   rR   r)   rS   rM   Úmeanr   rl   ÚsizeÚelement_sizeÚintr4   rG   rZ   Úfloat32r|   rL   rÅ   Úno_gradÚ_sanitize_kwargsr¾   ÚclipÚfinfoÚmaxÚ_compute_best_scaler   r‹   )r:   r®   Úprev_oprÁ   rÂ   Úmodule2inspectÚkwargsÚ_mrb   Ú	org_shapeÚw_scaleÚw_meanÚinp_flatÚnum_elementsÚnum_channelsÚelement_size_bytesÚ
chunk_sizeÚx_sumr‘   ÚendÚ	chunk_sumÚx_meanr8   Úfp16_outputÚbest_scalesÚms   &&&&&&&                   r   rŽ   ÚAwqQuantizer._search_best_scale&  sã  € ð Ò!Ü�v“; !Ô#Ð#Ð#Ø# A�YˆNà˜&Ô Ø�J‰J�{Ô#ð �f‰f”T˜.×3Ñ3Ó5Ó6×=Ñ=Ó>ˆô —’±Ó7±¨"ŸIœI±Ñ7¸QÔ?ˆØ—L‘Lˆ	à—‘˜R §¡Ó1ˆð —*‘*“, &§*¡*£,×"3Ñ"3¸À4Ð"3Ó"HÈ4Õ"OÕPˆà—,‘,˜yÓ)ˆà—‘˜a“ˆÜ�VÔð —7‘7“9—=‘=“?×'Ñ'¨¨C¯I©I°b­MÓ:ˆØ—}‘} QÓ'ˆØ—}‘} QÓ'ˆØ%×2Ñ2Ó4°qÕ8Ðô ˜×.Ñ.Ð3EÕ3TÕUÓVˆ
Ü˜ \Ó2ˆ
ô —’˜L´·±ÀcÇjÁjÔQˆä�q˜,¨
Ö3ˆAÜ�a˜*•n lÓ3ˆCØ   3˜×*Ñ*¬5¯=©=Ó9×=Ñ=À!Ð=ÓDˆIØ�Y—\‘\ #§*¡*Ó-Õ-ŠEñ 4ð
 ˜,Õ&×*Ñ*¨3¯9©9Ó5ˆÜ�UÔô �]Š]�_�_Ø ×1Ñ1°&ÓIˆMØ×.Ñ.¨sÀMÓRˆKØ%×*Ñ*¬5¯;ª;°{×7HÑ7HÓ+I×+MÑ+MÌuÏ{Ê{Ð[f×[lÑ[lÓOm×OqÑOqÓrˆK÷ ð ×.Ñ.Ø˜ ¸Àmó
ˆô
 ˜Ó(Ü±6Ó:±6¨a”;˜v qÖ)±6Ñ:Ó;Øð
ð 	
ùò] 8÷H �_üò ;s   Á>NË	BN"Í?N5
Î"N2	c                óà   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  P                  RS[S[P
                  ,          RS[ P                  RS[/# )r   rµ   rØ   rá   rÓ   Úlinears2scalerâ   rÔ   )r@   rA   r^   rs   r   r_   r   )r   r   s   "€r   r   r    p  sz   ø€ ÷ J*ñ J*á�<‰<ðJ*ñ —‘ðJ*ñ —‘ð	J*ñ
 Ÿ™Ÿ™ðJ*ñ ™BŸI™I•ðJ*ñ —\‘\ðJ*ñ ñJ*r!   c                óä  € ^p. p	Rp
Rp\        R4      pVP                  4       P                  4        UUu/ uF  w  rÞWÞP                  4       bK  	  pppVP                  pVP                  R4      P                  V4      pVP                  R4      P                  V4      p\        V4       EFL  pVV,          pV P                  '       dG   VP                  V4      VP                  ^V,
          4      R,           ,          P                  RR7      pM0VP                  V4      P                  RR7      P                  R4      pVVP                  4       VP                  4       ,          P                  4       ,          pVP                  ^R4      P                  V4      p^V\        P                  ! V4      &   ^V\        P                   ! V4      &   V F`  pVP"                  P%                  V4       V P'                  VP"                  P(                  4      ^ ,          V,          VP"                  n        Kb  	  V P+                  WV4      pVP-                  \        P.                  ! VP0                  4      P                  \        P.                  ! VP0                  4      P                  4      pV P3                  VVV4      pV	P5                  V4       VV8  d   TpTp
VP7                  4       pVP9                  V4       EKO  	  V
R8X  d   \:        P<                  ! V	4       \>        h\        P                   ! V4      PA                  4       ^ 8X  g   Q V4       hVPC                  4       P                  4       # u uppi )a  
Compute loss and select best scales

L(s) = || Q(W * s) (s^-1 * X) - W * X ||
Q: weight quantization function | pseudo_quantize_tensor(W * s)
X: inputs from calib dataset    | X
W: original weights in FP16     | layer
s: per channel scaling factor   | s^-1 * X
NÚinfg-Cëâ6?rF   rH   )"ÚfloatÚ
state_dictrŠ   rl   r€   rR   rt   r|   r/   ÚpowrO   rÐ   rG   Úsqrtr@   ÚisinfrK   rb   Úmul_r[   rc   r¾   rÎ   rÏ   rÅ   Ú_compute_lossr¸   ÚcloneÚload_state_dictÚloggingÚdebugÚ	ExceptionrL   Údetach)r:   rµ   rØ   rá   rÓ   rç   râ   rÔ   Ún_gridÚhistoryÚ
best_ratiorã   Ú
best_errorr“   r”   Úorg_sdr€   ÚratiorY   Úscales_viewÚfcÚint_w_outputÚlosss   &&&&&&&&               r   rÑ   Ú AwqQuantizer._compute_best_scalep  s¼  € ð& ˆØˆØˆ
ØˆÜ˜5“\ˆ
à)7×)BÑ)BÓ)D×)JÑ)JÔ)LÔMÑ)L¡ �!—U‘U“W’*Ñ)LˆÑMà—‘ˆØ—‘˜R“×#Ñ# FÓ+ˆØ—‘˜R“×#Ñ# FÓ+ˆä˜6—]ˆEà˜F•NˆEð ××ÐØ Ÿ*™* UÓ+¨v¯z©z¸!¸e½)Ó/DÀtÕ/KÕL×SÑSÐX\ÐSÓ]‘àŸ™ EÓ*×0Ñ0°TÐ0Ó:×?Ñ?ÀÓC�Ø˜vŸz™z›|¨f¯j©j«lÕ:×@Ñ@ÓBÕBˆFØ Ÿ+™+ a¨Ó,×/Ñ/°Ó7ˆKð +,ˆF”5—;’;˜vÓ&Ñ'Ø*+ˆF”5—;’;˜vÓ&Ñ'ó $�Ø—	‘	—‘˜{Ô+à×/Ñ/°·	±	·±Ó?ÀÕBÀ[ÕPð —	‘	–ñ $ð  ×/Ñ/°À6ÓJˆLØ'×,Ñ,¬U¯[ª[¸×9KÑ9KÓ-L×-PÑ-PÔRW×R]ÒR]Ð^j×^pÑ^pÓRq×RuÑRuÓvˆLð ×%Ñ% k°<ÀÓHˆDà�N‰N˜4Ô Ø�jÔ Ø!�
Ø"�
Ø$Ÿl™l›n�Ø×*Ñ*¨6×2ñG #ðJ ˜ÔÜ�MŠM˜'Ô"ÜˆOä�{Š{˜;Ó'×+Ñ+Ó-°Ô2Ð?°KÓ?Ð2à×!Ñ!Ó#×'Ñ'Ó)Ð)ùóc Ns   µM,c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   râ   rÿ   r€   )r@   rA   r€   )r   r   s   "€r   r   r    ½  s5   ø€ ÷ ñ á—\‘\ðñ —l‘lðñ —‘ñ	r!   c                ó:  € R pVP                  R4      pVP                  R4      pVP                  ^ 4      pVP                  4       pV P                  V^,          ,          p	\	        W—4      p	\
        P                  ! WY4      p
\
        P                  ! Wi4      p\        W«4       Fm  w  rÍVP                  V4      VP                  V4      ,
          P                  4       P                  ^4      P                  4       P                  4       pWN,          pKo  	  WG,          pV# )g        rH   )rR   rÈ   rÉ   r4   rG   r@   r-   Úziprt   rê   rì   rL   rv   )r:   râ   rÿ   r€   r   Úfp16_output_flatÚint_w_output_flatrÚ   rÜ   rÝ   Úfp16_chunksÚint_w_chunksÚ
fp16_chunkÚint_w_chunkÚ
chunk_losss   &&&&           r   rð   ÚAwqQuantizer._compute_loss¼  sû   € ð ˆØ&×+Ñ+¨BÓ/ÐØ(×-Ñ-¨bÓ1ÐØ'×,Ñ,¨QÓ/ˆØ(×5Ñ5Ó7Ðð ×*Ñ*Ð/AÀAÕ/EÕFˆ
Ü˜Ó2ˆ
ô —k’kÐ"2Ó?ˆÜ—{’{Ð#4ÓAˆô (+¨;Ö'EÑ#ˆJØ$Ÿ-™-¨Ó/°+·.±.ÀÓ2HÕH×OÑOÓQ×UÑUÐVWÓX×\Ñ\Ó^×cÑcÓeˆJØÕŠDñ (Fð
 	Õˆàˆr!   c                óV  € . p. ROpV F—  p\        V Uu. uF  qwV9   NK	  	  up4      '       d   K(  W&,          P                  \        4       4       V P                  W&,          P                  W6,          4      pVP                  Wh34       W&,          P                  4        K™  	  V# u upi )Úq_)r  Úk_ÚqueryÚkeyÚWqkv)Úanyrt   r   Ú_compute_best_cliprb   r¸   rl   )	r:   r˜   r•   r–   rš   Úavoid_clippingr¯   Ú_rU   s	   &&&&     r   r�   ÚAwqQuantizer._search_best_clipÜ  s˜   € àˆ	Ú=ˆã!ˆDä¡~Ó6¡~ !˜”I¡~Ñ6×7Ò7ÙàÕ×"Ñ"¤?Ó#4Ô5Ø×-Ñ-ØÕ#×*Ñ*¨JÕ,<óˆGð ×Ñ˜d˜_Ô-ØÕ×#Ñ#Ö%ñ "ð Ðùò 7s   •B&
c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r>   r–   r?   )r   r   s   "€r   r   r    ð  s'   ø€ ÷ :'ñ :'á�<‰<ð:'ñ —L‘Lñ:'r!   c                ó  € VP                  4       ^8X  g   Q hVP                  pV P                  ^ 8”  d   V P                  MV^,          pVP                  RVP                  R,          4      pVP	                  ^VP                  ^ ,          RV4      p\        ^VP                  ^,          V,          4      pVRRRV13,          pVP	                  V^ ,          ^RV4      pV^ ,          R,          ^ 8X  d   RM^@p	V^ ,          V	,          ^ 8X  g   Q hTp
. p\        V^ ,          V	,          4       EF“  pW¬V	,          V^,           V	,           pVP                  4       P                  RRR7      pVP                  4       p\        P                  ! V4      R,          pVP                  VP                  4      pW!,          P                  RR7      p\        \        WC,          4      4       FÆ  pV^VV,          ,
          ,          pV) p\        P                   ! VVV4      pV P#                  V4      ^ ,          pVV,          P                  RR7      pVV,
          P%                  ^4      P'                  ^R7      P                  VP                  4      p??VV8  pVV,          VV&   VV,          VV&   KÈ  	  VP)                  V4       EK–  	  \        P*                  ! V^ R7      p\-        V4       \-        X4       VP/                  ^4      # )	r   :NNNNé   TrC   g    eÍÍAr·   rH   )rD   rI   r)   rR   rJ   rÐ   r|   rS   rM   rñ   r@   Ú	ones_likert   r€   rL   rÊ   rO   r[   rì   rÇ   r¸   r¹   r   Úsqueeze)r:   r>   r–   r÷   Ú
max_shrinkÚn_sample_tokenrT   r)   Ú	step_sizeÚoc_batch_sizeÚw_allÚbest_max_val_allÚi_bÚorg_max_valÚbest_max_valÚmin_errsÚorg_outÚi_srU   rV   Úcur_wÚq_wÚcur_outÚerrÚcur_best_idxs   &&&&&&                   r   r  ÚAwqQuantizer._compute_best_clipï  s�  € ð �u‰u‹w˜!Œ|Ðˆ|Ø—g‘gˆð )-¯©¸!Ô(;�T—_’_ÀÈQÅˆ
Ø—_‘_ R¨×)9Ñ)9¸"Õ)=Ó>ˆ
Ø×'Ñ'¨¨:×+;Ñ+;¸AÕ+>ÀÀJÓOˆ
ô ˜˜:×+Ñ+¨AÕ.°.Õ@ÓAˆ	Ø ¡; Y ; Õ/ˆ
à�I‰I�k !•n a¨¨ZÓ8ˆà*¨1�~°Õ3°qÔ8™¸bˆØ˜1�~ Õ-°Ô2Ð2Ð2ØˆØÐä˜ Q�¨=Õ8×9ˆCØ˜MÕ)¨S°1­W¸Õ,EÐFˆAàŸ%™%›'Ÿ,™,¨2°t˜,Ó<ˆKà&×,Ñ,Ó.ˆLÜ—’ {Ó3°cÕ9ˆHØ#Ÿ™ q§x¡xÓ0ˆJØ!•~×*Ñ*¨rÐ*Ó2ˆGäœS Õ!4Ó5Ö6�Ø%¨¨S°6­\Õ)9Õ:�Ø"˜(�ÜŸš A w°Ó8�Ø×1Ñ1°%Ó8¸Õ;�Ø%¨Õ+×0Ñ0°RÐ0Ó8�ð  Õ(×-Ñ-¨aÓ0×5Ñ5¸!Ð5Ó<×AÑAÀ(Ç.Á.ÓQ�ØØØ" X™~�Ø),¨\Õ):�˜Ñ&Ø-4°\Õ-B�˜\Ó*ñ 7ð ×#Ñ# L×1ñ1 :ô4 —y’yÐ!1°qÔ9ˆä�ZÔ Ü�WÔà×#Ñ# AÓ&Ð&r!   c           	     ó€  aa€ V P                   P                  V P                  4      p\        V P                  V P
                  VVV P                  V P                  R 7      p\        P                  ! V^ R7      p. o/ o\        4       pV^ ,          P                  V4      V^ &   V P                   P                  V P                  V4        ! VV3R lR\        P                  4      pV! V^ ,          4      V^ &    V P                  VP                  \        V P                  P!                  4       4      P"                  4      4       V^ ,          P&                  V^ &   V P                  P(                  ! V3/ SB oSP+                  R4       ?S^ ,          oV^ ,          P-                  4       V^ &   V P                   P                  V P                  R4       \/        4        SP1                  R4      e   SR,          P                  V4      SR&   M RV P                   P2                  9   d   RSR&   VSS3#   \$         d     Lùi ; i)	)rc   r'   r#   r$   r-   r.   r·   c                   óB   <a a€ ] tR tRt oV 3R ltVV3R ltRtVtV ;t# )Ú(AwqQuantizer.init_quant.<locals>.CatcheriB  c                ó0   <€ \         SV `  4        Wn        R # rq   )Úsuperr;   r®   )r:   r®   Ú	__class__s   &&€r   r;   Ú1AwqQuantizer.init_quant.<locals>.Catcher.__init__C  s   ø€ Ü‘Ñ Ô"Ø$–r!   c                óì   <€ \        V4      ^ 8”  d   V^ ,          p?M1\        VP                  4       4      ^ ,          pVP                  V4      pSP	                  V4       SP                  V4       \        h)r   )r}   ÚlistÚkeysrÆ   r¸   Úupdater¬   )r:   ÚargsrÔ   Úhidden_statesÚ	first_keyr9   Úlayer_kwargss   &*,  €€r   ÚforwardÚ0AwqQuantizer.init_quant.<locals>.Catcher.forwardG  s]   ø€ ä�t“9˜q”=Ø$(¨¥G�MÙä $ V§[¡[£]Ó 3°AÕ 6�IØ$*§J¡J¨yÓ$9�Mà—‘˜MÔ*Ø×#Ñ# FÔ+Ü Ð r!   )r®   )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r;   r>  Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r4  r   r9   r=  s   @@€€r   ÚCatcherr1  B  s   ú‡ € õ%÷!ö !r!   rG  Ú	input_idsrl   rn   NÚqwen)r%   Úget_model_layersr&   r   r,   r'   r-   r.   r@   r¹   r   rt   r†   r^   rs   r~   r   r€   r¬   r®   Úprepare_inputs_for_generationrÆ   rl   r   r…   Ú
model_type)	r:   r#   r$   r7   Úsamplesr’   rG  r9   r=  s	   &&&    @@r   r6   ÚAwqQuantizer.init_quant,  sâ  ù€ Ø—.‘.×1Ñ1°$·*±*Ó=ˆÜ#Ø—‘Ø—n‘nØØ#Ø—*‘*Ø×(Ñ(ô
ˆô —)’)˜G¨Ô+ˆàˆØˆä%Ó'ˆØ˜Q•Z—]‘] ;Ó/ˆ�‰
Ø�‰×!Ñ! $§*¡*¨kÔ:÷
	!ð 	!”b—i‘iô 	!ñ& ˜W Q�ZÓ(ˆ�‰
ð	Ø�J‰J�w—z‘z¤$ t§z¡z×'<Ñ'<Ó'>Ó"?×"FÑ"FÓGÔHð ˜Q•Z×&Ñ&ˆ�‰
ð —z‘z×?Ò?ÀÑXÈ<ÑXˆà×Ñ˜Ô%àØ�A�wˆà˜Q•Z—^‘^Ó%ˆ�‰
Ø�‰×!Ñ! $§*¡*¨eÔ4äŒà×ÑÐ,Ó-Ò9Ø-9Ð:JÕ-K×-NÑ-NØó.ˆLÐ)Ò*ð �t—~‘~×0Ñ0Ô0Ø-1ˆLÐ)Ñ*à˜ dÐ*Ð*øô3 ô 	Ùð	ús   Ã8AH/ È/H=È<H=c                óŽ  € R  p\        \        4      p. pV P                  P                  R8X  d   / VCRVP                  /CpV P                  P                  R8X  g   V P                  P                  R8X  d   / VCRVP
                  /CpV P                  P                  R8X  d   / VCRVP
                  /CpV F?  pVP                  W&,          P                  \        P                  ! W6VR7      4      4       KA  	  V P                  P                  \        VP                  4       4      P                  4      V n
        V P                  V P                   V4      pV P#                  V P                  W4      V n
        V F  pVP%                  4        K  	  R p	VP'                  4        U
Uu/ uF  w  r«W©! W«4      bK  	  pp
pV# u upp
i )	c                 ó‚   € V^ ,          pVP                  4       P                  4       pWC,          P                  V4       R# )r   N)rö   rl   r¸   )rä   rµ   Úyr¯   Ú	feat_dicts   &&&&&r   Úcache_input_hookÚ6AwqQuantizer._get_input_feat.<locals>.cache_input_hooku  s-   € Ø�!•ˆAØ—‘“
—‘Ó ˆAØ�O×"Ñ" 1Ö%r!   ÚmixtralÚblock_sparse_moeÚdeepseek_v2Údeepseek_v3ÚmlpÚ	qwen3_moe)r¯   rR  c                 óz   € \         P                  ! V^ R7      pVP                  ^ ,          ^ 8w  g   Q V  R24       hV# )r   r·   zá has a zero dimension. This can happen if no data was passed through (e.g. an expert in MoE not being activated). Try increasing max_calib_samples (warning: this can significantly increase quantization time and memory usage.))r@   r¹   rI   )r“   r”   rµ   s   && r   Úcat_and_assertÚ4AwqQuantizer._get_input_feat.<locals>.cat_and_assert£  sE   € Ü—	’	˜! Ô#ˆAØ—7‘7˜1•: ”?ð Ø�#ð Bð Bó�?ð ˆHr!   )r   r7  r%   rL  rV  rY  r¸   Úregister_forward_hookÚ	functoolsÚpartialr9   rt   r~   r   r€   rÍ   r8   r¾   ÚremoverŠ   )r:   r˜   r•   rS  r–   Úhandlesr¯   r8   Úhr\  r“   r”   s   &&&         r   rŒ   ÚAwqQuantizer._get_input_feats  s§  € ò	&ô
 !¤Ó&ˆ
Øˆð �>‰>×$Ñ$¨	Ô1ðØðà" E×$:Ñ$:ñˆMð
 �>‰>×$Ñ$¨Ô5¸¿¹×9RÑ9RÐVcÔ9cðØðà�u—y‘yñˆMð
 �>‰>×$Ñ$¨Ô3ðØðà�u—y‘yñˆMó
 "ˆDØ�N‰NØÕ#×9Ñ9Ü×%Ò%Ð&6ÈZÔXóöñ "ð —I‘I—L‘L¤ e×&6Ñ&6Ó&8Ó!9×!@Ñ!@ÓAˆŒ	ð ×-Ñ-¨d×.@Ñ.@À%ÓHˆà×(Ñ(¨¯©°EÓIˆŒ	ÛˆAØ�H‰HŽJñ ò	ð ;E×:JÑ:JÔ:LÔMÑ:L±$°!�a˜¨Ó-Ò-Ñ:Lˆ
ÑMàÐùó Ns   Æ)Gc                ó¨   € \         P                  ! VP                  4      P                  p/ pVP	                  4        F  w  rVWS9   g   K  WdV&   K  	  V# )a0  
Remove the arguments that are not supported in the module's
forward pass to avoid breaking behaviour between different versions
of transformers.

Args:
    inputs_kwargs (`dict`):
        The input dictionary to pass to the model layer
    module (`torch.nn.Module`):
        Target module to quantize.
)ÚinspectÚ	signaturer>  r   rŠ   )r:   Úinputs_kwargsr®   Úmodule_signatureÚsanitized_kwargsr“   r”   s   &&&    r   rÍ   ÚAwqQuantizer._sanitize_kwargs¯  sP   € ô #×,Ò,¨V¯^©^Ó<×GÑGÐØÐØ!×'Ñ'Ö)‰DˆAØÖ$Ø&' Ó#ñ *ð  Ðr!   )r
   r%   r,   r/   r0   r)   r9   r2   r3   r4   r&   r8   r7   r5   r1   r-   r.   r'   r+   r(   r*   )NFTNé€   é   i   @rq   )é   g      à?rm  )rl  rm  )r@  rA  rB  rC  r;   r[   rf   r›   rŸ   r�   r@   rÌ   r¾   rŽ   rÑ   rð   r�   r  r6   rŒ   rÍ   rD  rE  )r   s   @r   r   r      sþ   ø‡ € ÷+
ò +
÷Z# ð # ÷Jò ò Yòv÷'ð 'ðR ‡]‚]ƒ_÷ó ðð2 ‡]‚]ƒ_ð Ø÷G
ñ G
ó ðG
ðb ÷J*ò J*ðX ‡]‚]ƒ_÷ó ðð> ‡]‚]ƒ_ñó ðð$ ‡]‚]ƒ_÷:'ñ :'ó ð:'ôxE+òN:÷x ð  r!   r   )"r‡   r@   rf  ró   r_  Útorch.nnr^   r   Útypingr   r   r   Úcollectionsr   Úawq.utils.calib_datar   Úawq.quantize.scaler	   r
   Úawq.utils.utilsr   r   Úawq.modules.linearr   r   r   r   Úawq.utils.moduler   r   r   r   r   r   r   r!   r   Ú<module>rw     sL   ðÛ Û Û Û Û Ý Ý ß 'Ñ 'Ý #Ý 2ß 6ß 9÷ó ÷õ ÷d
 ó d
 r!   