+
    œ°³j3:  ã                   óÆ   € ^ RI Ht ^RIHtHtHtHt ^RIHt ^RI	H
t
 ]! 4       '       d   ^ RIt]'       d   ^RIHt ^RIHt ]P                   ! ]4      t ! R R	]4      tR# )
é    )ÚTYPE_CHECKING)Úis_accelerate_availableÚis_torch_availableÚis_torch_xpu_availableÚlogging)ÚHfQuantizer)Úget_module_from_nameN)ÚPreTrainedModel)ÚFineGrainedFP8Configc                   óö   a a€ ] tR t^t oRtRtV 3R ltR tV3R lR ltV3R lV 3R llt	V3R	 lR
 lt
V3R lR ltR tR tR t]V3R lR l4       t]V3R lR l4       tR tR tR tV3R ltRtVtV ;t# )ÚFineGrainedFP8HfQuantizerzz
FP8 quantization implementation supporting both standard and MoE models.
Supports both e4m3fn formats based on platform.
Fc                ó*   <€ \         SV `  ! V3/ VB  R # )N)ÚsuperÚ__init__)ÚselfÚquantization_configÚkwargsÚ	__class__s   &&,€Úr/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/quantizers/quantizer_finegrained_fp8.pyr   Ú"FineGrainedFP8HfQuantizer.__init__   s   ø€ Ü‰ÒÐ,Ñ7°Ô7ó    c                ó¼  € \        4       '       g   \        R 4      hV P                  P                  '       d   R# \        P
                  P                  4       '       gV   \        4       '       gF   V P                  '       d)   \        P                  R4       RV P                  n        R# \        R4      h\        P
                  P                  4       '       dg   \        P
                  P                  4       pVw  rEV^8  g   V^8X  d7   V^	8  d0   \        P                  RV RV R24       RV P                  n        R# VP                  R4      pVf   \        P                  R	4       R# \        V\        4      '       d^   V P                  '       gJ   \!        V4      ^8”  d8   R
VP#                  4       9   g   RVP#                  4       9   d   \%        R4      hR# R# R# R# )zMLoading an FP8 quantized model requires accelerate (`pip install accelerate`)Nz„Using FP8 quantized models requires a GPU or XPU, we will default to dequantizing the model to bf16 since no GPU or XPU is availableTzANo GPU or XPU found. A GPU or XPU is needed for FP8 quantization.ziFP8 quantized models is only supported on GPUs with compute capability >= 8.9 (e.g 4090/H100), actual = `Ú.zƒ`. We will default to dequantizing the model to bf16. Feel free to use a different quantization method like bitsandbytes or torchaoÚ
device_mapz×You have loaded an FP8 model on CPU and have a CUDA or XPU device available, make sure to set your model on a GPU or XPU device in order to run your model. To remove this warning, pass device_map = 'cuda' or 'xpu'. ÚcpuÚdiskzìYou are attempting to load an FP8 model with a device_map that contains a cpu/disk device.This is not supported when the model is quantized on the fly. Please use a quantized checkpoint or remove the cpu/disk device from the device_map.)r   ÚImportErrorr   Ú
dequantizeÚtorchÚcudaÚis_availabler   Úpre_quantizedÚloggerÚwarning_onceÚRuntimeErrorÚget_device_capabilityÚgetÚ
isinstanceÚdictÚlenÚvaluesÚ
ValueError)r   Úargsr   Úcompute_capabilityÚmajorÚminorr   s   &*,    r   Úvalidate_environmentÚ.FineGrainedFP8HfQuantizer.validate_environment   s›  € Ü&×(Ò(ÜÐmÓnÐnà×#Ñ#×.×.Ð.Ùä�z‰z×&Ñ&×(Ò(Ô1G×1IÒ1IØ×!×!Ð!Ü×#Ñ#ð [ôð 7;�×(Ñ(Ô3Ùä"Ð#fÓgÐgä�:‰:×"Ñ"×$Ò$Ü!&§¡×!AÑ!AÓ!CÐØ-‰LˆEØ˜”	˜u¨œz¨e°a¬iÜ×#Ñ#ð#Ø#( '¨¨5¨'ð 2Zð[ôð
 7;�×(Ñ(Ô3Ùà—Z‘Z Ó-ˆ
ØÒÜ×Ñð6öô
 ˜
¤D×)Ò)à×&×&Ð&Ü˜
“O aÔ'Ø˜j×/Ñ/Ó1Ô1°V¸z×?PÑ?PÓ?RÔ5Rä ðkóð ñ 6Sñ (ñ 'ñ *r   c                ó*   <€ V ^8„  d   QhRRRS[ RS[/# )é   Úmodelr
   Ú
param_nameÚreturn)ÚstrÚbool)ÚformatÚ__classdict__s   "€r   Ú__annotate__Ú&FineGrainedFP8HfQuantizer.__annotate__N   s$   ø€ ÷ 
ñ 
Ð.?ð 
ÉSð 
Ñ_cñ 
r   c                ó�   € ^RI HpHp \        W4      w  rg\	        WeV34      '       d   V P
                  '       g   VR8X  d   R# R# R# )r4   )Ú
FP8ExpertsÚ	FP8LinearÚbiasFT)Úintegrations.finegrained_fp8r?   r@   r	   r(   r"   )r   r5   r6   r   r?   r@   ÚmoduleÚtensor_names   &&&,    r   Úparam_needs_quantizationÚ2FineGrainedFP8HfQuantizer.param_needs_quantizationN   s>   € çHä2°5ÓEÑˆÜ�f¨*Ð5×6Ò6Ø×!×!Ð! [°FÔ%:ÙáÙr   c                ó.   <€ V ^8„  d   QhRRRS[ RRRS[/# )r4   r5   r
   r6   Úparamztorch.Tensorr7   )r8   Úfloat)r:   r;   s   "€r   r<   r=   Z   s2   ø€ ÷ Dñ DÐ(9ð DÁsð DÐSað DÑfkñ Dr   c                óV   <€ V P                  W4      '       d   ^# \        SV `	  WV4      # )z4Return the element size (in bytes) for `param_name`.)rE   r   Úparam_element_size)r   r5   r6   rH   r   s   &&&&€r   rK   Ú,FineGrainedFP8HfQuantizer.param_element_sizeZ   s)   ø€ à×(Ñ(¨×;Ò;áÜ‰wÑ)¨%¸UÓCÐCr   c                ó   <€ V ^8„  d   QhRR/# ©r4   r5   r
   © )r:   r;   s   "€r   r<   r=   a   s   ø€ ÷ Cñ CÐ7Hñ Cr   c                óö   € V P                   P                  pV'       g   R# ^RIHp V! V4      p. pV F2  pTpV F  pVP	                  V4      w  ryK  	  VP                  V4       K4  	  WPP                   n        R# )u  Rewrite the skip-list to the model's own module tree.
For models that were already released, if they have a list of modules to not quantize
we need to apply the weight renaming / weight conversion opÃ©rations to get the actual
layer name of the model in `transformers`.
N)Úget_model_conversion_mapping)r   Úmodules_to_not_convertÚconversion_mappingrQ   Úrename_source_keyÚappend)
r   r5   ÚskiprQ   Ú	renamingsÚremappedÚnameÚrenamedÚrenameÚ_s
   &&        r   Ú!_normalize_modules_to_not_convertÚ;FineGrainedFP8HfQuantizer._normalize_modules_to_not_converta   su   € ð ×'Ñ'×>Ñ>ˆßÙåEá0°Ó7ˆ	ØˆÛˆDØˆGÛ#�Ø#×5Ñ5°gÓ>‘
�™ñ $à�O‰O˜GÖ$ñ	 ð
 ;C× Ñ Ö7r   c                ó   <€ V ^8„  d   QhRR/# rN   rO   )r:   r;   s   "€r   r<   r=   v   s   ø€ ÷ 
ñ 
à ñ
r   c                ó|  € ^RI HpHp V P                  V4       V P	                  WP
                  P                  VP                  4      V n        V P
                  P                  pV P                  '       d   V'       d   V! WV P                  4       V! VV P                  V P
                  V P                  R7      pR# )r4   )Úreplace_with_fp8_embeddingÚreplace_with_fp8_linear)rR   r   r"   N)
rB   ra   rb   r]   Úget_modules_to_not_convertr   rR   Ú_keep_in_fp32_modulesÚmodules_to_convertr"   )r   r5   r   ra   rb   re   s   &&,   r   Ú$_process_model_before_weight_loadingÚ>FineGrainedFP8HfQuantizer._process_model_before_weight_loadingv   s›   € ÷
 	gà×.Ñ.¨uÔ5Ø&*×&EÑ&EØ×+Ñ+×BÑBÀE×D_ÑD_ó'
ˆÔ#ð "×5Ñ5×HÑHÐØ××Ð×"4Ù& uÀ$×B]ÑB]Ô^á'ØØ#'×#>Ñ#>Ø $× 8Ñ 8Ø×,Ñ,ô	
Šr   c           
     ó2  € V P                   P                  R 8X  dè   ^RIHp V! 4       pVP	                  4        UUu. uFA  w  rVVP                  R4      '       g   K  VP                  \        P                  8X  g   K?  VNKC  	  pppV Fw  pVP                  R4      w  r‰p
VP                  V4      p\        Wº4      p\        Wº\        P                  P                  VP                  P!                  V4      RR7      4       Ky  	  ^RIHp V! V4       V# u uppi )Úue8m0)Ú_get_ue8m0_dtypeÚ
_scale_invr   F)Úrequires_grad)Ú!_disable_deepgemm_on_multi_device)r   Ú	scale_fmtrB   rj   Únamed_parametersÚendswithÚdtyper   Úfloat32Ú
rpartitionÚget_submoduleÚgetattrÚsetattrÚnnÚ	ParameterÚdataÚtorm   )r   r5   r   rj   ri   rY   rH   Úfloat32_scalesÚmodule_namer\   ÚattrrC   Úscalerm   s   &&,           r   Ú#_process_model_after_weight_loadingÚ=FineGrainedFP8HfQuantizer._process_model_after_weight_loading�   sð   € ð
 ×#Ñ#×-Ñ-°Ô8ÝGá$Ó&ˆEð $)×#9Ñ#9Ô#;ôá#;‘K�DØ—=‘= ×.ô à38·;±;Ä%Ç-Á-Ñ3O÷ �Ù#;ð ñ ó
 '�Ø'+§¡°sÓ';Ñ$� Ø×,Ñ,¨[Ó9�Ü Ó-�Ü˜¤e§h¡h×&8Ñ&8¸¿¹¿¹ÀuÓ9MÐ]bÐ&8Ó&cÖdñ	 'õ 	Uá)¨%Ô0Øˆùó!s   ¼DÁDÁ=Dc                ó8  € R VP                   P                  9   d%   RRRRRRRRRRRRRR	R
R	RRRRRRRRRR	RR	/pW!n        ^RIHp \        VRR4      pVP                  P                  V/ 4      pR FŸ  p\        WR4      ;'       g    / pVP                  4        UU	u/ uF  w  r‰W…P                  W™4      bK  	  p
pp	\        V
P                  4       4       F#  w  r¼VR8X  g   K  V
P                  V R2V4       K%  	  W§8w  g   K“  \        WV
4       K¡  	  V# u up	pi )ÚQwen3z layers.*.self_attn.q_proj.weightÚcolwisez*layers.*.self_attn.q_proj.weight_scale_invz layers.*.self_attn.k_proj.weightz*layers.*.self_attn.k_proj.weight_scale_invz layers.*.self_attn.v_proj.weightz*layers.*.self_attn.v_proj.weight_scale_invz layers.*.self_attn.o_proj.weightÚrowwisez*layers.*.self_attn.o_proj.weight_scale_invzlayers.*.mlp.gate_proj.weightz'layers.*.mlp.gate_proj.weight_scale_invzlayers.*.mlp.up_proj.weightz%layers.*.mlp.up_proj.weight_scale_invzlayers.*.mlp.down_proj.weightz'layers.*.mlp.down_proj.weight_scale_inv)r?   Ú_experts_implementationNÚgrouped_gemmrk   )Úbase_model_tp_planÚbase_model_ep_plan)r   Ú__name__r‡   rB   r?   ru   Ú_impl_tp_layer_overridesr'   ÚitemsÚlistÚ
setdefaultrv   )r   ÚconfigÚ	text_planr?   ÚimplÚlayer_overridesÚ	plan_attrÚ	base_planÚkÚvÚupdated_planÚkeyÚstyles   &&           r   Úupdate_tp_planÚ(FineGrainedFP8HfQuantizer.update_tp_plan¨   s?  € Ø�f×&Ñ&×/Ñ/Ô/à2°IØ<¸iØ2°IØ<¸iØ2°IØ<¸iØ2°IØ<¸iØ/°Ø9¸9Ø-¨yØ7¸Ø/°Ø9¸9ðˆIð" )2Ô%å=ä�vÐ8¸$Ó?ˆØ$×=Ñ=×AÑAÀ$ÈÓKˆÛEˆIÜ °4Ó8×>Ð>¸BˆIð
 FOÇ_Á_ÔEVÔWÑEV¹T¸Q˜A×2Ñ2°1Ó8Ò8ÑEVˆLÑWô # <×#5Ñ#5Ó#7Ö8‘
�Ø˜NÖ*Ø ×+Ñ+¨s¨e°:Ð,>ÀÖFñ 9ð Ö(Ü˜¨<Ö8ñ Fð  ˆùó Xs   ÂDc                ó   € R # ©TrO   ©r   s   &r   Úis_serializableÚ)FineGrainedFP8HfQuantizer.is_serializableÓ   s   € Ùr   c                ó    <€ V ^8„  d   QhRS[ /# ©r4   r7   ©r9   )r:   r;   s   "€r   r<   r=   ×   s   ø€ ÷ ñ ™dñ r   c                ó   € R # )FrO   r�   s   &r   Úis_trainableÚ&FineGrainedFP8HfQuantizer.is_trainableÖ   s   € ár   c                ó    <€ V ^8„  d   QhRS[ /# r¡   r¢   )r:   r;   s   "€r   r<   r=   Û   s   ø€ ÷ ñ ¡ñ r   c                ó   € R # rœ   rO   r�   s   &r   Úis_compileableÚ(FineGrainedFP8HfQuantizer.is_compileableÚ   s   € ár   c                ó   € ^RI Hp V! V 4      # )r4   )ÚFp8Quantize)rB   r«   )r   r«   s   & r   Úget_quantize_opsÚ*FineGrainedFP8HfQuantizer.get_quantize_opsÞ   s   € Ý>á˜4Ó Ð r   c                ó¤   € ^RI Hp ^RIHp V P                  '       d2   V P
                  P                  '       d   V! . RORV! V 4      .R7      .# . # )r4   )ÚWeightConverter©ÚFp8DequantizeÚweight©Úsource_patternsÚtarget_patternsÚ
operations)zweight$Úweight_scale_invÚactivation_scale)Úcore_model_loadingr¯   rB   r±   r"   r   r   )r   r¯   r±   s   &  r   Úget_weight_conversionsÚ0FineGrainedFP8HfQuantizer.get_weight_conversionsã   sQ   € Ý8Ý@à××Ð $×":Ñ":×"E×"EÐ"Eñ  Ú$WØ$,Ù -¨dÓ 3Ð4ôðð ð ˆ	r   c           	     óh  € ^RI HpHp ^RIHp V! RRR7      pV.\        V4      ,           pV FR  p\        Wb4      '       g   K  VP                   Uu. uF   qwP                  R4      '       d   V R2MTNK"  	  upVn        KT  	  V P                  '       d   V P                  P                  '       g   WP                  4       ,           # . pV EF)  p\        Wb4      '       g   VP                  V4       K(  VP                   Uu. uF  qwP                  R4      '       g   K  VNK  	  p	pV	'       d¶   V	 Uu. uF  qwR,           NK  	  p
pV	 Uu. uF  qwR\        R4      )  R	,           NK  	  ppVP                   Uu. uF  qwP                  R4      '       d   K  VNK  	  ppW«,           V,           pV! V 4      .\        VP                   4      ,           pV! VVP                  VR
7      pVP                  V4       EK,  	  VP#                  V P                  4       4       V# u upi u upi u upi u upi u upi )u]  When loading with ``dequantize=True``, attach an :class:`Fp8Dequantize` op to
every existing :class:`WeightConverter` so that per-block scales are folded into
the weight *before* any later merge/concat ops collapse the per-expert structure.

For each model-supplied converter that has a ``.weight`` source, we:
  1. anchor the existing weight patterns with ``$`` so they don't accidentally
     also match the ``.weight_scale_inv`` keys (the regex is searched, so the
     unanchored prefix would match both, sending scales to the wrong bucket);
  2. add anchored ``*.weight_scale_inv`` sources next to each weight pattern so
     the loader collects scale tensors alongside the weight tensors into the
     *same* converter bucket (both keys rewrite to the same target);
  3. prepend a fresh :class:`Fp8Dequantize` op so dequant runs first, before
     any merge/concat collapses the per-expert structure.

The generic ``weight$ + weight_scale_inv â†’ weight`` converter from
:meth:`get_weight_conversions` is still appended at the end as a fallback for
plain ``nn.Linear`` weights with no model-specific converter.
)r¯   ÚWeightRenamingr°   z^(.+)\.scale$z\1.weight_scale_inv)r´   rµ   z.weightÚ$Nz.weight_scale_inv$r³   )r¹   r¯   r½   rB   r±   rŒ   r(   Ú_original_target_patternsrp   r"   r   r   rº   rU   r´   r*   r¶   Úextend)r   Úweight_conversionsr¯   r½   r±   Úscale_renameÚconvÚpÚupdatedÚweight_sourcesÚanchored_weightÚscale_sourcesÚotherÚnew_sourcesÚnew_opss   &&             r   Úupdate_weight_conversionsÚ3FineGrainedFP8HfQuantizer.update_weight_conversionsó   sï  € ÷& 	IÝ@ñ &Ð6FÐXnÔoˆØ*˜^¬dÐ3EÓ.FÕFÐó 'ˆDÜ˜$×0Ô0àEI×EcÒEcó2ÙEcÀŸz™z¨)×4Ò4�q�c˜‘G¸!Ò;ÑEcñ2�Ö.ñ 'ð ×"×"Ð" t×'?Ñ'?×'J×'JÐ'JØ%×(CÑ(CÓ(EÕEÐEàˆÜ&ˆDô ˜d×4Ò4Ø—‘˜tÔ$ÙØ)-×)=Ò)=ÓWÑ)= AÇÁÈI×AVŸa˜aÑ)=ˆNÐWßÙ4BÓ"C±N¨q s§7 7±N�Ð"CÙVdÓ eÑVdÐQRÐ#4¤c¨)£n _Ð!5Ð8L×!LÐ!LÑVd�Ð eØ$(×$8Ò$8ÓVÑ$8˜qÇ
Á
È9×@UŸ˜Ñ$8�ÐVØ-Õ=ÀÕE�Ù(¨Ó.Ð/´$°t·±Ó2GÕG�Ù&Ø$/Ø$(×$BÑ$BØ&ô�ð
 �N‰N˜4× ñ% 'ð( 	�‰�t×2Ñ2Ó4Ô5Øˆùò;2ùò Xùâ"CùÚ eùÚVs*   Á&HÄH ÄH Ä3H%Å
H*Å8H/ÆH/c                ó$   <€ V ^8„  d   Qh/ R;R&   # )r4   r   r   rO   )r:   r;   s   "€r   r<   r=      s   ø‡ ‚ ð 0Ñ/ò r   )rR   )r‰   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Úrequires_calibrationr   r1   rE   rK   r]   rf   r   r™   rž   Úpropertyr¤   r¨   r¬   rº   rÌ   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r   r;   s   @@r   r   r      s˜   ù‡ € ñð
 !Ðõ8ò.÷`
ð 
÷Dó D÷Cð C÷*
ð 
ò.ò6)òVð ÷ó ðð ÷ó ðò!ò
ò ?÷C … r   r   )Útypingr   Úutilsr   r   r   r   Úbaser   Úquantizers_utilsr	   r   Úmodeling_utilsr
   Úutils.quantization_configr   Ú
get_loggerr‰   r#   r   rO   r   r   Ú<module>rà      sI   ðÝ  ç `Ó `Ý Ý 2ñ ×ÒÛçÝ0Ý@à	×	Ò	˜HÓ	%€ô` ö `r   