+
    œ°³j>&  ã                   óÒ   € ^ RI Ht ^RIHt ]'       d   ^RIHt ^RIHt ^RIH	t	H
t
HtHtHtHtHt ^RIHt ]! 4       '       d   ^ RIt]P&                  ! ]4      t ! R R	]4      tR# )
é    )ÚTYPE_CHECKING)ÚHfQuantizer)ÚPreTrainedModel)ÚFbgemmFp8Config)Úis_accelerate_availableÚis_fbgemm_gpu_availableÚis_kernels_availableÚis_torch_availableÚis_torch_cuda_availableÚis_torch_xpu_availableÚlogging)Úget_module_from_nameNc                   óÎ   a a€ ] tR t^)t oRtRtV 3R ltR tV3R lR ltV3R lR lt	V3R	 lV 3R
 llt
V3R lR ltR tR tR t]V3R lR l4       tR tV3R ltRtVtV ;t# )ÚFbgemmFp8HfQuantizerz'
FP8 quantization using fbgemm kernels
Fc                ó*   <€ \         SV `  ! V3/ VB  R # )N)ÚsuperÚ__init__)ÚselfÚquantization_configÚkwargsÚ	__class__s   &&,€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/quantizers/quantizer_fbgemm_fp8.pyr   ÚFbgemmFp8HfQuantizer.__init__1   s   ø€ Ü‰ÒÐ,Ñ7°Ô7ó    c                óà  € \        4       '       g   \        4       '       g   \        R 4      h\        4       '       d   \        4       '       g   \        R4      h\        4       '       d   \	        4       '       g   \        R4      h\        4       '       g   \        R4      h\        4       '       d5   \        P                  P                  4       pVw  rEV^	8  d   \        R4      hVP                  R4      pVf   \        P                  R4       R# \        V\        4      '       dL   V P                  '       g8   RVP!                  4       9   g   R	VP!                  4       9   d   \        R
4      hR# R# R# )z3Using fbgemm fp8 quantization requires a GPU or XPUz@Using FP8 fbgemm on XPU requires kernels (`pip install kernels`)züLoading an FP8 fbgemm quantized model on CUDA requires fbgemm-gpu libraryPlease install the latest version of fbgemm-gpu library by following : https://pytorch.org/FBGEMM/fbgemm_gpu-development/InstallationInstructions.html#fbgemm-gpu-install-librarieszWLoading an FP8 quantized model requires accelerate (`pip install --upgrade accelerate`)zXFP8 quantized models is only supported on GPUs with compute capability >= 9.0 (e.g H100)Ú
device_mapNzÛYou have loaded an FP8 model on CPU and have a CUDA/XPU device available, make sure to set your model on a GPU/XPU device in order to run your model. To remove this warning, pass device_map = 'cuda' or 'xpu' or 'auto'. ÚcpuÚdiskzòYou are attempting to load an FP8 model with a device_map that contains a CPU or disk device.This is not supported when the model is quantized on the fly. Please use a quantized checkpoint or remove the CPU or disk device from the device_map.)r   r   ÚImportErrorr	   r   r   ÚtorchÚcudaÚget_device_capabilityÚ
ValueErrorÚgetÚloggerÚwarning_onceÚ
isinstanceÚdictÚpre_quantizedÚvalues)r   Úargsr   Úcompute_capabilityÚmajorÚ_r   s   &*,    r   Úvalidate_environmentÚ)FbgemmFp8HfQuantizer.validate_environment4   sB  € Ü&×(Ò(Ô1G×1IÒ1IÜÐSÓTÐTÜ!×#Ò#Ô,@×,BÒ,BÜÐ`ÓaÐaÜ"×$Ò$Ô-D×-FÒ-FÜðFóð ô '×(Ò(ÜØióð ô #×$Ò$Ü!&§¡×!AÑ!AÓ!CÐØ)‰HˆEØ�qŒyÜ Ønóð ð —Z‘Z Ó-ˆ
ØÒÜ×ÑðSöô ˜
¤D×)Ò)Ø×%×%Ð%¨5°J×4EÑ4EÓ4GÔ+GÈ6ÐU_×UfÑUfÓUhÔKhÜ ðnóð ñ LiÑ%ñ *r   c                ó"   <€ V ^8„  d   QhRRRR/# )é   Údtypeztorch.dtypeÚreturn© )ÚformatÚ__classdict__s   "€r   Ú__annotate__Ú!FbgemmFp8HfQuantizer.__annotate__X   s   ø€ ÷ ñ  -ð °Mñ r   c                ó‚   € V\         P                  8w  d*   \        P                  R V R24       \         P                  pV# )zSetting dtype to zP, but only bfloat16 is supported right now. Overwriting torch_dtype to bfloat16.)r    Úbfloat16r%   r&   )r   r3   s   &&r   Úupdate_dtypeÚ!FbgemmFp8HfQuantizer.update_dtypeX   s9   € Ø”E—N‘NÔ"Ü×ÑØ# E 7Ð*zÐ{ôô —N‘NˆEØˆr   c                ó*   <€ V ^8„  d   QhRRRS[ RS[/# )r2   Úmodelr   Ú
param_namer4   )ÚstrÚbool)r6   r7   s   "€r   r8   r9   `   s$   ø€ ÷ ñ Ð.?ð ÉSð Ñ_cñ r   c                óè   € ^RI HpHp \        W4      w  rg\	        Wd4      '       d   V P
                  '       g   VR8X  d   R# R# \	        We4      '       d   V P
                  '       g   VR8X  d   R# R# R# )r2   ©ÚFbgemmFp8LinearÚFbgemmFp8Llama4TextExpertsÚbiasFT)ÚintegrationsrE   rF   r   r'   r)   )r   r?   r@   r   rE   rF   ÚmoduleÚtensor_names   &&&,    r   Úparam_needs_quantizationÚ-FbgemmFp8HfQuantizer.param_needs_quantization`   s]   € ßNä2°5ÓEÑˆä�f×.Ò.Ø×!×!Ð! [°FÔ%:ÙáÜ�f×9Ò9Ø×!×!Ð! [°FÔ%:ÙáÙr   c                ó.   <€ V ^8„  d   QhRRRS[ RRRS[/# )r2   r?   r   r@   Úparamztorch.Tensorr4   )rA   Úfloat)r6   r7   s   "€r   r8   r9   q   s2   ø€ ÷ Dñ DÐ(9ð DÁsð DÐSað DÑfkñ Dr   c                óV   <€ V P                  W4      '       d   ^# \        SV `	  WV4      # )z4Return the element size (in bytes) for `param_name`.)rK   r   Úparam_element_size)r   r?   r@   rN   r   s   &&&&€r   rQ   Ú'FbgemmFp8HfQuantizer.param_element_sizeq   s)   ø€ à×(Ñ(¨×;Ò;áÜ‰wÑ)¨%¸UÓCÐCr   c                ó   <€ V ^8„  d   QhRR/# )r2   r?   r   r5   )r6   r7   s   "€r   r8   r9   x   s   ø€ ÷ 
ñ 
à ñ
r   c                óÐ   € ^RI Hp V P                  WP                  P                  VP
                  4      V n        V! VV P                  V P                  V P                  R7      pR# )r2   )Úreplace_with_fbgemm_fp8_linear)Úmodules_to_not_convertr   r)   N)rH   rU   Úget_modules_to_not_convertr   rV   Ú_keep_in_fp32_modulesr)   )r   r?   r   rU   s   &&, r   Ú$_process_model_before_weight_loadingÚ9FbgemmFp8HfQuantizer._process_model_before_weight_loadingx   s\   € õ
 	Bà&*×&EÑ&EØ×+Ñ+×BÑBÀE×D_ÑD_ó'
ˆÔ#ñ /ØØ#'×#>Ñ#>Ø $× 8Ñ 8Ø×,Ñ,ô	
Šr   c                óô   € ^RI HpHp VP                  4        F[  p\	        WSV34      '       g   K  \        VR4      '       g   K,  VP                  P                  V P                  P                  4       K]  	  V# )zÉ
Force update the input scale upper bound after weight loading and device dispatch are complete.
This resolves issues where persistent buffers are zeroed out or overwritten during the loading process.
rD   Úinput_scale_ub)
Úintegrations.fbgemm_fp8rE   rF   Úmodulesr'   Úhasattrr\   Úfill_r   Úactivation_scale_ub)r   r?   r   rE   rF   Úms   &&,   r   Ú#_process_model_after_weight_loadingÚ8FbgemmFp8HfQuantizer._process_model_after_weight_loadingŠ   s^   € ÷
 	Zà—‘–ˆAÜ˜!Ð/IÐJ×KÔKÜ˜1Ð.×/Ô/à×$Ñ$×*Ñ*¨4×+CÑ+C×+WÑ+WÖXñ	 !ð
 ˆr   c                ó"  € R VP                   P                  9   dt   / RRbRRbRRbRRbRRbRRbRR	bR
RbRRbRRbRRbRRbRRbRRbRR	bRRbRRbRRRRRR	RRRRRR/CpVP                  4       e   W!P                  4       n        V# W!n        V# V# )ÚLlama4z layers.*.self_attn.q_proj.weightÚcolwisez&layers.*.self_attn.q_proj.weight_scalez layers.*.self_attn.k_proj.weightz&layers.*.self_attn.k_proj.weight_scalez layers.*.self_attn.v_proj.weightz&layers.*.self_attn.v_proj.weight_scalez layers.*.self_attn.o_proj.weightÚrowwisezlayers.*.input_layernorm.weightÚsequence_parallelz(layers.*.post_attention_layernorm.weightznorm.weightz4layers.*.feed_forward.shared_expert.gate_proj.weightz:layers.*.feed_forward.shared_expert.gate_proj.weight_scalez2layers.*.feed_forward.shared_expert.up_proj.weightz8layers.*.feed_forward.shared_expert.up_proj.weight_scalez4layers.*.feed_forward.shared_expert.down_proj.weightz0layers.*.feed_forward.experts.*.gate_proj.weightz6layers.*.feed_forward.experts.*.gate_proj.weight_scalez.layers.*.feed_forward.experts.*.up_proj.weightz4layers.*.feed_forward.experts.*.up_proj.weight_scalez0layers.*.feed_forward.experts.*.down_proj.weightz*layers.*.feed_forward.experts.gate_up_projÚpacked_rowwisez0layers.*.feed_forward.experts.gate_up_proj_scalez'layers.*.feed_forward.experts.down_proj)r   Ú__name__Úget_text_configÚbase_model_tp_plan)r   ÚconfigÚ	text_plans   && r   Úupdate_tp_planÚ#FbgemmFp8HfQuantizer.update_tp_plan˜   sX  € Ø�v×'Ñ'×0Ñ0Ô0ð!ð 3°Ið	!ð
 9¸)ð!ð 3°Ið!ð 9¸)ð!ð 3°Ið!ð 9¸)ð!ð 3°Ið!ð 2Ð3Fð!ð ;Ð<Oð!ð Ð2ð!ð$ GÈ	ð%!ð& MÈið'!ð( EÀið)!ð* KÈIð+!ð, GÈ	ð-!ð. CÀIð/!ð0 IÈ)ð1!ð2 AÀ)ØFÈ	ØBÀIð =Ð>NØBÐDTØ9¸9ñA!ˆIðD ×%Ñ%Ó'Ò3Ø>G×&Ñ&Ó(Ô;ð ˆMð -6Ô)ØˆMàˆr   c                ó   € R # )Tr5   ©r   s   &r   Úis_serializableÚ$FbgemmFp8HfQuantizer.is_serializableÄ   s   € Ùr   c                ó    <€ V ^8„  d   QhRS[ /# )r2   r4   )rB   )r6   r7   s   "€r   r8   r9   È   s   ø€ ÷ ñ ™dñ r   c                ó   € R # )Fr5   rs   s   &r   Úis_trainableÚ!FbgemmFp8HfQuantizer.is_trainableÇ   s   € ár   c                ó   € ^RI Hp V! V 4      # )r2   )ÚFbgemmFp8Quantize)r]   r{   )r   r{   s   & r   Úget_quantize_opsÚ%FbgemmFp8HfQuantizer.get_quantize_opsË   s   € Ý?á  Ó&Ð&r   c                ó$   <€ V ^8„  d   Qh/ R;R&   # )r2   r   r   r5   )r6   r7   s   "€r   r8   r9   )   s   ø‡ ‚ ð +Ñ*ò r   )rV   )rk   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Úrequires_calibrationr   r/   r<   rK   rQ   rY   rc   rp   rt   Úpropertyrx   r|   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r   r7   s   @@r   r   r   )   sx   ù‡ € ñð !Ðõ8ò"÷Hð ÷ð ÷"Dó D÷
ð 
ò$ò*òXð ÷ó ðò'÷E … r   r   )Útypingr   Úbaser   Úmodeling_utilsr   Úutils.quantization_configr   Úutilsr   r   r	   r
   r   r   r   Úquantizers_utilsr   r    Ú
get_loggerrk   r%   r   r5   r   r   Ú<module>r�      sX   ðõ !å ÷ Ý0Ý;÷÷ ñ õ 3ñ ×ÒÛà	×	Ò	˜HÓ	%€ôe'˜;ö e'r   