+
    œ°³j]Î  ã                  ó¢  a € 0 t $ ^ RIHt ^ RIt^ RIt^ RIHt ^ RIHt ^ RI	t	^ RI
Ht ^ RI
Ht ^RIHt ^RIHt ^RIHtHt ^R	IHt ^R
IHt ^RIHtHtHtHtHt ^RIH t H!t! ^RI"H#t#H$t$ ]PJ                  ! ]&4      t']	PP                  t)]	PT                  ! ])4      PV                  t,]	PT                  ! ])4      PZ                  t.]P^                  R R l4       t0R t1]! RR7       ! R R4      4       t2Rs3R]4R&   ]	Pj                  Pl                  R R l4       t7R R lt8R R lt9R;R R llt:R<R R  llt;R=R! R" llt< ! R# R$]Pz                  4      t> ! R% R&]P~                  4      t@ ! R' R(]>4      tAR) R* ltBR+ R, ltC ! R- R.]Pˆ                  4      tE ! R/ R0]#4      tF]F! 4       tGR1 R2 ltHR>R3 R4 lltIR?R5 R6 lltJ ! R7 R8]4      tK ! R9 R:]4      tLR# )@é    )ÚannotationsN)ÚCallable)Ú	dataclass)Ú
functional)ÚACT2FN)ÚConversionOps)Úget_module_from_nameÚshould_convert_module)Úlogging)Úis_kernels_available)Ú deepgemm_fp8_fp4_experts_forwardÚdeepgemm_fp8_fp4_linearÚ(deepgemm_fp8_fp4_megamoe_experts_forwardÚis_deepgemm_loadableÚis_sm100)Ú_MISSING_KERNELS_MESSAGEÚlazy_load_kernel)ÚExpertsInterfaceÚuse_experts_implementationc               ó   € V ^8„  d   QhRR/# )é   Úreturnútorch.dtype© )Úformats   "Új/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/integrations/finegrained_fp8.pyÚ__annotate__r   2   s   € ÷  ñ  ˜+ñ  ó    c                 óˆ   € \        \        R4      '       g   \        R\        P                   R24      h\        P                  # )uG  Return ``torch.float8_e8m0fnu`` or raise a clear error on torch without FP8 support.

UE8M0 scales are always stored/consumed as this single dtype â€” the kernels (Triton
finegrained + DeepGEMM) read it natively, and supporting the same scales in mixed
container dtypes would be a mess â€” so fail loudly rather than fall back.Úfloat8_e8m0fnuzbscale_fmt='ue8m0' requires torch.float8_e8m0fnu, which is only available in PyTorch >= 2.7 (found z.). Upgrade torch to use UE8M0 FP8 checkpoints.)ÚhasattrÚtorchÚRuntimeErrorÚ__version__r    r   r   r   Ú_get_ue8m0_dtyper%   1   sG   € ô ”5Ð*×+Ò+Üð%Ü%*×%6Ñ%6Ð$7Ð7eðgó
ð 	
ô ×ÑÐr   c                ó–   € V F!  p\        W4      '       g   K  \        W4      u # 	  \        \        V 4      P                   R V 24      h)z has none of: )r!   ÚgetattrÚAttributeErrorÚtypeÚ__name__)ÚobjÚnamesÚnames   &* r   Ú_first_attrr.   @   sE   € ÛˆÜ�3×ÔÜ˜3Ó%Ò%ñ ô œD ›I×.Ñ.Ð/¨~¸e¸WÐEÓ
FÐFr   T)Úfrozenc                  ó:   € ] tR t^Gt$ RtR]R&   R]R&   R]R&   RtR# )ÚFineGrainedFP8zNEntry points exposed by the `kernels-community/finegrained-fp8` Triton kernel.r   ÚmatmulÚbatched_matmulÚgrouped_matmulr   N)r*   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__r   r   r   r1   r1   G   s   ‡ áXàÓØÓØ×r   r1   zFineGrainedFP8 | NoneÚ_FINEGRAINED_FP8c               ó   € V ^8„  d   QhRR/# )r   r   ÚNoner   )r   s   "r   r   r   W   s   € ÷ 3ñ 3 dñ 3r   c                 ó¨  € \         e   R# \        4       '       g   \        R\         24      h\	        R4      p V f   \        R4      h\        V RR4      p\        V RR4      p\        V RR4      pRV3RV3RV33 UUu. uF  w  rEVe   K  VNK  	  pppV'       d%   \        RR	P                  V4       R
\         24      h\        VVVR7      s R# u uppi )a¿  
Load the finegrained-fp8 Triton kernel once into the `_FINEGRAINED_FP8` module global.

`@allow_in_graph` makes `torch.compile` treat the untraceable hub download + dynamic import as a
single opaque node instead of tracing into it; it returns `None` (proxyable) and populates the
global, which `load_finegrained_fp8_kernel` then returns.

Under NO circumstances may this function return a value: an `@allow_in_graph` fx node's
return must be proxyable, and returning the bundle (e.g. from the warm-cache
short-circuit) breaks torch.compile with `Unsupported: torch.* op returned non-Tensor`.

Raises `ImportError` if the `kernels` package is missing, or the kernel or required
symbols cannot be found.
Nz$finegrained-fp8 kernel unavailable: zfinegrained-fp8u‰   Failed to load the finegrained-fp8 kernel â€” check that `kernels-community/finegrained-fp8` has a build matching the current torch/CUDA.Ú	matmul_2dÚmatmul_batchedÚmatmul_groupedz4finegrained-fp8 kernel is missing required symbols: ú, z. )r2   r3   r4   )r;   r   ÚImportErrorr   r   r'   Újoinr1   )Úkernelr2   r3   r4   r-   ÚattrÚmissings          r   Ú_load_finegrained_fp8_kernelrH   V   s  € ô" Ò#Ùä×!Ò!ÜÐ@ÔAYÐ@ZÐ[Ó\Ð\äÐ/Ó0€FØ‚~Üð;ó
ð 	
ô
 �V˜[¨$Ó/€FÜ˜VÐ%5°tÓ<€NÜ˜VÐ%5°tÓ<€Nð
 ˜&Ð!Ø˜~Ð.Ø˜~Ð.ñ
ôñ
‰JˆDð
 ÷ 	ˆñ
ð ñ ÷ ÜØBÀ4Ç9Á9ÈWÓCUÐBVÐVXÔYqÐXrÐsó
ð 	
ô &ØØ%Ø%ôÒùós   Á<CÂ	Cc               ó   € V ^8„  d   QhRR/# )r   r   r1   r   )r   s   "r   r   r   �   s   € ÷ ñ  ^ñ r   c                 ó"   € \        4        \        # ©N)rH   r;   r   r   r   Úload_finegrained_fp8_kernelrL   �   s   € Ü Ô"ÜÐr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   ÚaÚintÚbr   r   )r   s   "r   r   r   ’   s!   € ÷ ñ ˆSð �Sð ˜Sñ r   c                ó.   € W,           ^,
          V,          # )zCeiling division.r   )rN   rP   s   &&r   Ú_cdivrR   ’   s   € à�E�A�I˜!ÕÐr   c               ó@   € V ^8„  d   QhRRRRRRRRRRRRR	R
RR
RRRR/
# )r   Únum_expertsrO   Úproj_outÚproj_inÚweight_dtyper   Úsf_dtypeÚweight_k_divÚ	sf_gran_nú
int | NoneÚ	sf_gran_kÚ
min_sf_outr   z!tuple[nn.Parameter, nn.Parameter]r   )r   s   "r   r   r   —   sl   € ÷ ñ Øðàðð ðð ð	ð
 ðð ðð ðð ðð ðð 'ñr   c	                ób  € \         P                  ! WW%,          VR7      p	\        P                  ! W™P	                  4       R7      p
\        Ve   \        W4      M^V4      pVe   \        W'4      M^p\         P                  ! WWÄR7      p\        P                  ! WÝP	                  4       R7      pW®3# )u¿  Allocate `(weight, weight_scale_inv)` parameters for one expert projection.

`weight_k_div` halves the K dim for FP4-packed storage (2 e2m1 values per byte).
`sf_gran_n` / `sf_gran_k` set per-block (None â†’ per-row/per-tensor) SF granularity.
`min_sf_out` floors the SF tensor's output dim â€” used by the fused gate_up
projection to keep room for both halves (pass `2`) even when `proj_out < sf_gran_n`
would otherwise collapse the SF dim to 1.
©Údtype©Úrequires_grad)r"   ÚemptyÚnnÚ	ParameterÚis_floating_pointÚmaxrR   )rT   rU   rV   rW   rX   rY   rZ   r\   r]   Úweight_tÚweightÚsf_outÚsf_inÚsf_tÚsfs   &&&&&&&&&      r   Ú_alloc_expert_projrn   —   s‰   € ô& �{Š{˜;°'Õ2IÐQ]Ô^€HÜ�\Š\˜(×2LÑ2LÓ2NÔO€FÜ¨yÒ/D”�xÔ+È!ÈZÓX€FØ)2Ò)>ŒE�'Ô%ÀA€EÜ�;Š;�{¨EÔB€DÜ	�Š�d×*@Ñ*@Ó*BÔ	C€BØˆ:Ðr   c               ó4   € V ^8„  d   QhRRRRRRRRRRR	RR
R/# )r   Úinputútorch.Tensorri   Úweight_scale_invÚ
block_sizeúlist[int] | NoneÚbiasútorch.Tensor | NoneÚactivation_scaler   r   )r   s   "r   r   r   ³   sN   € ÷ ñ Øðàðð #ðð !ð	ð
 ðð *ðð ñr   c           	     ó†   € \        4       pVP                  V VVVV P                  VR7      pVe   VP                  V4       V# )ua  Triton FP8/FP4 linear: fused act-quant + matmul, then optional bias add.

``activation_scale=None`` â†’ dynamic per-K-block scales (inline); set it for
static per-tensor quant. ``weight_scale_inv`` accepts fp32 or UE8M0; the
dispatcher routes FP4 (``int8``-packed) weights automatically. ``output_dtype``
defaults to ``input.dtype`` when left ``None``.
©rw   )rL   r2   r`   Úadd_)rp   ri   rr   rs   ru   rw   Úfinegrained_fp8Úoutputs   &&&&&&  r   Úfinegrained_fp8_linearr}   ³   sP   € ô 2Ó3€OØ×#Ñ#ØØØØØ�‰Ø)ð $ó €Fð ÒØ�‰�DÔØ€Mr   c               ó8   € V ^8„  d   QhRRRRRRRRRRR	RR
RRR/# )r   rp   rq   ri   rr   rs   rt   ru   rv   rw   Úallow_deepgemmÚboolr   r   )r   s   "r   r   r   Ð   sl   € ÷ Jgñ JgØðJgàðJgð #ðJgð !ð	Jgð
 ðJgð *ðJgð ðJgð ñJgr   c           	     ó�  € T;'       dÛ    \        4       ;'       dÉ    VRJ ;'       d½    \        4       ;'       d    VP                  \        P                  8H  '       * ;'       d�    VP                  \        P
                  8H  ;'       g9    VRJ;'       dP    V^ ,          V^,          u;8H  ;'       d    ^€8H  Mu ;'       d$    \        P                  P                  RR4      R8g  pV'       d    \        V VVVVVR7      # \        WW#WE4      #   \        \        \        3 d$   p\        P                  RT R24        Rp?LARp?ii ; i)uË  End-to-end FP8/FP4 linear used by `FP8Linear` and the eager `FP8Experts` loop.

Dispatch order â€” both backends handle FP8 and FP4 weights with fp32 or UE8M0 scales:
  1. DeepGEMM (`deepgemm_fp8_fp4_linear`) â€” 3-6Ã— faster on the shapes it supports.
     Preferred for FP4, UE8M0 SFs, and 128Ã—128 block FP8.
  2. Triton finegrained-fp8 fallback â€” used when DeepGEMM is unavailable, when the
     caller passes ``activation_scale`` (DeepGEMM is dynamic-only), or for any
     shape DeepGEMM declined.

Args:
    input: (..., K) bf16/fp16 activations.
    weight: (N, K) `float8_e4m3fn` or (N, K // 2) `int8` (FP4-packed).
    weight_scale_inv: per-block weight scales â€” `float32` (V3-style) or `float8_e8m0fnu`
        (V4-style; reinterpreted as int32 at the DeepGEMM kernel boundary).
    block_size: [block_n, block_k] for FP8 block-wise quant, or None/[N, K] for per-tensor.
        Ignored for FP4 weights (the kernel infers SF granularity from the dtype).
    bias: optional bias added to the matmul output.
    activation_scale: pass a per-tensor scalar to use static activation quant; leave `None`
        for dynamic (per-token) quant.
    allow_deepgemm: set ``False`` to force the Triton fallback for this call. Used when the
        model spans multiple CUDA devices in one process â€” DeepGEMM's cached kernels are bound
        to a single CUDA context and produce garbage across devices (see the multi-device guard
        in ``quantizer_finegrained_fp8.py``).
NÚ$TRANSFORMERS_DISABLE_DEEPGEMM_LINEARÚ0Ú1)rs   rw   ru   z=DeepGEMM declined this call, falling back to Triton. Reason: zW Set `TRANSFORMERS_DISABLE_DEEPGEMM_LINEAR=1` to skip DeepGEMM for FP8 linear entirely.)r   r   r`   r"   Úfloat32Úint8ÚosÚenvironÚgetr   rC   ÚNotImplementedErrorÚ
ValueErrorÚloggerÚwarning_oncer}   )	rp   ri   rr   rs   ru   rw   r   Údeepgemm_preferredÚes	   &&&&&&&  r   Ú
fp8_linearr�   Ð   sY  € ðV 	÷ 	Oð 	OÜ Ó"÷	Oð 	Oà Ð$÷	Oð 	Oô “×GÐGÐ 0× 6Ñ 6¼%¿-¹-Ñ GÔH÷	Oð 	Oð �\‰\œUŸZ™ZÑ'×mÐm¨J¸dÐ,B÷		Oð -mÀzÐRSÅ}ÐXbÐcdÕXe×GlÔGlÐilÔGl÷		Oð 	Oô
 �J‰J�N‰NÐAÀ3ÓGÈ3ÑNð ÷ ð	Ü*ØØØ Ø%Ø!1Øôð ô* " %Ð1AÈtÓfÐføô Ô0´*Ð=ô 	ô ×ÑØOÐPQÈsð Sið i÷ñ ûð	ús   Ã.D ÄEÄ"E Å Ec                  óB   a € ] tR tRtRtRR V 3R llltR R ltRtV ;t# )	Ú	FP8Lineari  Fc               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   Úin_featuresrO   Úout_featuresrs   útuple[int, int] | NoneÚactivation_schemeÚstrÚ	scale_fmtÚhas_biasr€   r   )r   s   "r   r   ÚFP8Linear.__annotate__#  sF   € ÷ $2ñ $2àð$2ð ð$2ð +ð	$2ð
 ð$2ð ð$2ð ñ$2r   c                	ó‚  <€ \         S
V `  W4       W`n        W0n        W@n        \
        P                  P                  \
        P                  ! W!\        R 7      4      V n
        V P                  fA   \        P                  ! \
        P                  ! R\
        P                  R 7      4      V n        MÎVR8X  d   \        4       M\
        P                  pW P                  ^ ,          ,           ^,
          V P                  ^ ,          ,          pWP                  ^,          ,           ^,
          V P                  ^,          ,          p	\        P                  ! \
        P                  ! W‰VR 7      VP                  R7      V n        V P                  R8X  dA   \        P                  ! \
        P                  ! R\
        P                  R 7      4      V n        MV P#                  RR4       V P                  '       d<   \        P                  ! \
        P                  ! V P$                  4      4      V n        R# V P#                  RR4       R# )r_   Nç      ð?Úue8m0ra   Ústaticrw   ru   )ÚsuperÚ__init__rš   rs   r—   r"   rd   re   rc   Ú
_FP8_DTYPEri   Útensorr…   rr   r%   rf   rw   Úregister_parameterr•   ru   )Úselfr”   r•   rs   r—   r™   rš   rX   Úscale_out_featuresÚscale_in_featuresÚ	__class__s   &&&&&&&   €r   r¡   ÚFP8Linear.__init__#  sr  ø€ ô 	‰Ñ˜Ô3à ŒØ$ŒØ!2ÔÜ—h‘h×(Ñ(¬¯ª°\ÔV`Ô)aÓbˆŒà�?‰?Ò"ä$&§L¢L´·²¸cÌÏÉÔ1WÓ$XˆDÕ!à-6¸'Ô-AÔ'Ô)ÄuÇ}Á}ˆHØ".·±ÀÕ1CÕ"CÀaÕ"GÈDÏOÉOÐ\]ÕL^Õ!^ÐØ!,¯©¸qÕ/AÕ!AÀAÕ!EÈ$Ï/É/ÐZ[ÕJ\Õ \ÐÜ$&§L¢LÜ—’Ð.ÈÔRØ&×8Ñ8ô%ˆDÔ!ð
 ×!Ñ! XÔ-Ü$&§L¢L´·²¸cÌÏÉÔ1WÓ$XˆDÕ!à×#Ñ#Ð$6¸Ô=à�=�=ˆ=ÜŸš¤U§[¢[°×1BÑ1BÓ%CÓDˆDŽIà×#Ñ# F¨DÖ1r   c               ó    € V ^8„  d   QhRRRR/# ©r   rp   rq   r   r   )r   s   "r   r   r›   I  s   € ÷ 
ñ 
˜\ð 
¨lñ 
r   c           
     	ó<  € V P                   P                  4       ^8”  d,   \        P                  ! WP                   V P                  4      # \        VV P                   V P                  V P                  V P                  V P                  V P                  '       * R7      # )é   )rs   rw   ru   r   )
ri   Úelement_sizeÚFÚlinearru   r�   rr   rs   rw   Ú_deepgemm_disabled)r¥   rp   s   &&r   ÚforwardÚFP8Linear.forwardI  ss   € Ø�;‰;×#Ñ#Ó%¨Ô)Ü—8’8˜E§;¡;°·	±	Ó:Ð:äØØ�K‰KØ×!Ñ!Ø—‘Ø!×2Ñ2Ø—‘Ø#×6Ñ6Ô6ô
ð 	
r   )rw   r—   ru   rs   rš   ri   rr   ©NÚdynamicÚfloatF)	r*   r5   r6   r7   r±   r¡   r²   r:   Ú__classcell__©r¨   s   @r   r’   r’     s    ø† ð Ð÷$2ò $2÷L
ó 
r   r’   c                  óH   a € ] tR tRtRtRR V 3R llltR V 3R lltRtV ;t# )	ÚFP8EmbeddingiX  a^  `nn.Embedding` whose table is stored in FP8 with a single per-tensor `weight_scale`.

Qwen4-Exp's hashed n-gram table is ~48 GiB in FP8, too big to dequantize at load time, so the
rescale runs on the rows a lookup gathers instead. Build it on the `meta` device: like
`FP8Linear`, `__init__` lets `nn.Embedding` allocate a full-precision table first.
c               ó$   € V ^8„  d   QhRRRRRR/# )r   Únum_embeddingsrO   Úembedding_dimÚpadding_idxr[   r   )r   s   "r   r   ÚFP8Embedding.__annotate__`  s-   € ÷ 	Mñ 	Màð	Mð ð	Mð  ñ		Mr   c                	óú   <€ \         SV `  WVR 7       \        P                  ! \        P
                  ! W\        R7      RR7      V n        \        P                  ! \        P                  ! ^4      RR7      V n	        R# ))r¾   r_   Fra   N)
r    r¡   rd   re   r"   rc   r¢   ri   ÚonesÚweight_scale)r¥   r¼   r½   r¾   r¨   s   &&&&€r   r¡   ÚFP8Embedding.__init__`  sQ   ø€ ô 	‰Ñ˜ÀKÐÔPÜ—l’l¤5§;¢;¨~ÔT^Ô#_ÐotÔuˆŒäŸLšL¬¯ª°A«ÀeÔLˆÖr   c               ó    € V ^8„  d   QhRRRR/# r«   r   )r   s   "r   r   r¿   k  s   € ÷ Dñ D˜\ð D¨lñ Dr   c                	óŽ   <€ \         SV `  V4      pVP                  V P                  P                  4      V P                  ,          # rK   )r    r²   ÚtorÂ   r`   )r¥   rp   Úrowsr¨   s   && €r   r²   ÚFP8Embedding.forwardk  s7   ø€ Ü‰w‰˜uÓ%ˆØ�w‰w�t×(Ñ(×.Ñ.Ó/°$×2CÑ2CÕCÐCr   )ri   rÂ   rK   ©	r*   r5   r6   r7   r8   r¡   r²   r:   r·   r¸   s   @r   rº   rº   X  s    ø† ñ÷	Mò 	M÷Dö Dr   rº   c                  óB   a € ] tR tRtRtRR V 3R llltR R ltRtV ;t# )	ÚFP8GroupedLinearip  uÓ  FP8 drop-in for block-diagonal grouped linears.

The underlying nn.Linear stores a single `(n_groups * out_per_group, in_per_group)`
weight; logically that's `n_groups` independent `(out_per_group, in_per_group)`
sub-matrices, each consuming a disjoint slice of the input's last-but-one dim.
Forward expects input of shape `(..., n_groups, in_per_group)` and returns
`(..., n_groups, out_per_group)` â€” same contract as the vanilla bf16 grouped
linear it replaces.

c               ó4   € V ^8„  d   QhRRRRRRRRRRR	RR
R/# )r   Úin_features_per_grouprO   r•   Ún_groupsrs   r–   r—   r˜   r™   rš   r€   r   )r   s   "r   r   ÚFP8GroupedLinear.__annotate__|  sP   € ÷ !ñ !à"ð!ð ð!ð ð	!ð
 +ð!ð ð!ð ð!ð ñ!r   c           	     	ó>   <€ \         SV `  VVVVVVR 7       W0n        R# )©r”   r•   rs   r—   r™   rš   N)r    r¡   rÎ   )	r¥   rÍ   r•   rÎ   rs   r—   r™   rš   r¨   s	   &&&&&&&&€r   r¡   ÚFP8GroupedLinear.__init__|  s0   ø€ ô 	‰ÑØ-Ø%Ø!Ø/ØØð 	ô 	
ð !Žr   c               ó    € V ^8„  d   QhRRRR/# )r   Úxrq   r   r   )r   s   "r   r   rÏ   �  s   € ÷ $ñ $˜ð $¨,ñ $r   c           	     	óJ  € VP                   R R pVP                   R,          pV P                  P                  4       ^8”  dõ   V P                  P                  V P                  RV4      P                  ^^4      pVP                  RV P                  V4      P                  ^ ^4      p\        P                  ! W4      P                  ^ ^4      pVP                  ! . VOV P                  NRN5!  pV P                  '       d6   VP                  V P                  P                  V P                  R4      4       V# V P                  pV P                  pVP                  V P                  RV4      pVP                  R^ 4      P                  RV4      pVP                  V P                  VP                  ^ 4      V P                  ,          VP                  ^4      4      pVP                  ^ 4      V P                  ,          p\        P                  ! V P                  3WqP                   \        P"                  R7      p\        P$                  ! ^V P                  ^,           VP                   \        P"                  R7      V,          p	\'        4       p
V
P)                  VVVV	VV P*                  R7      pVP                  ! V P                  .VORN5!  P                  ^ R4      pV P                  '       d6   VP                  V P                  P                  V P                  R4      4       V# )N)Údevicer`   ©ÚoffsetsÚtokens_per_expertrs   éþÿÿÿéÿÿÿÿ)Úshaperi   r®   ÚviewrÎ   Ú	transposeÚreshaper"   Úbmmrš   rz   ru   rr   ÚmovedimÚsizeÚfullrÖ   Úint32ÚarangerL   r4   rs   )r¥   rÔ   Úinput_shapeÚ
hidden_dimÚwÚyÚ	scale_invÚtokens_per_grouprÙ   rØ   r{   s   &&         r   r²   ÚFP8GroupedLinear.forward�  sV  € Ø—g‘g˜c˜r�lˆØ—W‘W˜R•[ˆ
à�;‰;×#Ñ#Ó%¨Ô)Ø—‘× Ñ  §¡°°JÓ?×IÑIÈ!ÈQÓOˆAØ—	‘	˜"˜dŸm™m¨ZÓ8×BÑBÀ1ÀaÓHˆAÜ—	’	˜!“×)Ñ)¨!¨QÓ/ˆAØ—	’	Ð:˜;Ð:¨¯©Ð:°rÓ:ˆAØ�}�}ˆ}Ø—‘�t—y‘y—~‘~ d§m¡m°RÓ8Ô9ØˆHà�K‰KˆØ×)Ñ)ˆ	à�F‰F�4—=‘= " jÓ1ˆØ�I‰I�b˜!Ó×$Ñ$ R¨Ó4ˆØ—N‘N 4§=¡=°)·.±.ÀÓ2CÀtÇ}Á}Õ2TÐV_×VdÑVdÐefÓVgÓhˆ	àŸ6™6 !›9¨¯©Õ5ÐÜ!ŸJšJ¨¯©Ð'7Ð9I×RZÑRZÔbg×bmÑbmÔnÐÜ—,’,˜q $§-¡-°!Õ"3¸A¿H¹HÌEÏKÉKÔXÐ[kÕkˆä5Ó7ˆØ×*Ñ*ØØØØØ/Ø—‘ð +ó 
ˆð �IŠI�d—m‘mÐ6 kÐ6°2Ó6×>Ñ>¸qÀ"ÓEˆØ�=�=ˆ=Ø�F‰F�4—9‘9—>‘> $§-¡-°Ó4Ô5Øˆr   )rÎ   r´   rÉ   r¸   s   @r   rË   rË   p  s   ø† ñ	÷!ò !÷($ó $r   rË   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# ©r   r¥   ztorch.nn.ModuleÚhidden_statesrq   Útop_k_indexÚtop_k_weightsr   r   )r   s   "r   r   r   ·  sA   € ÷ G7ñ G7Ø
ðG7àðG7ð ðG7ð  ð	G7ð
 ñG7r   c                ó  € V P                   R 8X  d   \        R4      h\        4       pVP                  R4      pVP                  ^ 4      pVP                  R4      pVP	                  V^ R7      pVP                  R4      p	VP                  R4      p
W P                  8¬  P                  R4      pV P                  '       d   V P                  MV P                  pV P                  '       d   V P                  MV P                  pV P                  pV P                  pVP                  VVVV P                   V
R7      pV P                  '       d   V P#                  V4      pMV P%                  V4      pVP                  VVVV P                   V
R7      pVV	P'                  VP(                  4      P                  R4      ,          pVP+                  VR4       VP-                  WeV4      P/                  ^R7      pVP'                  VP(                  4      # )rŸ   z‘batched_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.©Údim)rs   Ú
expert_idsç        rÛ   )r—   rŠ   rL   râ   Úrepeat_interleaverß   rT   Ú	unsqueezeÚhas_gateÚgate_up_projÚup_projÚgate_up_proj_scale_invÚup_proj_scale_invÚ	down_projÚdown_proj_scale_invr3   rs   Ú_apply_gateÚact_fnrÆ   r`   Úmasked_fill_rÝ   Úsum)r¥   rï   rð   rñ   r{   Ú	num_top_kÚ
num_tokensrç   Úselected_hidden_statesÚsample_weightsrõ   Úsentinel_maskÚ	weight_upÚweight_scale_upÚweight_downÚweight_scale_downrU   Úweighted_outÚfinal_hidden_statess   &&&&               r   Úfp8_batched_mm_experts_forwardr  ·  sí  € ð ×Ñ Ô)Ü!ðWó
ð 	
ô
 2Ó3€Oà× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jð +×<Ñ<¸YÈAÐ<ÓNÐØ"×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jð
  ×#3Ñ#3Ñ3×>Ñ>¸rÓB€Mà%)§]§] ]�×!Ò!¸¿¹€IØ59·]·]°]�d×1Ò1È×H^ÑH^€OØ—.‘.€KØ×0Ñ0Ðð ×-Ñ-ØØØØ—?‘?Øð .ó €Hð ‡}‡}€}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆð ×-Ñ-ØØØØ—?‘?Øð .ó €Hð ˜n×/Ñ/°·±Ó?×IÑIÈ"ÓMÕM€Lð ×Ñ˜m¨SÔ1ð '×+Ñ+¨JÀ:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# rî   r   )r   s   "r   r   r     sA   € ÷ [7ñ [7Ø
ð[7àð[7ð ð[7ð  ð	[7ð
 ñ[7r   c           	     ó  € V P                   R 8X  d   \        R4      h\        4       pVP                  pVP	                  R	4      pVP	                  ^ 4      pVP	                  R	4      pVP                  R	4      p	VP                  R	4      p
\        P                  ! V
4      w  r¼WV,          ,          pWœ,          pVP                  R8X  d   VP                  4       MVP                  4       p\        P                  ! WðP                  ^ V P                  ^,
          R7      p\        P                  ! V^ \        P                  R7      pW°P                  8¬  P                  R	4      pV P                   '       d   V P"                  MV P$                  pV P                   '       d   V P&                  MV P(                  pV P*                  pV P,                  pVP/                  VVVVVV P0                  R7      pV P                   '       d   V P3                  V4      pMV P5                  V4      pVP/                  VVVVVV P0                  R7      pVVP7                  VP8                  4      P                  R	4      ,          pVP;                  VR4       \        P<                  ! V4      p\        P>                  ! VP	                  ^ 4      VR7      VV&   VV,          pVPA                  WvV4      PC                  ^R7      pVP7                  VP8                  4      # )
rŸ   z‘grouped_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.Úcpu)ÚbinsÚminrg   )rô   r`   r×   rö   )rÖ   ró   rÛ   )"r—   rŠ   rL   rÖ   râ   rß   r"   Úsortr)   r¶   rO   ÚhistcrT   Úcumsumrä   rø   rù   rú   rû   rü   rý   rþ   rÿ   r4   rs   r   r  rÆ   r`   r  Ú
empty_likerå   rÝ   r  )r¥   rï   rð   rñ   r{   rÖ   r  r  rç   r  rõ   Úexpert_ids_gÚpermÚselected_hidden_states_gÚsample_weights_gÚhistc_inputrÙ   rØ   r  r	  r
  r  r  rU   r  Úinv_permr  s   &&&&                       r   Úfp8_grouped_mm_experts_forwardr    s®  € ð ×Ñ Ô)Ü!ðWó
ð 	
ô
 2Ó3€Oà×!Ñ!€FØ× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jð #×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jô Ÿš JÓ/Ñ€LØ,°YÕ->Õ?ÐØ%Õ+Ðð
 +1¯+©+¸Ô*>�,×$Ñ$Ô&ÀL×DTÑDTÓDV€KÜŸš K×6FÑ6FÈAÐSW×ScÑScÐfgÕSgÔhÐÜ�lŠlÐ,°!¼5¿;¹;ÔG€Gð "×%5Ñ%5Ñ5×@Ñ@ÀÓD€Mà%)§]§] ]�×!Ò!¸¿¹€IØ59·]·]°]�d×1Ò1È×H^ÑH^€OØ—.‘.€KØ×0Ñ0Ðð ×-Ñ-Ø ØØØØ+Ø—?‘?ð .ó €Hð ‡}‡}€}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆð ×-Ñ-ØØØØØ+Ø—?‘?ð .ó €Hð Ð.×1Ñ1°(·.±.ÓA×KÑKÈBÓOÕO€Lð ×Ñ˜m¨SÔ1ô ×Ò Ó%€HÜ—\’\ $§)¡)¨A£,°vÔ>€HˆT�NØ Õ)€Lð '×+Ñ+¨JÀ:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   c                  óz   a € ] tR tRt$ RtRRRRR//tR]R	&   RR
 V 3R llltR R ltR R lt	RR R llt
RtV ;t# )Ú
FP8Expertsi_  FÚdeepgemm_megamoeÚmoe_tp_expertsÚmegamoe_expertsÚ	ep_routerÚmegamoe_routerzdict[str, dict[str, str]]Ú_impl_tp_layer_overridesc          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )	r   rs   r–   r—   r˜   r™   rš   r€   rù   r   )r   s   "r   r   ÚFP8Experts.__annotate__s  sL   € ÷ Dnñ Dnð +ðDnð ð	Dnð
 ðDnð ðDnð ñDnr   c           
     	óÆ  <€ \         S
V `  4        VR J g   Q R4       hWn        WPn        W`n        W n        VP                  V n        W0n        \        VRR4      V n
        \        VRR4      V n        \        VRR4      V n        \        VRR4      V n        \        \        VR	R
4      ,          V n        \        VRR4      V n        \        VRR4      R8H  pVR8X  d   \%        4       M\&        P(                  pV'       d   R\&        P*                  RVR^R^R^ /p	M(R\,        RTRVe
   V^ ,          MRRVe
   V^,          MR/p	V P                  '       dV   \/        V P                  ^V P                  ,          V P                  3R^/V	B w  V n        V n        V P5                  RR4       MK\/        V P                  V P                  V P                  3/ V	B w  V n        V n        V P5                  RR4       \/        V P                  V P                  V P                  3/ V	B w  V n        V n        V P5                  RR4       V P                  R8X  d•   \>        P@                  ! \&        PB                  ! V P                  \&        P(                  R7      4      V n"        \>        P@                  ! \&        PB                  ! V P                  \&        P(                  R7      4      V n#        R# R# )FzWFP8Experts does not support bias for now, please open an issue if you want this featureÚnum_local_expertsrT   Úmoe_intermediate_sizeÚintermediate_sizeÚswiglu_alphaNÚswiglu_limitÚhidden_activationÚ
hidden_actÚexpert_dtypeÚfp8Úfp4rž   rW   rX   rY   rZ   r\   r]   Úgate_up_proj_biasÚup_proj_biasÚdown_proj_biasrŸ   r_   )$r    r¡   Úconfigrš   rù   rs   Úhidden_sizerç   r—   r.   rT   Úintermediate_dimr'   r.  r/  r   r  Úlimitr%   r"   r…   r†   r¢   rn   rú   rü   r¤   rû   rý   rþ   rÿ   rd   re   rÁ   Úgate_up_proj_activation_scaleÚdown_proj_activation_scale)r¥   r8  rs   r—   r™   rš   rù   Úis_fp4rX   Úalloc_kwargsr¨   s   &&&&&&&   €r   r¡   ÚFP8Experts.__init__s  s’  ø€ ô 	‰ÑÔà˜5Ó ð 	
Øeó	
Ð ð ŒØ ŒØ ŒØ$ŒØ ×,Ñ,ˆŒØ!2ÔÜ& vÐ/BÀMÓRˆÔÜ +¨FÐ4KÐM`Ó aˆÔÜ# F¨N¸DÓAˆÔÜ# F¨N¸DÓAˆÔÜœ[¨Ð1DÀlÓSÕTˆŒÜ˜V ^°TÓ:ˆŒ
ô ˜ °Ó7¸5Ñ@ˆØ)2°gÔ)=Ô#Ô%Ä5Ç=Á=ˆßà¤§
¡
Ø˜HØ Ø˜QØ˜Rð‰Lð ¤
Ø˜HØ¨jÒ.D˜Z¨ž]È$Ø¨jÒ.D˜Z¨ž]È$ð	ˆLð �=�=ˆ=Ü=OØ× Ñ  ! d×&;Ñ&;Õ";¸T¿_¹_ñ>ØYZð>Ø^jñ>Ñ:ˆDÔ˜tÔ:ð ×#Ñ#Ð$7¸Õ>ä3EØ× Ñ  $×"7Ñ"7¸¿¹ñ4ØLXñ4Ñ0ˆDŒL˜$Ô0ð ×#Ñ# N°DÔ9ä3EØ×Ñ˜dŸo™o¨t×/DÑ/Dñ4
ØHTñ4
Ñ0ˆŒ˜Ô0ð 	×ÑÐ 0°$Ô7à×!Ñ! XÔ-Ü13·²¼e¿jºjÈ×IYÑIYÔaf×anÑanÔ>oÓ1pˆDÔ.Ü.0¯lªl¼5¿:º:Àd×FVÑFVÔ^c×^kÑ^kÔ;lÓ.mˆDÖ+ñ .r   c               ó    € V ^8„  d   QhRRRR/# )r   Úgate_uprq   r   r   )r   s   "r   r   r)  ¹  s   € ÷ &ñ & <ð &°Lñ &r   c                	ó  € VP                  ^RR7      w  r#V P                  e‚   VP                  V P                  R7      pVP                  V P                  ) V P                  R7      pV\        P
                  ! W P                  ,          4      ,          pVR,           V,          # V P                  eE   VP                  V P                  R7      pVP                  V P                  ) V P                  R7      pV P                  V4      V,          # )r   ró   )rg   ©r  rg   r�   rÛ   )Úchunkr.  Úclampr/  r"   Úsigmoidr;  r  )r¥   rB  ÚgateÚupÚglus   &&   r   r   ÚFP8Experts._apply_gate¹  sÒ   € Ø—=‘= ¨�=Ó+‰ˆØ×ÑÒ(à—:‘: $×"3Ñ"3�:Ó4ˆDØ—‘˜t×0Ñ0Ð0°d×6GÑ6G�ÓHˆBØœŸš t×.?Ñ.?Õ'?Ó@Õ@ˆCØ˜•H Õ#Ð#Ø�Z‰ZÒ#Ø—:‘: $§*¡*�:Ó-ˆDØ—‘˜tŸz™z˜k¨t¯z©z�Ó:ˆBØ�{‰{˜4Ó  2Õ%Ð%r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   rï   rq   rð   rñ   r   r   )r   s   "r   r   r)  Æ  s,   € ÷ (;ñ (;Ø)ð(;Ø8Dð(;ØUað(;à	ñ(;r   c                	óÆ  € \         P                  ! V\         P                  R 7      p\         P                  ! 4       ;_uu_ 4        \         P                  P
                  P                  W P                  ^,           R7      pVP                  ^^^ 4      p\         P                  ! VP                  R	R7      ^ 4      P                  RR7      P                  R4      pRRR4       X EFÌ  pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
V P                  R8X  d   V P                  V,          MRpT P!                  T
V P"                  '       d   V P$                  V,          MV P&                  V,          V P"                  '       d   V P(                  V,          MV P*                  V,          VR7      pV P"                  '       d   V P-                  V4      MV P/                  V4      pV P                  R8X  d   V P0                  V,          MRpV P!                  VV P2                  V,          V P4                  V,          VR7      pW9VR3,          pWÎP7                  VP8                  4      ,          pVP;                  ^ WŸP7                  VP8                  4      4       EKÏ  	  VP7                  VP8                  4      #   + '       g   i     ELÿ; i)
r_   )Únum_classesró   F)Úas_tupleNrŸ   ry   rÛ   )rÛ   rÚ   )r"   Ú
zeros_liker…   Úno_gradrd   r   Úone_hotrT   ÚpermuteÚgreaterr  ÚnonzerorÝ   Úwherer—   r<  r°   rù   rú   rû   rü   rý   r   r  r=  rþ   rÿ   rÆ   r`   Ú
index_add_)r¥   rï   rð   rñ   r  Úexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgate_up_act_scalerU   Údown_act_scaleÚrouting_weightsr  s   &&&&            r   r²   ÚFP8Experts.forwardÆ  sG  € ô
 $×.Ò.¨}ÄEÇMÁMÔRÐä�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_ÐbcÕOcÐ5ÓdˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÐZ_ÐPÓ`×eÑeÐfhÓiˆJ÷ ô
 %ˆJØ×-Ñ-Ô-Ùä#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMàBF×BXÑBXÐ\dÔBd�×2Ñ2°:Ö>Ðjnð ð —{‘{ØØ15··°�×!Ñ! *Ö-ÀDÇLÁLÐQ[ÕD\Ø;?¿=¿=¸=�×+Ñ+¨JÖ7Èd×NdÑNdÐeoÕNpØ!2ð	 #ó ˆHð 6:·]·]°]�t×'Ñ'¨Ô1ÈÏÉÐT\ÓH]ˆHà?C×?UÑ?UÐYaÔ?a�×/Ñ/°
Ö;Ðgkð ð —{‘{ØØ—‘˜zÕ*Ø×(Ñ(¨Õ4Ø!/ð	 #ó ˆHð ,°yÀ$Ð,FÕGˆOØ#×&8Ñ&8¸¿¹Ó&HÕHˆLØ×*Ñ*¨1¨i¿¹ÐI\×IbÑIbÓ9c×dñ7 %ð8 #×%Ñ% m×&9Ñ&9Ó:Ð:÷C �_ˆ_ús   ÁBKËK 	c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   rp   rq   ri   rr   rw   rv   r   r   )r   s   "r   r   r)  ð  s<   € ÷ 
ñ 
àð
ð ð
ð 'ð	
ð
 .ð
ð 
ñ
r   c           	     	ó®   € VP                  4       ^8”  d   \        P                  ! WR4      # \        VVVV P                  VV P
                  '       * R7      # )r­   N)rw   r   )r®   r¯   r°   r�   rs   r±   )r¥   rp   ri   rr   rw   s   &&&&&r   r°   ÚFP8Experts.linearð  sR   € ð ×ÑÓ  1Ô$Ü—8’8˜E¨4Ó0Ð0äØØØØ�O‰OØ-Ø#×6Ñ6Ô6ô
ð 	
r   )r  r—   rs   r8  rþ   r=  rÿ   rú   r<  rü   rš   rù   rç   r:  r;  rT   r.  r/  rû   rý   )Nrµ   r¶   FTrK   )r*   r5   r6   r7   r±   r'  r9   r¡   r   r²   r°   r:   r·   r¸   s   @r   r!  r!  _  sU   ø‡ ð Ðð 	ØÐ/ØÐ)ð
ð;ÐÐ7ó ÷Dnò DnõL&õ(;÷T
õ 
r   r!  c                  ó.   € ] tR tRtRtR]R]R]R]/t	Rt
R# )	ÚFP8ExpertsInterfacei  z?Interface for registering custom FP8 experts forward functions.Ú
batched_mmÚ
grouped_mmÚdeepgemmr"  r   N)r*   r5   r6   r7   r8   r  r  r   r   Ú_global_mappingr:   r   r   r   rf  rf    s&   † ÙIð 	Ð4ØÐ4ØÐ4ØÐDð	„Or   rf  c               ó    € V ^8„  d   QhRRRR/# )r   Úmodelz	nn.Moduler   r=   r   )r   s   "r   r   r     s   € ÷ ñ ¨Yð ¸4ñ r   c                óâ  € V P                  4        Uu. uF#  p\        V\        \        34      '       g   K!  VNK%  	  pp\	        4       pV Fe  p\        VP                  4       R4      pVf   K#  VP                  P                  R8X  g   K@  VP                  VP                  P                  4       Kg  	  \        V4      ^8:  d   R# V F
  pRVn        K  	  \        P                  R4       R# u upi )a  Internal, temporary helper (not public API): flag every FP8 module to skip DeepGEMM when the
model spans >1 CUDA device in one process.

DeepGEMM loads each kernel via `cuKernelGetFunction`, which binds the `CUfunction` handle to the
CUDA context live at load time; driving that cached handle from another device launches it against
the wrong context and produces garbage. (Build-time fix: compile DeepGEMM with
`DG_JIT_USE_RUNTIME_API=1` for a context-free `cudaKernel_t` loader; until our wheel picks that up
we avoid single-process multi-device.) Setting `_deepgemm_disabled` routes both the linear and
experts paths through Triton/grouped_mm. A model that fits on one device keeps DeepGEMM even with
other GPUs visible; TP/EP put one device per process, so this is a no-op there.
NÚcudaTaB  This FP8 model spans multiple CUDA devices in one process; routing its FP8 linear and experts layers through Triton/grouped_mm instead of DeepGEMM (DeepGEMM's cached kernels are bound to a single CUDA context and corrupt across devices). Run tensor/expert parallel (one device per process) to use the faster DeepGEMM path.)ÚmodulesÚ
isinstancer’   r!  ÚsetÚnextÚ
parametersrÖ   r)   ÚaddÚindexÚlenr±   rŒ   r�   )rl  ÚmÚfp8_modulesÚcuda_devicesÚparams   &    r   Ú!_disable_deepgemm_on_multi_devicer{    s½   € ð $Ÿm™mœoÓX™o˜´¸AÄ	Ì:Ð?V×1W—1�1™o€KÐXÜ“5€LÛˆÜ�Q—\‘\“^ TÓ*ˆØÔ §¡×!2Ñ!2°fÖ!<Ø×Ñ˜UŸ\™\×/Ñ/Ö0ñ ô ˆ<Ó˜AÔÙÛˆØ#ˆÖñ ä
×Ñð	4öùò Ys
   “C,¶C,c               ó    € V ^8„  d   QhRRRR/# )r   Úpatternsz	list[str]Úmodules_to_not_convertúlist[str] | Noner   )r   s   "r   r   r   0  s   € ÷ ñ °	ð ÐScñ r   c                ó4  a€ \        V P                  4       4       Fæ  w  op\        V4      \        P                  J g   K%  \
        ;QJ d    V3R lV 4       F  '       g   K   RM	  RM! V3R lV 4       4      '       g   Kg  \        SV4      '       g   K{  \        P                  ! R4      ;_uu_ 4        \        VP                  VP                  VP                  4      pRRR4       RXn        V P                  SV4       Kè  	  V #   + '       g   i     L/; i)zµSwap every `nn.Embedding` whose name ends with one of `patterns` for `FP8Embedding`.

Also runs under `dequantize=True`: a table worth quantizing is one we cannot afford to expand.
c              3  óF   <"  € T F  pSP                  V4      x € K  	  R # 5irK   )Úendswith)Ú.0Úpr-   s   & €r   Ú	<genexpr>Ú-replace_with_fp8_embedding.<locals>.<genexpr>6  s   øé € Ð/SÉ(ÀQ°·±¸a×0@Ð0@Ë(ùs   ƒ!TFÚmetaN)ÚlistÚnamed_modulesr)   rd   Ú	EmbeddingÚanyr
   r"   rÖ   rº   r¼   r½   r¾   Ú_hf_quantized_needs_local_tpÚset_submodule)rl  r}  r~  ÚmoduleÚ
new_moduler-   s   &&&  @r   Úreplace_with_fp8_embeddingr�  0  s¿   ø€ ô
 ˜U×0Ñ0Ó2Ö3‰ˆˆfÜ�‹<œ2Ÿ<™<Õ'¯C«CÔ/SÉ(Ó/S¯C¯CªCÔ/SÉ(Ó/S×,SÔ,SÜ(¨Ð/E×FÒFÙÜ—’˜f×%Õ%Ü)¨&×*?Ñ*?À×AUÑAUÐW]×WiÑWiÓj�
÷ &à6:ˆJÔ3Ø×Ñ  jÖ1ñ 4ð €L÷	 &×%ús   Â4,DÄDc               ó   € V ^8„  d   QhRR/# )r   r~  r  r   )r   s   "r   r   r   @  s   € ÷ Rñ RØ#3ñRr   c                ó¸  € VP                   '       d   V # RpV P                  4        EFü  w  rV\        WQ4      '       g   K  Rp\        P                  ! R4      ;_uu_ 4        VP                  R4      '       d…   \        VRR4      p\        VRR4      p	\        VRV P                  P                  4       4      p
\        \        \        V	VR	7      pV! V
VP                  VP                  VP                  V	VR
7      pEM\        V4      \         P"                  J dQ   \%        VP&                  VP(                  VP                  VP                  VP                  VP*                  RJR7      pM”\-        V\         P"                  4      '       du   R\        V4      P.                  9   d[   \1        VP&                  VP(                  VP2                  VP                  VP                  VP                  VP*                  RJR7      pVe   RVn        V P7                  WW4       RpRRR4       EKÿ  	  V'       g   \8        P;                  R4       V #   + '       g   i     EK2  ; i)a‚  
A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.

Parameters:
    model (`torch.nn.Module`):
        Input model or `torch.nn.Module` as the function is run recursively.
    modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
        Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons.
    quantization_config (`FineGrainedFP8Config`):
        The quantization config object that contains the quantization parameters.
    pre_quantized (`bool`, defaults to `False`):
        Whether the model is pre-quantized or not
FNr‡  z.expertsrù   Trš   r8  )Úexperts_classÚexperts_interfacerš   rù   )r8  rs   r—   r™   rš   rù   rÑ   ÚGroupedLinear)rÍ   r•   rÎ   rs   r—   r™   rš   z�You are loading your model using fp8 but no linear modules were found in your model. Please double check your model architecture.)Ú
dequantizer‰  r
   r"   rÖ   r‚  r'   r8  Úget_text_configr   r!  ÚALL_FP8_EXPERTS_FUNCTIONSÚweight_block_sizer—   r™   r)   rd   ÚLinearr’   r”   r•   ru   rp  r*   rË   rÎ   rŒ  r�  rŒ   Úwarning)rl  r~  Úquantization_configÚpre_quantizedÚhas_been_replacedÚmodule_namerŽ  r�  rù   rš   r8  Ú	new_classs   &&&&        r   Úreplace_with_fp8_linearr¡  @  sþ  € ð" ×%×%Ð%ØˆàÐØ$×2Ñ2×4ÑˆÜ$ [×IÒIÙàˆ
Ü�\Š\˜&×!Õ!Ø×#Ñ# J×/Ò/Ü" 6¨:°tÓ<�Ü" 6¨:°uÓ=�Ü  ¨°5·<±<×3OÑ3OÓ3QÓR�Ü6Ü",Ü&?Ø%Ø%ô	�	ñ 'Ø!Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø%Ø%ô’
ô �f“¤§¡Ó*ä&Ø &× 2Ñ 2Ø!'×!4Ñ!4Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø#Ÿ[™[°Ð4ô‘
ô ˜F¤B§I¡I×.Ò.°?ÄdÈ6Ãl×F[ÑF[Ô3[ô .Ø*0×*<Ñ*<Ø!'×!4Ñ!4Ø#Ÿ_™_Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø#Ÿ[™[°Ð4ô�
ð Ò%à:>�
Ô7Ø×#Ñ# KÔ<Ø$(Ð!÷c "Ò!ñ  5÷p Ü�‰ð<ô	
ð €L÷q "×!Ñ!ús   Á F;IÉIc                  óZ   € ] tR tRtRtR tR R ltR R ltR R	 lt]	R
 R l4       t
RtR# )ÚFp8Quantizei•  zV
A quantization operation that creates two tensors, weight and scale out of a weight.
c                	ó   € Wn         R # rK   ©Úhf_quantizer©r¥   r¦  s   &&r   r¡   ÚFp8Quantize.__init__š  ó   € Ø(Ör   c               ó    € V ^8„  d   QhRRRR/# )r   Úvaluerq   r   ztuple[int, int]r   )r   s   "r   r   ÚFp8Quantize.__annotate__�  s   € ÷ 	!ñ 	!¨ð 	!¸/ñ 	!r   c                	ó‚  € R pV P                   P                  er   \        V P                   P                  \        4      '       d'   V P                   P                  P	                  R4      pM!\        V P                   P                  RR 4      pVf'   VP                  R,          VP                  R,          3p\        V4      # )Nr™  rÚ   rÛ   )r¦  rœ  rp  Údictr‰   r'   rÜ   Útuple)r¥   r«  rs   s   && r   Ú_resolve_block_sizeÚFp8Quantize._resolve_block_size�  s–   € Øˆ
Ø×Ñ×0Ñ0Ò<Ü˜$×+Ñ+×?Ñ?Ä×FÒFØ!×.Ñ.×BÑB×FÑFÐGZÓ[‘
ä$ T×%6Ñ%6×%JÑ%JÐL_ÐaeÓf�
ØÒØŸ+™+ b�/¨5¯;©;°r­?Ð;ˆJÜ�ZÓ Ð r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úkeyr˜   r«  rq   r   údict[str, torch.Tensor]r   )r   s   "r   r   r¬  ¨  s"   € ÷ &7ñ &7 ð &7¨\ð &7Ð>Uñ &7r   c                	ó&  € VP                   ^8  d   W/# V P                  V4      w  r4VP                  R,          VP                  R,          reWS,          ^ 8w  g   Wd,          ^ 8w  d   W/# VP                  RR pWS,          pWd,          p	VP                  p
VP                  \        P
                  4      pVP                  ! . VOVNVNV	NVN5!  pVP                  4       P                  RR7      p\        P                  ! V^ 8„  V\        P                  ! V4      4      p\        V,          p\        P                  ! V^ 8„  V\        P                  ! V4      4      pRV,          P                  \        P
                  4      pV P                  P                  P                  R8X  d»   \        P                  ! R\        P                   ! \        P"                  ! VP%                  \        P&                  ! \        P
                  4      P(                  R7      4      4      4      pVP                  \+        4       4      pRVP                  \        P
                  4      ,          pVP-                  R4      P-                  R4      pVV,          p\        P$                  ! V\.        \        R7      P                  \0        4      pVP                  V
4      pVP3                  R4      '       d!   VP5                  R	^4      ^ ,          R
,           MVR,           pVVVV/# )r   Nró   r�   rž   ç       @)r  rD  ú.weightÚ.ú.weight_scale_invÚ
_scale_invrÚ   rÛ   éýÿÿÿ)r»  rÛ   )Úndimr°  rÜ   rÆ   r"   r…   rß   ÚabsÚamaxrV  Ú	ones_likeÚ_FP8_MAXr¦  rœ  r™   ÚpowÚceilÚlog2rF  ÚfinfoÚtinyr%   rø   Ú_FP8_MINr¢   r‚  Úrsplit)r¥   r³  r«  Úblock_mÚblock_nrÇ   ÚcolsÚleading_shapeÚ
rows_tilesÚ
cols_tilesÚoriginal_shapeÚ
value_fp32ÚreshapedÚmax_absÚsafe_max_absÚscalesÚ
inv_scalesÚscales_broadcastÚscaledÚ	quantizedÚ	scale_keys   &&&                  r   Ú_quantize_oneÚFp8Quantize._quantize_one¨  s\  € ð �:‰:˜Œ>Ø�<ÐØ×3Ñ3°EÓ:ÑˆØ—[‘[ •_ e§k¡k°"¥oˆdØ�>˜QÔ $¥.°AÔ"5Ø�<Ðð Ÿ™ C RÐ(ˆØ•_ˆ
Ø•_ˆ
ØŸ™ˆØ—X‘XœeŸm™mÓ,ˆ
à×%Ò%Ð_ }Ð_°jÐ_À'Ð_È:Ð_ÐW^Ó_ˆà—,‘,“.×%Ñ%¨(Ð%Ó3ˆÜ—{’{ 7¨Q¡;°¼¿ºÈÓ9QÓRˆä˜LÕ(ˆÜ—’˜W q™[¨&´%·/²/À&Ó2IÓJˆØ˜F•l×&Ñ&¤u§}¡}Ó5ˆ
ð ×Ñ×0Ñ0×:Ñ:¸gÔEÜŸš 3¬¯
ª
´5·:²:¸j×>NÑ>NÔSX×S^ÒS^Ô_d×_lÑ_lÓSm×SrÑSrÐ>NÓ>sÓ3tÓ(uÓvˆJØ#Ÿ™Ô'7Ó'9Ó:ˆJØ˜:Ÿ=™=¬¯©Ó7Õ7ˆFà!×+Ñ+¨BÓ/×9Ñ9¸"Ó=ÐØÐ,Õ,ˆÜ—K’K ¬H¼(ÔC×FÑFÄzÓRˆ	Ø×%Ñ% nÓ5ˆ	ØCFÇ<Á<ÐPY×CZÒCZ�C—J‘J˜s AÓ& qÕ)Ð,?Ö?Ð`cÐfrÕ`rˆ	Ø�Y 	¨:Ð6Ð6r   c               ó    € V ^8„  d   QhRRRR/# )r   Ú
input_dictrq   r   r´  r   )r   s   "r   r   r¬  Ð  s   € ÷ ñ  ,ð Ð=Tñ r   c                	ó¾   € / pVP                  4        FF  w  rE\        V\        4      '       d
   V^ ,          MTpVP                  V P	                  WF4      4       KH  	  V# )r   )Úitemsrp  rˆ  ÚupdaterÙ  )r¥   rÜ  ÚkwargsÚresultr³  r«  r£   s   &&,    r   ÚconvertÚFp8Quantize.convertÐ  sS   € ð +-ˆØ$×*Ñ*Ö,‰JˆCÜ!+¨E´4×!8Ò!8�U˜1–X¸eˆFØ�M‰M˜$×,Ñ,¨SÓ9Ö:ñ -ð ˆr   c               ó   € V ^8„  d   QhRR/# ©r   r   r   r   )r   s   "r   r   r¬  Û  s   € ÷ 0ñ 0˜Mñ 0r   c                	ó,   € \        V P                  4      # rK   )ÚFp8Dequantizer¦  ©r¥   s   &r   Ú
reverse_opÚFp8Quantize.reverse_opÚ  s   € ä˜T×.Ñ.Ó/Ð/r   r¥  N)r*   r5   r6   r7   r8   r¡   r°  rÙ  râ  Úpropertyré  r:   r   r   r   r£  r£  •  s0   † ñò)õ	!õ&7õPð ô0ó ô0r   r£  c                  ó~   € ] tR tRtRtR tR R ltRtR R ltRR	 R
 llt	R R lt
RR R llt]R R l4       tRtR# )rç  iß  ux  Dequantize FP8 weights using their per-block ``weight_scale_inv``.

Designed to run as the *first* op in any :class:`WeightConverter` chain when
loading with ``dequantize=True`` â€” :meth:`update_weight_conversions` on the
FP8 quantizer attaches it to each existing model-specific converter so that
per-expert (weight, scale) pairs are folded into full-precision tensors before
the chain's merge / concat ops collapse the per-expert structure.

Pattern semantics
    Input ``input_dict`` carries one entry per source pattern; each value is a
    list of tensors (one per ``*`` match). For every weight pattern that has a
    sibling ``*.weight_scale_inv`` pattern in the dict, this op pairs them up by
    index, dequantizes per-pair, and emits the dequantized list under the
    original *weight* key. Scale entries are dropped from the output so the
    remaining ops only see weights.
c                	ó   € Wn         R # rK   r¥  r§  s   &&r   r¡   ÚFp8Dequantize.__init__ñ  r©  r   c               ó    € V ^8„  d   QhRRRR/# )r   Úweight_patternr˜   r   r   )r   s   "r   r   ÚFp8Dequantize.__annotate__ô  s   € ÷ 
2ñ 
2°ð 
2¸ñ 
2r   c                	óê   € VP                  R 4      pV'       d   VRR MTpVP                  R4      '       d   VR\        R4      )  R,           pMVR8X  d   RpM	VR,           pV'       d
   VR ,           # T# )Ú$Nr·  r¹  ri   rr   rº  rÛ   )r‚  rv  )r¥   rð  ÚanchoredÚbaseÚscales   &&   r   Ú_scale_pattern_forÚ Fp8Dequantize._scale_pattern_forô  sr   € à!×*Ñ*¨3Ó/ˆß&.ˆ~˜c˜rÑ"°NˆØ�=‰=˜×#Ò#ØÐ*œC 	›N˜?Ð+Ð.AÕA‰EØ�XÔØ&‰Eà˜<Õ'ˆEß&ˆu�s�{Ð1¨EÐ1r   c               ó    € V ^8„  d   QhRRRR/# )r   Úpackedrq   r   r   )r   s   "r   r   rñ    s   € ÷ Jñ J ,ð J°<ñ Jr   c                óü  € \         P                  ! V P                  \         P                  VP                  R7      pVP                  4       P                  \         P                  4      pV^,          P                  4       pV^,	          ^,          P                  4       p\         P                  ! W$,          W%,          .RR7      pVP                  ! . VP                  RR O^VP                  R,          ,          N5!  # )uR   Two ``e2m1`` FP4 values per byte â†’ float32 tensor twice as wide on the last dim.)r`   rÖ   ró   NrÛ   )r"   r£   Ú_FP4_E2M1_LUTr…   rÖ   Ú
contiguousrÝ   Úuint8ÚlongÚstackrß   rÜ   )r¥   rú  ÚlutÚu8ÚlowÚhighÚunpackeds   &&     r   Ú_unpack_fp4ÚFp8Dequantize._unpack_fp4  s°   € ä�lŠl˜4×-Ñ-´U·]±]È6Ï=É=ÔYˆØ×ÑÓ ×%Ñ%¤e§k¡kÓ2ˆØ�C�x�o‰oÓˆØ�q•˜C•×%Ñ%Ó'ˆÜ—;’; ¥¨#­)Ð4¸"Ô=ˆØ×ÒÐI §¡¨c¨rÐ!2ÐI°A¸¿¹ÀRÕ8HÕ4HÓIÐIr   Nc               ó(   € V ^8„  d   QhRRRRRRRR/# )r   r×  rq   rÓ  Úoutput_dtypeútorch.dtype | Noner   r   )r   s   "r   r   rñ    s2   € ÷ +@ñ +@Ø%ð+@Ø/;ð+@ØK]ð+@à	ñ+@r   c                	óX  € \        \        R R4      pVP                  \        P                  8X  g   Ve$   VP                  V8X  d   V P	                  V4      pMVP                  \        P                  4      pVP                  RR w  rg VP                  RR w  r‰Wh,          '       g   Wy,          '       d   \        RV RV RV RV	 R2	4      hWh,          p
Wy,          pVfN   VP                  P                  '       d"   VP                  4       ^8¼  d   VP                  M\        P                  pVP                  \        P                  8X  d6   VP                  \        P                  4      R,
          P                  4       pMVP                  \        P                  4      pVP                  pVP                  RWŠW›4      pVP                  RW‰4      P!                  R4      P!                  ^4      pWï,          P                  V4      P                  V4      #   \         d    ^^r˜ ELŽi ; i)	Úfloat4_e2m1fn_x2NzWeight shape (rB   z) not divisible by scale grid (z).g     À_@rÚ   rÛ   )r'   r"   r`   r†   r  rÆ   r…   rÜ   Ú	Exceptionr‹   rf   r®   Úbfloat16rþ  Úexp2rß   rø   )r¥   r×  rÓ  r	  Ú	fp4_dtypeÚquantized_fp32rÇ   rÊ  Ú
scale_rowsÚ
scale_colsrÈ  rÉ  Ús_fp32rÎ  ÚqÚss   &&&&            r   Ú_dequantize_oneÚFp8Dequantize._dequantize_one  sÑ  € ô
 œEÐ#5°tÓ<ˆ	Ø�?‰?œeŸj™jÔ(¨YÒ-BÀyÇÁÐZcÔGcØ!×-Ñ-¨iÓ8‰Nà&Ÿ\™\¬%¯-©-Ó8ˆNØ#×)Ñ)¨"¨#Ð.‰
ˆð	*Ø%+§\¡\°"°#Ð%6Ñ"ˆJð ×Ô × 1Ô 1ÜØ    b¨¨Ð.MÈjÈ\ÐY[Ð\fÐ[gÐgiÐjóð ð Õ$ˆØÕ$ˆð
 Òà &§¡× >× >Ð >À6×CVÑCVÓCXÐ\]ÔC]�—’Ôch×cqÑcqð ð �<‰<œ5Ÿ;™;Ô&Ø—i‘i¤§¡Ó.°Õ6×<Ñ<Ó>‰Fà—Y‘YœuŸ}™}Ó-ˆFØ'×-Ñ-ˆØ×"Ñ" 2 z¸JÓPˆØ�N‰N˜2˜zÓ6×@Ñ@ÀÓD×NÑNÈqÓQˆØ•�z‰z˜,Ó'×/Ñ/°Ó?Ð?øô7 ô 	*à%&¨›
ð	*ús   Â	H ÈH)È(H)c               ó$   € V ^8„  d   QhRRRRRR/# )r   rl  útorch.nn.Module | NoneÚfull_layer_nameú
str | Noner   r
  r   )r   s   "r   r   rñ  ;  s$   € ÷ -ñ -Ð'=ð -ÐPZð -Ð_qñ -r   c                	ób   € Ve   Vf   R # \        W4      w  r4\        W4R 4      p\        VRR 4      # )Nr`   )r	   r'   )r¥   rl  r  rŽ  Útensor_namerz  s   &&&   r   Ú_get_target_dtypeÚFp8Dequantize._get_target_dtype;  s7   € ØŠ=˜OÒ3ÙÜ2°5ÓJÑˆÜ˜¨TÓ2ˆÜ�u˜g tÓ,Ð,r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   rÜ  z,dict[str, list[torch.Tensor] | torch.Tensor]r  r  rl  r  r   r   )r   s   "r   r   rñ  B  s2   € ÷ ,ñ ,à@ð,ð $ð,ð &ð	,ð 
6ñ,r   c                	ó&  € V P                  W24      pR V9   d{   Ve   TMRpVR ,          p\        V\        4      '       d
   V^ ,          MTpRV9   d?   VR,          p\        V\        4      '       d
   V^ ,          MTpW`P                  WxVR7      /# Wg/# / p	VP	                  4        Fâ  w  r«RV
9   g   RV
9   d   K  V P                  V
4      pWÁ9  d   W¹V
&   K2  \        V\        4      '       d   TMV.pW,          p\        V\        4      '       d   TMV.p\        V4      \        V4      8w  d(   \        RV
 R\        V4       R\        V4       R24      h\        WØ4       UUu. uF  w  rïV P                  WïVR7      NK  	  uppWš&   Kä  	  V	# u uppi )	zweight$ri   rr   )r	  rw   z/Fp8Dequantize: weight/scale count mismatch for z (z weights vs z	 scales).)	r  rp  rˆ  r  rÞ  r÷  rv  r‹   Úzip)r¥   rÜ  r  rl  rà  r	  Ú
target_keyr×  rÓ  rá  r³  r«  rØ  Úweightsrè   r  s   &&&&,           r   râ  ÚFp8Dequantize.convertB  s¥  € ð ×-Ñ-¨eÓEˆð
 ˜
Ô"ð -<Ò,G™ÈXˆJØ" 9Õ-ˆIÜ(2°9¼d×(CÒ(C˜	 !žÈˆIØ! ZÔ/Ø#Ð$6Õ7�Ü&0°¼×&>Ò&>˜ žÀF�Ø"×$8Ñ$8¸ÐYeÐ$8Ó$fÐgÐgØÐ*Ð*ð @BˆØ$×*Ñ*Ö,‰JˆCØ! SÔ(Ð,>À#Ô,EÙØ×/Ñ/°Ó4ˆIØÔ*à#�s‘ÙÜ)¨%´×6Ò6‘e¸U¸GˆGØÕ*ˆFÜ)¨&´$×7Ò7‘V¸f¸XˆFÜ�7‹|œs 6›{Ô*Ü ØEÀcÀUð KÜ˜G›�~ \´#°f³+°¸iðIóð ô ^aÐahÔ]qÔrÑ]qÑUYÐUV˜4×/Ñ/°À<Ð/ÖPÑ]qÒrˆF‹Kñ! -ð" ˆùó ss   Å#Fc               ó   € V ^8„  d   QhRR/# rå  r   )r   s   "r   r   rñ  q  s   € ÷ .ñ .˜Mñ .r   c                	ó,   € \        V P                  4      # rK   )r£  r¦  rè  s   &r   ré  ÚFp8Dequantize.reverse_opp  s   € ô
 ˜4×,Ñ,Ó-Ð-r   r¥  )rö   g      à?r�   g      ø?r¶  g      @g      @g      @g       €g      à¿g      ð¿g      ø¿g       Àg      Àg      Àg      ÀrK   )NN)r*   r5   r6   r7   r8   r¡   r÷  rü  r  r  r  râ  rë  ré  r:   r   r   r   rç  rç  ß  sE   † ñò")õ
2ð m€MõJ÷+@õZ-÷,ð\ ô.ó ô.r   rç  )r­   NNr­   )NNN)NNNTrK   )NNF)MÚ__conditional_annotations__Ú
__future__r   Ú	functoolsr‡   Úcollections.abcr   Údataclassesr   r"   Útorch.nnrd   r   r¯   Úactivationsr   Úcore_model_loadingr   Úquantizers.quantizers_utilsr	   r
   Úutilsr   Úutils.import_utilsr   ri  r   r   r   r   r   Úhub_kernelsr   r   Úmoer   r   Ú
get_loggerr*   rŒ   Úfloat8_e4m3fnr¢   rÄ  r  rÆ  rg   rÀ  Úcacher%   r.   r1   r;   r9   Ú_dynamoÚallow_in_graphrH   rL   rR   rn   r}   r�   rš  r’   rŠ  rº   rË   r  r  ÚModuler!  rf  r˜  r{  r�  r¡  r£  rç  )r*  s   @r   Ú<module>r=     sª  øð÷ #Ð "ã Û 	Ý $Ý !ã Ý Ý $å  Ý .ß UÝ Ý 5÷õ ÷ Dß =ð 
×	Ò	˜HÓ	%€ð × Ñ €
Ø�;Š;�zÓ"×&Ñ&€Ø�;Š;�zÓ"×&Ñ&€ð ‡�ô ó ð òGñ �$Ô÷ð ó ðð +/Ð Ð'Ó .ð ‡�×Ñô3ó ð3õlõ
÷
÷8÷:JgôZ8
�—	‘	ô 8
ôvD�2—<‘<ô Dô0D�yô DõNG7õT[7ô|b
�—‘ô b
ôJÐ*ô ñ 0Ó1Ð õ÷<÷ RôjG0�-ô G0ôTV.�Mö V.r   