+
    œ°³jKO  ã                   óÄ  € R t ^ RIHt ^ RIHtHt ^ RIt^ RIHu H	t
 ^ RIHt ^ RIHt ^RIHt ^RIHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHt ^RIHt ^RIHt ^RI H!t! ^RI"H#t#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+ ^RI,H-t- ^RI.H/t/H0t0 ^RI1H2t2 ^RI3H4t4H5t5H6t6H7t7 ^RI8H9t9H:t: ^RI;H<t<H=t= ]$P|                  ! ]?4      t@]#! RR7      ] ! R R]24      4       4       tA ! R R]74      tB ! R R ]+4      tC ! R! R"]54      tD ! R# R$]94      tE ! R% R&]<4      tF ! R' R(]=4      tG]' ! R) R*])4      4       tH ! R+ R,]-4      tI ! R- R.]64      tJ ! R/ R0]/4      tK ! R1 R2]44      tL. R3OtMR# )4zPyTorch Laguna model.)ÚCallable)ÚAnyÚLiteralN)Ústrict)Únn)Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚlogging)ÚTransformersKwargsÚno_inherit_decorator)ÚAfmoeAttention)ÚGemma3RotaryEmbedding)ÚGlm4MoeLiteDecoderLayer)Ú
LlamaModelÚeager_attention_forward)ÚQwen2MoeConfig)ÚQwen2MoeForCausalLMÚQwen2MoeMLPÚQwen2MoePreTrainedModelÚQwen2MoeRMSNorm)ÚQwen3_5MoeTopKRouterÚapply_rotary_pos_emb)ÚQwen3MoeExpertsÚQwen3MoeSparseMoeBlockzpoolside/laguna-XS.2)Ú
checkpointc                   óJ  a € ] tR t^0t o RtRt/ RRbRRbRRbRRbRR	bR
RbRRbRRbRRbRR	bRRbRR	bRRbRRbRRbRR	btRtRt^(t	^0t
^tRtRt^tRtRtRt^€tRtRtRtRtR tRtR!t]! 4       t]! 4       t]! 4       t]! 4       t]! 4       t]! 4       t R" t!R# t"R$ t#V 3R% lt$R&t%V t&R# )'ÚLagunaConfiguD  
gating (`bool` or `str`, *optional*, defaults to `True`):
    Softplus output-gate granularity. ``True`` or ``"per-head"`` applies one gate per head,
    broadcast across ``head_dim``; ``"per-element"`` applies one gate per ``(head, head_dim)``
    channel.
num_attention_heads_per_layer (`list[int]`, *optional*):
    Per-layer override for ``num_attention_heads``. Length must equal ``num_hidden_layers``.
mlp_layer_types (`list[str]`, *optional*):
    Per-layer MLP type â€” ``"dense"`` or ``"sparse"``. Length must equal
    ``num_hidden_layers``. Defaults to first layer dense, rest sparse.
moe_routed_scaling_factor (`float`, *optional*, defaults to 1.0):
    Scalar applied to routed-expert output before combining with the shared-expert output.
moe_apply_router_weight_on_input (`bool`, *optional*, defaults to `False`):
    Whether to apply router weights to the MoE input rather than the output. Not supported
    in transformers yet; ``True`` will raise a ``NotImplementedError`` for now.
moe_router_logit_softcapping (`float`, *optional*, defaults to 0.0):
    Scaling factor when applying tanh softcapping on the logits of the MoE router logits.

Example:

```python
>>> from transformers import LagunaModel, LagunaConfig

>>> configuration = LagunaConfig()
>>> model = LagunaModel(configuration)
>>> configuration = model.config
```
Úlagunazlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.g_projzlayers.*.self_attn.o_projÚrowwisezlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projz!layers.*.mlp.experts.gate_up_projÚpacked_colwisezlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsÚmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_proji ˆ i    i   é   i   FTNç      ð?ç        c           	     ó¨  € V P                   f   R.V P                  ,          V n         V P                  f)   R.R.V P                  ^,
          ,          ,           V n        V P                  f$   V P                  .V P                  ,          V n        RRRRRRR	/R
RRRRRR//pV P
                  f   W n        \        P                  ! V 3/ VBRR
R0/B  R # )NÚfull_attentionÚdenseÚsparseÚ	rope_typeÚdefaultÚ
rope_thetag    €„AÚpartial_rotary_factorg      à?Úsliding_attentiong     ˆÃ@r.   Úignore_keys_at_rope_validation)Úlayer_typesÚnum_hidden_layersÚmlp_layer_typesÚnum_attention_heads_per_layerÚnum_attention_headsÚrope_parametersr
   Ú__post_init__)ÚselfÚkwargsÚdefault_rope_paramss   &, Új/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/laguna/modular_laguna.pyr@   ÚLagunaConfig.__post_init__„   sê   € Ø×ÑÒ#Ø 0Ð1°D×4JÑ4JÕJˆDÔØ×ÑÒ'Ø$+ 9°¨z¸T×=SÑ=SÐVWÕ=WÕ/XÕ#XˆDÔ Ø×-Ñ-Ò5Ø26×2JÑ2JÐ1KÈd×NdÑNdÕ1dˆDÔ.ð ˜{¨I°|ÀXÐOfÐhkÐlØ +¨y¸,ÈÐQhÐjmÐ!nðe
Ðð ×ÑÒ'Ø#6Ô ô 	×&Ò&Øñ	
Øñ	
Ø<OÐQaÐ;bõ	
ó    c                ó   € V# ©N© )rA   rB   s   &,rD   Úconvert_rope_params_to_dictÚ(LagunaConfig.convert_rope_params_to_dict˜   s   € ØˆrF   c                óN  € V P                   '       d   \        R4      hV P                  eT   \        V P                  4      V P                  8w  d0   \        R\        V P                  4       RV P                   R24      h\        V P                  4      V P                  8w  d0   \        R\        V P                  4       RV P                   R24      h\        V P                  4      V P                  8w  d0   \        R\        V P                  4       RV P                   R24      hR# )z'Part of ``@strict``-powered validation.zhmoe_apply_router_weight_on_input=True is not yet supported in the transformers implementation of Laguna.Nz&num_attention_heads_per_layer length (z ) must equal num_hidden_layers (z).zlayer_types length (zmlp_layer_types length ()Ú moe_apply_router_weight_on_inputÚNotImplementedErrorr=   Úlenr;   Ú
ValueErrorr:   r<   )rA   s   &rD   Úvalidate_architectureÚ"LagunaConfig.validate_architecture›   s/  € à×0×0Ð0Ü%ð9óð ð
 ×.Ñ.Ò:Ü�D×6Ñ6Ó7¸4×;QÑ;QÔQäØ8¼¸T×=_Ñ=_Ó9`Ð8að b1Ø15×1GÑ1GÐ0HÈðLóð ô ˆt×ÑÓ  D×$:Ñ$:Ô:ÜØ&¤s¨4×+;Ñ+;Ó'<Ð&=ð >1Ø15×1GÑ1GÐ0HÈðLóð ô ˆt×#Ñ#Ó$¨×(>Ñ(>Ô>ÜØ*¬3¨t×/CÑ/CÓ+DÐ*Eð F1Ø15×1GÑ1GÐ0HÈðLóð ñ ?rF   c                óJ  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R	&   S[ ;R
&   S[ ;R&   S[ ;R&   S[;R&   S[S[,          ;R&   S[S[ ,          R,          ;R&   S[S[,          R,          ;R&   S[;R&   S[;R&   S[;R&   # )é   Ú
vocab_sizeÚintermediate_sizer;   r>   Únum_key_value_headsÚmax_position_embeddingsÚnum_expertsÚnum_experts_per_tokÚmoe_intermediate_sizeÚshared_expert_intermediate_sizeÚsliding_windowÚhead_dimÚattention_biasÚgatingNr=   r<   Úmoe_routed_scaling_factorrM   Úmoe_router_logit_softcapping)ÚintÚboolÚstrÚlistÚfloat)ÚformatÚ__classdict__s   "€rD   Ú__annotate__ÚLagunaConfig.__annotate__0   s8  ø‡ ‚ ñh Ññi ñj Ñ!ñk ñl Ññm ñn Ñ!ño ñp Ñ ñq ñr !Ñ)ñs ñt Ññu ñv Ñ ñw ñx Ñ$ñy ñz &)Ñ.ñ{ ñ| Ññ} ñB ÑñC ñD Ñ ñE ñF ‘3�JÑñG ñH $(©¥9¨tÕ#3Ñ:ñI ñL ™#•Y Õ%Ñ,ñM ñN  %Ñ*ñO ñP '+Ñ2ñQ ñR #(Ñ-òS rF   )r:   r<   r=   r?   )'Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚbase_model_tp_planrU   rV   r;   r>   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   r=   r<   ra   rM   rb   ÚAttributeErrorÚdecoder_sparse_stepÚmlp_only_layersÚqkv_biasÚnorm_topk_probÚuse_sliding_windowÚmax_window_layersr@   rJ   rQ   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©ri   s   @rD   r&   r&   0   s”  ø‡ € ñð: €JðØ# Yðà# Yðð 	$ Yðð 	$ Yð	ð
 	$ Yðð 	$Ð%Eðð 	$Ð%Eðð 	! )ðð 	 	ðð 	! )ðð 	,Ð-=ðð 	)¨)ðð 	Ð 0ðð 	0°ðð 	.¨yðð  	0°ð!Ðð& €JØ!ÐØÐØ!ÐØ ÐØ#)ÐØ€KØ ÐØ!$ÐØ+.Ð#Ø€Nð €HØ €NØ€FØ6:Ð!à(,€OØ'*ÐØ-2Ð$Ø*-Ð ñ )Ó*ÐÙ$Ó&€OÙÓ€HÙ#Ó%€NÙ'Ó)ÐÙ&Ó(Ðò
ò(ò÷W ƒ rF   r&   c                   ó   € ] tR t^¶tRtR# )ÚLagunaRMSNormrI   N©rl   rm   rn   ro   r{   rI   rF   rD   r   r   ¶   ó   † ÛrF   r   c                   óX   a a€ ] tR t^ºt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚLagunaRotaryEmbeddingc                ó    <€ V ^8„  d   QhRS[ /# ©rT   Úconfig©r&   )rh   ri   s   "€rD   rj   Ú"LagunaRotaryEmbedding.__annotate__»   s   ø€ ÷ !ñ !™|ñ !rF   c                ó&   <€ \         SV `  V4       R # rH   )ÚsuperÚ__init__)rA   r†   ÚdeviceÚ	__class__s   &&&€rD   r‹   ÚLagunaRotaryEmbedding.__init__»   s   ø€ Ü‰Ñ˜Ö rF   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )rT   r†   Ú
layer_typeNÚreturn)r&   re   ÚtupleÚtorchÚTensorrg   )rh   ri   s   "€rD   rj   rˆ   ¾   s7   ø€ ÷ 5ñ 5Ùð5Ù7:¸Tµzð5á	‰u�|‰|™UÐ"Õ	#ñ5rF   c           	     ó¦  € V P                   V,          R,          pV P                   V,          P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        We,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          p	V	P                  V4      V3# )a9  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`, *optional*):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
r6   r7   r.   r^   N)Údtype)
r?   ÚgetÚgetattrÚhidden_sizer>   rc   r“   Úarangerg   Úto)
r†   rŒ   r�   rB   Úbaser7   r^   ÚdimÚattention_factorÚinv_freqs
   &&&,      rD   Úcompute_default_rope_parametersÚ5LagunaRotaryEmbedding.compute_default_rope_parameters¾   s²   € ð ×%Ñ% jÕ1°,Õ?ˆà &× 6Ñ 6°zÕ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐð ˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4rF   rI   rH   )NN)	rl   rm   rn   ro   r‹   r    r{   r|   Ú__classcell__©r�   ri   s   @@rD   rƒ   rƒ   º   s   ù‡ € ÷!õ !÷5÷ 5ò 5rF   rƒ   c                   ó   € ] tR t^ÚtRtR# )Ú	LagunaMLPrI   Nr€   rI   rF   rD   r¥   r¥   Ú   r�   rF   r¥   c                   óD   a a€ ] tR t^Þt oV 3R ltV3R lR ltRtVtV ;t# )ÚLagunaTopKRouterc                ó¼   <€ \         SV `  4        \        P                  ! \        P
                  ! VP                  4      R R7      V n        VP                  V n	        R# )F)Úrequires_gradN)
rŠ   r‹   r   Ú	Parameterr“   ÚzerosrY   Úe_score_correction_biasrb   Úrouter_logit_softcapping©rA   r†   r�   s   &&€rD   r‹   ÚLagunaTopKRouter.__init__ß   s?   ø€ Ü‰ÑÔÜ')§|¢|´E·K²KÀ×@RÑ@RÓ4SÐchÔ'iˆÔ$Ø(.×(KÑ(KˆÖ%rF   c                ó�   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  S[ P                  3,          /# ©rT   Úhidden_statesr‘   )r“   r”   r’   )rh   ri   s   "€rD   rj   ÚLagunaTopKRouter.__annotate__ä   s?   ø€ ÷ @ñ @á—|‘|ð@ñ 
‰u�|‰|™UŸ\™\©5¯<©<Ð7Õ	8ñ@rF   c                ó„  € VP                  RV P                  4      p\        P                  ! WP                  4      P                  4       pV P                  R8”  d8   \        P                  ! W P                  ,          4      V P                  ,          p\        P                  ! V4      pW0P                  P                  VP                  4      ,           p\        P                  ! W@P                  RR7      w  rVVP                  RV4      pWwP!                  RRR7      ,          pVP                  VP                  4      pW'V3# )é   r/   )r�   T)r�   Úkeepdiméÿÿÿÿ)ÚreshapeÚ
hidden_dimÚFÚlinearÚweightrg   r­   r“   ÚtanhÚsigmoidr¬   r›   r–   ÚtopkÚtop_kÚgatherÚsum)rA   r²   Úrouter_logitsÚrouting_scoresÚscores_for_selectionÚ_Úselected_expertsÚrouting_weightss   &&      rD   ÚforwardÚLagunaTopKRouter.forwardä   sû   € ð &×-Ñ-¨b°$·/±/ÓBˆÜŸš ·±Ó<×BÑBÓDˆà×(Ñ(¨3Ô.Ü!ŸJšJ }×7TÑ7TÕ'TÓUÐX\×XuÑXuÕuˆMäŸš }Ó5ˆà-×0LÑ0L×0OÑ0OÐP^×PdÑPdÓ0eÕeÐÜ#ŸjšjÐ)=¿z¹zÈrÔRÑˆØ(×/Ñ/°Ð4DÓEˆØ)×,?Ñ,?ÀBÐPTÐ,?Ó,UÕUˆØ)×,Ñ,¨]×-@Ñ-@ÓAˆàÐ/?Ð?Ð?rF   )r¬   r­   ©	rl   rm   rn   ro   r‹   rÉ   r{   r|   r¢   r£   s   @@rD   r§   r§   Þ   s   ù‡ € õL÷
@÷ @ð @rF   r§   c                   ó   € ] tR t^ùtRtR# )ÚLagunaExpertsrI   Nr€   rI   rF   rD   rÍ   rÍ   ù   r�   rF   rÍ   c                   óP   a a€ ] tR t^ýt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚLagunaSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# r…   r‡   )rh   ri   s   "€rD   rj   Ú!LagunaSparseMoeBlock.__annotate__þ   s   ø€ ÷ Fñ F™|ñ FrF   c                ó~   <€ \         SV `  V4       \        WP                  R 7      V n        VP
                  V n        R# )©rV   N)rŠ   r‹   r¥   r\   Úshared_expertsra   Úrouted_scaling_factorr®   s   &&€rD   r‹   ÚLagunaSparseMoeBlock.__init__þ   s1   ø€ Ü‰Ñ˜Ô Ü'¨×BhÑBhÔiˆÔØ%+×%EÑ%EˆÖ"rF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r±   )r“   r”   )rh   ri   s   "€rD   rj   rÑ     s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rF   c                ó  € VP                   w  r#pVP                  RV4      pV P                  V4      pV P                  V4      w  rgpV P	                  WV4      pWP
                  ,          pW,           pVP                  W#V4      pV# )rµ   r·   )ÚshapeÚviewrÔ   ÚgateÚexpertsrÕ   r¸   )	rA   r²   Ú
batch_sizeÚsequence_lengthr¹   Úshared_outputrÆ   rÈ   rÇ   s	   &&       rD   rÉ   ÚLagunaSparseMoeBlock.forward  s†   € Ø2?×2EÑ2EÑ/ˆ
 ZØ%×*Ñ*¨2¨zÓ:ˆØ×+Ñ+¨MÓ:ˆà/3¯y©y¸Ó/GÑ,ˆÐ,ØŸ™ ]ÀoÓVˆà%×(BÑ(BÕBˆØ%Õ5ˆà%×-Ñ-¨jÈ:ÓVˆØÐrF   )rÕ   rÔ   rË   r£   s   @@rD   rÏ   rÏ   ý   s    ù‡ € ÷Fó F÷
÷ ð rF   rÏ   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚLagunaAttentioni  zSAfmoe-style SWA/GQA attention with Laguna-specific gating and per-layer head count.c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )rT   r†   Ú	layer_idxÚ	num_heads©r&   rc   )rh   ri   s   "€rD   rj   ÚLagunaAttention.__annotate__  s'   ø€ ÷ Lñ L™|ð L¹ð LÉñ LrF   c                óÈ  <€ W0n         \        SV `	  W4       V P                   VP                  ,          V n        \
        P                  ! VP                  V P                   V P                  ,          VP                  R 7      V n
        \
        P                  ! V P                   V P                  ,          VP                  VP                  R 7      V n        V =VP                  RJ ;'       g    VP                  R8H  V n        V P                  '       d   V P                   MV P                   V P                  ,          p\
        P                  ! VP                  VRR 7      V n        R# ))ÚbiasTzper-headFN)rå   rŠ   r‹   rW   Únum_key_value_groupsr   ÚLinearr™   r^   r_   Úq_projÚo_projÚ	gate_projr`   Úgate_per_headÚg_proj)rA   r†   rä   rå   Ú
g_proj_dimr�   s   &&&& €rD   r‹   ÚLagunaAttention.__init__  sï   ø€ à"Œä‰Ñ˜Ô+Ø$(§N¡N°f×6PÑ6PÕ$PˆÔ!ä—i’i × 2Ñ 2°D·N±NÀTÇ]Á]Õ4RÐY_×YnÑYnÔoˆŒÜ—i’i §¡°·±Õ >À×@RÑ@RÐY_×YnÑYnÔoˆŒàˆNØ#Ÿ]™]¨dÐ2×QÐQ°f·m±mÀzÑ6QˆÔØ'+×'9×'9Ð'9�T—^’^¸t¿~¹~ÐPT×P]ÑP]Õ?]ˆ
Ü—i’i × 2Ñ 2°JÀUÔKˆŽrF   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )rT   r²   Úposition_embeddingsÚattention_maskNÚpast_key_valuesrB   r‘   )r“   r”   r’   r   r   r   )rh   ri   s   "€rD   rj   rç   %  s‡   ø€ ÷ 3)ñ 3)á—|‘|ð3)ñ #¡5§<¡<±·±Ð#=Õ>ð3)ñ Ÿ™ tÕ+ð	3)ñ
  �ð3)ñ Ñ-Õ.ð3)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ3)rF   c           	     óÔ  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      pV P	                  V4      P                  V4      p	V P                  V4      P                  V4      p
V P                  V4      P                  ^^4      pV P                  V	4      P                  ^^4      p	V
P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \         4      pT! V VV	V
V3RV P"                  '       g   RMV P$                  RV P&                  RV P(                  /VB w  rïVP*                  ! . VORN5!  P-                  4       p\.        P0                  ! V P3                  V4      P5                  4       4      P7                  VP8                  4      pV P:                  '       dL   VP                  ! . VORNV P                  N5!  VP=                  R4      ,          P                  ! . VORN5!  pM	VV,          pV P?                  V4      pWï3# )NÚdropoutr/   Úscalingr]   r·   ) rÙ   r^   rì   rÚ   Úk_projÚv_projÚq_normÚ	transposeÚk_normr!   Úupdaterä   r   Úget_interfacer†   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrù   r]   r¸   Ú
contiguousrº   Úsoftplusrð   rg   r›   r–   rï   Ú	unsqueezerí   )rA   r²   rô   rõ   rö   rB   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightsrÛ   s   &&&&&,           rD   rÉ   ÚLagunaAttention.forward%  sI  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆà—{‘{ <Ó0×:Ñ:¸1¸aÓ@ˆØ—[‘[ Ó,×6Ñ6°q¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆä�zŠz˜$Ÿ+™+ mÓ4×:Ñ:Ó<Ó=×@Ñ@À×ARÑARÓSˆØ××ÐØ&×+Ò+ÐL¨[ÐL¸"ÐL¸d¿m¹mÓLÈtÏ~É~Ð^`ÓOaÕa×gÒgð ØðØ ó‰Kð &¨Õ,ˆKà—k‘k +Ó.ˆØÐ(Ð(rF   )rð   rï   rå   rê   rí   rì   rH   )
rl   rm   rn   ro   rp   r‹   rÉ   r{   r|   r¢   r£   s   @@rD   râ   râ     s#   ù‡ € á]÷Ló L÷3)÷ 3)ò 3)rF   râ   c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚLagunaDecoderLayeri[  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )rT   r†   rä   ræ   )rh   ri   s   "€rD   rj   ÚLagunaDecoderLayer.__annotate__\  s   ø€ ÷ 	cñ 	c™|ð 	c¹ñ 	crF   c                óÊ  € \         P                  P                  V 4       VP                  V n        \	        WVP
                  V,          4      V n        VP                  V,          R 8X  d   \        V4      V n	        M\        WP                  R7      V n	        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )r3   rÓ   )ÚepsN)r   ÚModuler‹   r™   râ   r=   Ú	self_attnr<   rÏ   Úmlpr¥   rV   r   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)rA   r†   rä   s   &&&rD   r‹   ÚLagunaDecoderLayer.__init__\  s£   € Ü
�	‰	×Ñ˜4Ô Ø!×-Ñ-ˆÔÜ(¨¸F×<`Ñ<`ÐajÕ<kÓlˆŒØ×!Ñ! )Õ,°Ô8Ü+¨FÓ3ˆD�Hä  ×;SÑ;SÔTˆDŒHÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÖ%rF   )r™   r  r  r  r  N)rl   rm   rn   ro   r‹   r{   r|   r}   s   @rD   r  r  [  s   ø‡ € ÷	cö 	crF   r  c                   óZ   a a€ ] tR tRt o]P
                  ! 4       V 3R l4       tRtVtV ;t	# )ÚLagunaPreTrainedModelih  c                óN  <€ \         SV `  V4       \        V\        4      '       d6   \        P
                  P                  P                  VP                  4       R# \        V\        4      '       d´   VP                   F¡  pVP                  pVP                  V,          R 8w  d   \        VP                  V,          ,          pV! VP                  VR7      w  rE\        P                  ! \!        W R24      V4       \        P                  ! \!        W R24      V4       K£  	  R# R# )r5   )r�   Ú	_inv_freqÚ_original_inv_freqN)rŠ   Ú_init_weightsÚ
isinstancer§   r“   r   ÚinitÚzeros_r¬   rƒ   r:   r    r4   r   r†   Úcopy_r˜   )rA   Úmoduler�   Úrope_init_fnÚcurr_inv_freqrÆ   r�   s   &&    €rD   r$  Ú#LagunaPreTrainedModel._init_weightsi  sØ   ø€ ä‰Ñ˜fÔ%Ü�fÔ.×/Ò/Ü�H‰H�M‰M× Ñ  ×!?Ñ!?Ö@Ü˜Ô 5×6Ò6Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó 1ñ 7rF   rI   )
rl   rm   rn   ro   r“   Úno_gradr$  r{   r|   r¢   r£   s   @@rD   r   r   h  s%   ù‡ € Ø
‡]‚]ƒ_ô^ó ÷^ð ^rF   r   c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )ÚLagunaModelix  Nc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
rT   Ú	input_idsNrõ   Úposition_idsrö   Úinputs_embedsÚ	use_cacherB   r‘   )	r“   Ú
LongTensorr”   r   ÚFloatTensorrd   r   r   r   )rh   ri   s   "€rD   rj   ÚLagunaModel.__annotate__y  s“   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ Ÿ™ tÕ+ð<
ñ ×&Ñ&¨Õ-ð	<
ñ
  �ð<
ñ ×(Ñ(¨4Õ/ð<
ñ ˜$•;ð<
ñ Ñ+Õ,ð<
ñ 
 ñ<
rF   c                óD  a€ VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g\   RV P                  RVRVRVRV/oR	V3R
 lRV3R l/p
/ p	\        V P                  P                  4       F  pW«,          ! 4       W›&   K  	  Tp/ p\        V P                  P                  4       F  pV P                  WÃV4      WÛ&   K  	  \        V P                   R V P                  P"                   4       FX  w  rïV! V3RW�P                  P                  V,          ,          RWÐP                  P                  V,          ,          RVRV/VB pKZ  	  V P%                  V4      p\'        TV'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)r†   )rŒ   r†   r3  rõ   rö   r2  r1   c                  ó   <€ \        R/ S B # ©NrI   )r   ©Úmask_kwargss   €rD   Ú<lambda>Ú%LagunaModel.forward.<locals>.<lambda>š  s   ø€ Ô*<Ñ*K¸{Ò*KrF   r8   c                  ó   <€ \        R/ S B # r:  )r   r;  s   €rD   r=  r>  ›  s   ø€ Ô-NÑ-]ÐQ\Ò-]rF   rô   )Úlast_hidden_staterö   )rP   Úembed_tokensr	   r†   Úget_seq_lengthr“   rš   rÙ   rŒ   r  r%  ÚdictÚsetr:   Ú
rotary_embÚ	enumerateÚlayersr;   Únormr   )rA   r1  rõ   r2  rö   r3  r4  rB   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsr�   r²   rô   ÚiÚdecoder_layerr<  s   &&&&&&&,        @rD   rÉ   ÚLagunaModel.forwardy  s  ø€ ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"KØ#Ô%]ð'Ð#ð #%ÐÜ! $§+¡+×"9Ñ"9Ö:�
Ø2IÖ2UÓ2WÐ#Ó/ñ ;ð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8¿¹×8OÑ8OÐPQÕ8RÕ$Sðð *ð	ð
 !0ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä%Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
rF   rI   )NNNNNN)rl   rm   rn   ro   rÉ   r{   r|   r}   s   @rD   r/  r/  x  s   ø‡ € ÷<
÷ <
ð <
rF   r/  c                   ó2   a a€ ] tR tRt oV 3R ltRtVtV ;t# )ÚLagunaForCausalLMi¸  c                ó$   <€ \         SV `  ! R/ VB # )a…  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
rI   )rŠ   rÉ   )rA   Úsuper_kwargsr�   s   &,€rD   rÉ   ÚLagunaForCausalLM.forward¹  s   ø€ ô ‰wŠÑ. Ñ.Ð.rF   rI   )rl   rm   rn   ro   rÉ   r{   r|   r¢   r£   s   @@rD   rP  rP  ¸  s   ù‡ € ÷/õ /rF   rP  )r&   rP  r/  r   )Nrp   Úcollections.abcr   Útypingr   r   r“   Útorch.nn.functionalr   Ú
functionalrº   Úhuggingface_hub.dataclassesr   Ú r   r&  Úcache_utilsr   r	   Úconfiguration_utilsr
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   Úafmoe.modeling_afmoer   Úgemma3.modeling_gemma3r   Ú$glm4_moe_lite.modeling_glm4_moe_liter   Úllama.modeling_llamar   r   Ú!qwen2_moe.configuration_qwen2_moer   Úqwen2_moe.modeling_qwen2_moer   r   r   r   Ú qwen3_5_moe.modeling_qwen3_5_moer    r!   Úqwen3_moe.modeling_qwen3_moer"   r#   Ú
get_loggerrl   Úloggerr&   r   rƒ   r¥   r§   rÍ   rÏ   râ   r  r   r/  rP  Ú__all__rI   rF   rD   Ú<module>ro     sF  ðñ å $ß ã ß Ð Ý .Ý å &ß .Ý 3ß RÝ BÝ 6Ý 6Ý 5Ý &ß ,ß EÝ 1Ý :Ý Jß FÝ >ß uÓ uß Yß Rð 
×	Ò	˜HÓ	%€ñ Ð1Ô2ØôA�>ó Aó ó 3ðAôH	�Oô 	ô5Ð1ô 5ô@	�ô 	ô@Ð+ô @ô6	�Oô 	ôÐ1ô ð* ôE)�nó E)ó ðE)ôP
cÐ0ô 
cô^Ð3ô ^ô =
�*ô =
ô@/Ð+ô /ò‚rF   