+
    œ°³jà†  ã                   ó.  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^RIHtHt ^R	IHtHt ^R
IHt ^RIHt ^RIHtHt ^RIHtH t  ^RI!H"t"H#t# ^RI$H%t% ^RI&H't'H(t( ^RI)H*t* ^RI+H,t,H-t-H.t. ^RI/H0t0H1t1 ^RI2H3t3 ]! R4       ! R R]Ph                  4      4       t5 ! R R]Ph                  4      t6 ! R R]Ph                  4      t7 ! R R]Ph                  4      t8] ! R R]Ph                  4      4       t9 ! R  R!]Ph                  4      t:R" t;R4R# lt<R$ R% lt=R5R& R' llt> ! R( R)]Ph                  4      t? ! R* R+]4      t@]' ! R, R-]#4      4       tA]' ! R. R/]A4      4       tBR6R0 R1 lltC]' ! R2 R3]A]4      4       tD. R7OtER# )8é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hub)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstringÚcan_return_tuple)Údeprecate_kwarg)ÚTransformersKwargsÚmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚLagunaConfigÚRMSNormc                   óZ   a a€ ] tR t^.t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚLagunaRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/laguna/modeling_laguna.pyÚ__annotate__ÚLagunaRMSNorm.__annotate__0   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z,
LagunaRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer&   Ú	__class__s   &&&€r+   r1   ÚLagunaRMSNorm.__init__0   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r%   Úhidden_statesr'   ©r3   ÚTensor)r)   r*   s   "€r+   r,   r-   8   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r.   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r%   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor3   Úfloat32ÚpowÚmeanÚrsqrtr6   r5   )r7   r=   Úinput_dtypeÚvariances   &&  r+   ÚforwardÚLagunaRMSNorm.forward8   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r.   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler5   Úshaper6   )r7   s   &r+   Ú
extra_reprÚLagunaRMSNorm.extra_repr?   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr.   )r6   r5   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r1   rK   rP   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r9   r*   s   @@r+   r#   r#   .   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr.   r#   c                   óÎ   a a€ ] tR t^Ct o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚLagunaRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r%   Úconfigr   )r)   r*   s   "€r+   r,   Ú"LagunaRotaryEmbedding.__annotate__E   s   ø€ ÷ Uñ U™|ñ Ur.   c           
     óÞ  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n        / V n	        V P                   Fû  pV P
                  P                  V,          pVf   K&  VR,          V P                  V&   V P                  pV P                  V,          R8w  d   \        V P                  V,          ,          pV! V P
                  W#R7      w  rg\        W R2\        P                  ! VRR7      4       \        W R2\        P                  ! VP!                  4       RR7      4       \        W R2V4       Ký  	  R # )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r0   r1   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr`   ÚsortedÚsetÚlayer_typesrc   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr   ÚBufferÚclone)	r7   r`   r\   rf   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr9   s	   &&&     €r+   r1   ÚLagunaRotaryEmbedding.__init__D   s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà)4°[Õ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÕ)¨YÔ6Ü2°4·>±>À*Õ3MÕN�Ù4@ÀÇÁÈfÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó +r.   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r%   r`   rf   Nr'   )r    ÚstrrN   r3   r?   r(   )r)   r*   s   "€r+   r,   ra   \   s7   ø€ ÷ 5ñ 5Ùð5Ù7:¸Tµzð5á	‰u�|‰|™UÐ"Õ	#ñ5r.   c           	     ó¦  € V P                   V,          R,          pV P                   V,          P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        We,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          p	V	P                  V4      V3# )a9  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`, *optional*):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimN©rC   )
rq   ÚgetÚgetattrr8   Únum_attention_headsÚintr3   Úaranger(   rD   )
r`   r\   rf   ÚkwargsÚbaser   r€   ÚdimÚattention_factorÚinv_freqs
   &&&,      r+   rr   Ú5LagunaRotaryEmbedding.compute_default_rope_parametersZ   s²   € ð" ×%Ñ% jÕ1°,Õ?ˆà &× 6Ñ 6°zÕ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐð ˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r.   c                óZ  € \        W R 24      p\        W R24      pVR
,          P                  VP                  ^ ,          R^4      P                  \        P
                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	\        P                  ! W™3RR7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP                  VP                  R	7      XP                  VP                  R	7      3#   + '       g   i     LH; i)rg   rj   N©rC   r\   ÚmpsÚcpuF)Údevice_typeÚenabled©r‰   r�   )NºNNNNrB   )r”   Nr”   )rƒ   ÚexpandrO   rD   r3   r(   r\   Ú
isinstanceÚtyper|   r   Ú	transposeÚcatÚcosÚsinrC   )r7   ÚxÚposition_idsrf   r‹   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr‘   ÚfreqsÚembrš   r›   s   &&&&         r+   rK   ÚLagunaRotaryEmbedding.forwardw   sJ  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã:A FÆF*	)r`   rp   rl   rm   rc   ©N)NN)rR   rS   rT   rU   r   r1   Ústaticmethodrr   r3   Úno_gradr   rK   rV   rW   rX   rY   s   @@r+   r[   r[   C   su   ù‡ € Ù�X vÔ.÷Uô Uó /ðUð* Ù�X vÔ.÷5ñ 5ó /ó ð5ð6 ‡]‚]ƒ_Øñ<ó ó ÷<ð <r.   r[   c                   ó<   a a€ ] tR t^�t oRV 3R lltR tRtVtV ;t# )Ú	LagunaMLPc                óä  <€ \         SV `  4        Wn        VP                  V n        Vf   VP                  MTV n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n	        \        VP                  ,          V n        R # )NF©Úbias)r0   r1   r`   r8   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r7   r`   r¬   r9   s   &&&€r+   r1   ÚLagunaMLP.__init__Ž   s¬   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ=NÒ=V ×!9Ò!9Ð\mˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r¤   )r°   r²   r®   r¯   )r7   rœ   r°   s   && r+   rK   ÚLagunaMLP.forward˜   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr.   )r²   r`   r°   r®   r8   r¬   r¯   r¤   ©	rR   rS   rT   rU   r1   rK   rV   rW   rX   rY   s   @@r+   r¨   r¨   �   s   ù‡ € ÷0÷ò r.   r¨   c                   óD   a a€ ] tR t^�t oV 3R ltV3R lR ltRtVtV ;t# )ÚLagunaTopKRouterc                óª  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        \        P                  ! \        P                  ! V P                  V P                  4      4      V n        \        P                  ! \        P                  ! VP                  4      R R7      V n        VP                  V n        R# )F)Úrequires_gradN)r0   r1   Únum_experts_per_tokÚtop_kÚnum_expertsr8   Ú
hidden_dimr   r2   r3   Úzerosr5   Úe_score_correction_biasÚmoe_router_logit_softcappingÚrouter_logit_softcapping©r7   r`   r9   s   &&€r+   r1   ÚLagunaTopKRouter.__init__ž   sŒ   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×-Ñ-ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŒÜ')§|¢|´E·K²KÀ×@RÑ@RÓ4SÐchÔ'iˆÔ$Ø(.×(KÑ(KˆÖ%r.   c                ó�   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  S[ P                  3,          /# r<   )r3   r?   rN   )r)   r*   s   "€r+   r,   ÚLagunaTopKRouter.__annotate__§   s?   ø€ ÷ @ñ @á—|‘|ð@ñ 
‰u�|‰|™UŸ\™\©5¯<©<Ð7Õ	8ñ@r.   c                ó„  € VP                  RV P                  4      p\        P                  ! WP                  4      P                  4       pV P                  R8”  d8   \        P                  ! W P                  ,          4      V P                  ,          p\        P                  ! V4      pW0P                  P                  VP                  4      ,           p\        P                  ! W@P                  RR7      w  rVVP                  RV4      pWwP!                  RRR7      ,          pVP                  VP                  4      pW'V3# )é   ç        r“   T)r‰   rA   rB   )Úreshaper¾   ÚFÚlinearr5   r(   rÂ   r3   ÚtanhÚsigmoidrÀ   rD   rC   Útopkr¼   ÚgatherÚsum)r7   r=   Úrouter_logitsÚrouting_scoresÚscores_for_selectionÚ_Úselected_expertsÚrouting_weightss   &&      r+   rK   ÚLagunaTopKRouter.forward§   sû   € ð &×-Ñ-¨b°$·/±/ÓBˆÜŸš ·±Ó<×BÑBÓDˆà×(Ñ(¨3Ô.Ü!ŸJšJ }×7TÑ7TÕ'TÓUÐX\×XuÑXuÕuˆMäŸš }Ó5ˆà-×0LÑ0L×0OÑ0OÐP^×PdÑPdÓ0eÕeÐÜ#ŸjšjÐ)=¿z¹zÈrÔRÑˆØ(×/Ñ/°Ð4DÓEˆØ)×,?Ñ,?ÀBÐPTÐ,?Ó,UÕUˆØ)×,Ñ,¨]×-@Ñ-@ÓAˆàÐ/?Ð?Ð?r.   )rÀ   r¾   r½   rÂ   r¼   r5   r¶   rY   s   @@r+   r¸   r¸   �   s   ù‡ € õL÷@÷ @ð @r.   r¸   c                   óH   a a€ ] tR t^¼t oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚLagunaExpertsz2Collection of expert weights stored as 3D tensors.c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P                  4      4      V n        \        P                  ! \        P                  ! V P                  V P                  V P                  4      4      V n        \        VP                  ,          V n        R# )r%   N)r0   r1   r½   r8   r¾   Úmoe_intermediate_sizeÚintermediate_dimr   r2   r3   ÚemptyÚgate_up_projr°   r   r±   r²   rÃ   s   &&€r+   r1   ÚLagunaExperts.__init__À   s­   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r%   r=   Útop_k_indexÚtop_k_weightsr'   r>   )r)   r*   s   "€r+   r,   ÚLagunaExperts.__annotate__É   sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r.   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesr“   NrB   )rB   éþÿÿÿ)r3   Ú
zeros_liker¦   r   Ú
functionalÚone_hotr½   ÚpermuteÚgreaterrÑ   ÚnonzeroÚwhererÌ   rß   Úchunkr²   r°   Ú
index_add_rD   rC   )r7   r=   râ   rã   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          r+   rK   ÚLagunaExperts.forwardÉ   ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)r²   r°   rß   r¾   rÝ   r½   ©
rR   rS   rT   rU   Ú__doc__r1   rK   rV   rW   rX   rY   s   @@r+   rÚ   rÚ   ¼   s   ù‡ € á<õ0÷#÷ #ð #r.   rÚ   c                   óP   a a€ ] tR t^ät oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚLagunaSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# r_   r   )r)   r*   s   "€r+   r,   Ú!LagunaSparseMoeBlock.__annotate__å   s   ø€ ÷ Fñ F™|ñ Fr.   c                ó¼   <€ \         SV `  4        \        V4      V n        \	        V4      V n        \        WP                  R 7      V n        VP                  V n
        R# )©r¬   N)r0   r1   rÚ   Úexpertsr¸   rø   r¨   Úshared_expert_intermediate_sizeÚshared_expertsÚmoe_routed_scaling_factorÚrouted_scaling_factorrÃ   s   &&€r+   r1   ÚLagunaSparseMoeBlock.__init__å   sG   ø€ Ü‰ÑÔÜ$ VÓ,ˆŒÜ$ VÓ,ˆŒ	Ü'¨×BhÑBhÔiˆÔØ%+×%EÑ%EˆÖ"r.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r<   r>   )r)   r*   s   "€r+   r,   r  ì   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r.   c                ó  € VP                   w  r#pVP                  RV4      pV P                  V4      pV P                  V4      w  rgpV P	                  WV4      pWP
                  ,          pW,           pVP                  W#V4      pV# )rÈ   rB   )rO   Úviewr  rø   r  r  rÊ   )	r7   r=   Ú
batch_sizeÚsequence_lengthr¾   Úshared_outputrÕ   r×   rÖ   s	   &&       r+   rK   ÚLagunaSparseMoeBlock.forwardì   s†   € Ø2?×2EÑ2EÑ/ˆ
 ZØ%×*Ñ*¨2¨zÓ:ˆØ×+Ñ+¨MÓ:ˆà/3¯y©y¸Ó/GÑ,ˆÐ,ØŸ™ ]ÀoÓVˆà%×(BÑ(BÕBˆØ%Õ5ˆà%×-Ñ-¨jÈ:ÓVˆØÐr.   )r  rø   r  r  r¶   rY   s   @@r+   rÿ   rÿ   ä   s    ù‡ € ÷Fó F÷÷ ð r.   rÿ   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr“   rB   )rO   r3   r™   )rœ   Úx1Úx2s   &  r+   Úrotate_halfr  û   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r.   c                ó¶  € VP                  V4      pVP                  V4      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜Wb,          \        V4      V,          ,           p
W‚,          \        V4      V,          ,           p\        P                  ! W§.RR7      p
\        P                  ! W¹.RR7      pW«3# )a{  Applies Rotary Position Embedding to the query and key tensors.

Removes the interleaving of cos and sin from GLM

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nr“   rB   )Ú	unsqueezerO   r  r3   r™   )ÚqÚkrš   r›   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds   &&&&&       r+   Úapply_rotary_pos_embr!    sÐ   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð —‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ð �{œ{¨5Ó1°CÕ7Õ8€GØ�{œ{¨5Ó1°CÕ7Õ8€Gô �iŠi˜Ð)¨rÔ2€GÜ�iŠi˜Ð)¨rÔ2€GØÐÐr.   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r%   r=   Ún_repr'   )r3   r?   r…   )r)   s   "r+   r,   r,   )  s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur.   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r”   r”   Nr”   r”   )rO   r•   rÊ   )r=   r#  ÚbatchÚnum_key_value_headsÚslenr€   s   &&    r+   Ú	repeat_kvr(  )  sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr.   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r%   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutr‡   )r   ÚModuler3   r?   r(   r   r   )r)   s   "r+   r,   r,   5  sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r.   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r%   )r‰   rC   )ÚpÚtrainingrB   )r(  Únum_key_value_groupsr3   Úmatmulr˜   r   ré   ÚsoftmaxrE   rD   rC   r0  r4  Ú
contiguous)r*  r+  r,  r-  r.  r/  r0  r‡   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r+   Úeager_attention_forwardr=  5  sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r.   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚLagunaAttentioniN  zSAfmoe-style SWA/GQA attention with Laguna-specific gating and per-layer head count.c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r%   r`   Ú	layer_idxÚ	num_heads©r    r…   )r)   r*   s   "€r+   r,   ÚLagunaAttention.__annotate__Q  s'   ø€ ÷ Lñ L™|ð L¹ð LÉñ Lr.   c                óà  <€ \         SV `  4        W0n        Wn        W n        \        VR VP                  VP                  ,          4      V n        V P                  VP                  ,          V n
        V P                  R,          V n        VP                  V n        RV n        \        P                  ! VP                  V P                  V P                  ,          VP                   R7      V n        \        P                  ! VP                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! V P                  V P                  ,          VP                  VP                   R7      V n        VP*                  V,          R8H  V n        V P,                  '       d   VP.                  MRV n        \1        V P                  VP2                  R7      V n        \1        V P                  VP2                  R7      V n        VP8                  RJ ;'       g    VP8                  R8H  V n        V P:                  '       d   V P                  MV P                  V P                  ,          p\        P                  ! VP                  VRR7      V n        R# )	r€   Trª   Úsliding_attentionN©r&   zper-headFg      à¿)r0   r1   rB  r`   rA  rƒ   r8   r„   r€   r&  r5  r/  Úattention_dropoutÚ	is_causalr   r­   Úattention_biasÚq_projÚk_projÚv_projÚo_projrp   Úis_local_attentionÚsliding_windowr#   Úrms_norm_epsÚq_normÚk_normÚgatingÚgate_per_headÚg_proj)r7   r`   rA  rB  Ú
g_proj_dimr9   s   &&&& €r+   r1   ÚLagunaAttention.__init__Q  s	  ø€ Ü‰ÑÔà"ŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$(§N¡N°f×6PÑ6PÕ$PˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’i × 2Ñ 2°D·N±NÀTÇ]Á]Õ4RÐY_×YnÑYnÔoˆŒÜ—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’i §¡°·±Õ >À×@RÑ@RÐY_×YnÑYnÔoˆŒð #)×"4Ñ"4°YÕ"?ÐCVÑ"VˆÔØ7;×7N×7NÐ7N˜f×3Ò3ÐTXˆÔä# D§M¡M°v×7JÑ7JÔKˆŒÜ# D§M¡M°v×7JÑ7JÔKˆŒØ#Ÿ]™]¨dÐ2×QÐQ°f·m±mÀzÑ6QˆÔØ'+×'9×'9Ð'9�T—^’^¸t¿~¹~ÐPT×P]ÑP]Õ?]ˆ
Ü—i’i × 2Ñ 2°JÀUÔKˆŽr.   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r%   r=   Úposition_embeddingsr.  NÚpast_key_valuesr‡   r'   )r3   r?   rN   r   r   r   )r)   r*   s   "€r+   r,   rD  p  s‡   ø€ ÷ 3)ñ 3)á—|‘|ð3)ñ #¡5§<¡<±·±Ð#=Õ>ð3)ñ Ÿ™ tÕ+ð	3)ñ
  �ð3)ñ Ñ-Õ.ð3)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ3)r.   c           	     óÔ  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      pV P	                  V4      P                  V4      p	V P                  V4      P                  V4      p
V P                  V4      P                  ^^4      pV P                  V	4      P                  ^^4      p	V
P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \         4      pT! V VV	V
V3RV P"                  '       g   RMV P$                  RV P&                  RV P(                  /VB w  rïVP*                  ! . VORN5!  P-                  4       p\.        P0                  ! V P3                  V4      P5                  4       4      P7                  VP8                  4      pV P:                  '       dL   VP                  ! . VORNV P                  N5!  VP=                  R4      ,          P                  ! . VORN5!  pM	VV,          pV P?                  V4      pWï3# )Nr0  rÉ   r/  rP  rB   ) rO   r€   rK  r  rL  rM  rR  r˜   rS  r!  ÚupdaterA  r   Úget_interfacer`   Ú_attn_implementationr=  r4  rH  r/  rP  rÊ   r8  rË   ÚsoftplusrV  r(   rD   rC   rU  r  rN  )r7   r=   rZ  r.  r[  r‡   Úinput_shapeÚhidden_shapeÚquery_statesr9  r:  rš   r›   Úattention_interfacer<  r;  rø   s   &&&&&,           r+   rK   ÚLagunaAttention.forwardp  sI  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆà—{‘{ <Ó0×:Ñ:¸1¸aÓ@ˆØ—[‘[ Ó,×6Ñ6°q¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆä�zŠz˜$Ÿ+™+ mÓ4×:Ñ:Ó<Ó=×@Ñ@À×ARÑARÓSˆØ××ÐØ&×+Ò+ÐL¨[ÐL¸"ÐL¸d¿m¹mÓLÈtÏ~É~Ð^`ÓOaÕa×gÒgð ØðØ ó‰Kð &¨Õ,ˆKà—k‘k +Ó.ˆØÐ(Ð(r.   )rH  r`   rV  rU  r€   rI  rO  rS  rL  rA  rB  r5  rN  rR  rK  r/  rP  rM  r¤   rü   rY   s   @@r+   r?  r?  N  s#   ù‡ € Ù]÷Ló L÷>3)÷ 3)ò 3)r.   r?  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLagunaDecoderLayeri¦  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r%   r`   rA  rC  )r)   r*   s   "€r+   r,   ÚLagunaDecoderLayer.__annotate__§  s   ø€ ÷ 	cñ 	c™|ð 	c¹ñ 	cr.   c                óª  <€ \         SV `  4        VP                  V n        \        WVP                  V,          4      V n        VP                  V,          R 8X  d   \        V4      V n        M\        WP                  R7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )Úsparser  rG  N)r0   r1   r8   r?  Únum_attention_heads_per_layerÚ	self_attnÚmlp_layer_typesrÿ   Úmlpr¨   r¬   r#   rQ  Úinput_layernormÚpost_attention_layernorm©r7   r`   rA  r9   s   &&&€r+   r1   ÚLagunaDecoderLayer.__init__§  s�   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ(¨¸F×<`Ñ<`ÐajÕ<kÓlˆŒØ×!Ñ! )Õ,°Ô8Ü+¨FÓ3ˆD�Hä  ×;SÑ;SÔTˆDŒHÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÖ%r.   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r%   r=   r.  Nr�   r[  Ú	use_cacherZ  r‡   r'   )r3   r?   Ú
LongTensorr   ÚboolrN   r   r   )r)   r*   s   "€r+   r,   ri  ²  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr.   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r=   r.  r�   r[  ru  rZ  © )rp  rm  rq  ro  )
r7   r=   r.  r�   r[  ru  rZ  r‡   ÚresidualrÕ   s
   &&&&&&&,  r+   rK   ÚLagunaDecoderLayer.forward²  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr.   )r8   rp  ro  rq  rm  )NNNFNr¶   rY   s   @@r+   rg  rg  ¦  s    ù‡ € ÷	có 	c÷÷ ò r.   rg  c                   ó®   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]! ]^ R7      R]R	]/t]P&                  ! 4       V 3R
 l4       tV3R ltRtVtV ;t# )ÚLagunaPreTrainedModeliÒ  ÚmodelTrg  r[  rÒ   )Úindexr=   Ú
attentionsc                ó¦  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dH   \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       M9\	        V\        4      '       d$   \        P                  ! VP                  R VR7       \	        V\        4      '       d6   \        P                  P                  P                  VP                  4       R# \	        V\         4      '       d´   VP"                   F¡  pVP$                  pVP&                  V,          R8w  d   \(        VP&                  V,          ,          pV! VP                  VR7      w  rV\        P*                  ! \-        W R24      V4       \        P*                  ! \-        W R24      V4       K£  	  R# R# )rÉ   )rG   Ústdrd   re   rg   ri   N)r0   Ú_init_weightsr`   Úinitializer_ranger–   rÚ   ÚinitÚnormal_rß   r°   r¸   r5   r3   r   Úzeros_rÀ   r[   rp   rr   rc   r   Úcopy_rƒ   )r7   r*  r‚  rf   rw   rx   rÕ   r9   s   &&     €r+   rƒ  Ú#LagunaPreTrainedModel._init_weightså  sB  ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœm×,Ò,Ü�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸Ö=Ü˜Ô 0×1Ò1Ü�LŠL˜Ÿ™¨S°cÕ:Ü�fÔ.×/Ò/Ü�H‰H�M‰M× Ñ  ×!?Ñ!?Ö@Ü˜Ô 5×6Ò6Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó 1ñ 7r.   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r_   r   )r)   r*   s   "€r+   r,   Ú"LagunaPreTrainedModel.__annotate__Ò  s   ø‡ ‚ áÑò r.   ry  )rR   rS   rT   rU   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   r¸   rg  r?  Ú_can_record_outputsr3   r¦   rƒ  Ú__annotate_func__rV   rW   rX   rY   s   @@r+   r}  r}  Ò  s…   ù‡ € ð  ÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà™Ð(8ÀÔBØÐ+Ø�oðÐð ‡]‚]ƒ_ô^ó ð^÷) … r.   r}  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚLagunaModeliú  c                ó    <€ V ^8„  d   QhRS[ /# r_   r   )r)   r*   s   "€r+   r,   ÚLagunaModel.__annotate__ü  s   ø€ ÷ ñ ™|ñ r.   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )rG  ©r`   FN)r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr8   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrg  Úlayersr#   rQ  Únormr[   Ú
rotary_embÚgradient_checkpointingÚ	post_initrr  s   && €r+   r1   ÚLagunaModel.__init__ü  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°vÔ>ˆŒØ&+ˆÔ#ð 	�‰Öùò es   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r%   Ú	input_idsNr.  r�   r[  Úinputs_embedsru  r‡   r'   )	r3   rv  r?   r   ÚFloatTensorrw  r   r   r   )r)   r*   s   "€r+   r,   rš    s“   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ Ÿ™ tÕ+ð<
ñ ×&Ñ&¨Õ-ð	<
ñ
  �ð<
ñ ×(Ñ(¨4Õ/ð<
ñ ˜$•;ð<
ñ Ñ+Õ,ð<
ñ 
 ñ<
r.   c                óD  a€ VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g\   RV P                  RVRVRVRV/oR	V3R
 lRV3R l/p
/ p	\        V P                  P                  4       F  pW«,          ! 4       W›&   K  	  Tp/ p\        V P                  P                  4       F  pV P                  WÃV4      WÛ&   K  	  \        V P                   R V P                  P"                   4       FX  w  rïV! V3RW�P                  P                  V,          ,          RWÐP                  P                  V,          ,          RVRV/VB pKZ  	  V P%                  V4      p\'        TV'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrœ  )r\   r`   r­  r.  r[  r�   Úfull_attentionc                  ó   <€ \        R/ S B # ©Nry  )r   ©Úmask_kwargss   €r+   Ú<lambda>Ú%LagunaModel.forward.<locals>.<lambda>0  s   ø€ Ô*<Ñ*K¸{Ò*Kr.   rF  c                  ó   <€ \        R/ S B # r²  )r   r³  s   €r+   rµ  r¶  1  s   ø€ Ô-NÑ-]ÐQ\Ò-]r.   rZ  )Úlast_hidden_stater[  )Ú
ValueErrorr¡  r   r`   Úget_seq_lengthr3   r†   rO   r\   r  r–   Údictro   rp   r§  Ú	enumerater¥  r¤  r¦  r   )r7   r¬  r.  r�   r[  r­  ru  r‡   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsrf   r=   rZ  ÚiÚdecoder_layerr´  s   &&&&&&&,        @r+   rK   ÚLagunaModel.forward  s  ø€ ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"KØ#Ô%]ð'Ð#ð #%ÐÜ! $§+¡+×"9Ñ"9Ö:�
Ø2IÖ2UÓ2WÐ#Ó/ñ ;ð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8¿¹×8OÑ8OÐPQÕ8RÕ$Sðð *ð	ð
 !0ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä%Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r.   )r¡  r¨  r¥  r¦  rž  r§  rŸ  )NNNNNN)rR   rS   rT   rU   r1   r   r   r   rK   rV   rW   rX   rY   s   @@r+   r˜  r˜  ú  s=   ù‡ € ÷ó ð   ØØ÷<
ñ <
ó ó ó  ÷<
ð <
r.   r˜  c          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )r%   Úgate_logitsNr½   r.  r'   )r3   r?   rN   r…   )r)   s   "r+   r,   r,   N  sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&r.   c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
rŽ   rÉ   )r\   rC   r“   )Ú	minlengthrB   )r–   rN   r\   r3   r¿   rE   rÊ   rD   r   ré   r7  rÏ   Úbincountr(   rÑ   rO   Úscatter_add_Úrepeat_interleaver  )rÄ  r½   r¼   r.  Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gater×   rÕ   rÖ   Útokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r+   Úload_balancing_loss_funcrÓ  N  sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r.   c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚLagunaForCausalLMi“  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr=   ÚlogitsÚkeep_full_weightc                óR  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        VP                  V n	        VP                  V n
        VP                  V n        V P                  4        R# )Frª   N)r0   r1   r˜  r~  rŸ  r   r­   r8   rÖ  Úrouter_aux_loss_coefr½   r»   r©  rÃ   s   &&€r+   r1   ÚLagunaForCausalLM.__init__›  s   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Ör.   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r%   r¬  Nr.  r�   r[  r­  Úlabelsru  Úoutput_router_logitsÚlogits_to_keepr‡   r'   )
r3   rv  r?   r   r®  rw  r…   r   r   r   )r)   r*   s   "€r+   r,   ÚLagunaForCausalLM.__annotate__©  sÖ   ø€ ÷ @
ñ @
á×#Ñ# dÕ*ð@
ñ Ÿ™ tÕ+ð@
ñ ×&Ñ&¨Õ-ð	@
ñ
  �ð@
ñ ×(Ñ(¨4Õ/ð@
ñ × Ñ  4Õ'ð@
ñ ˜$•;ð@
ñ # T�kð@
ñ ™eŸl™lÕ*ð@
ñ Ñ+Õ,ð@
ñ 
#ñ@
r.   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )a…  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Nr¬  r.  r�   r[  r­  ru  rß  r”   )ÚlossÚaux_lossrØ  r[  r=   r€  rÒ   ry  )r`   rß  r~  r¸  r–   r…   ÚslicerÖ  Úloss_functionrŸ  rÓ  rÒ   r½   r»   rÛ  rD   r\   r   r[  r=   r€  )r7   r¬  r.  r�   r[  r­  rÞ  ru  rß  rà  r‡   Úoutputsr=   Úslice_indicesrØ  rã  rä  s   &&&&&&&&&&,      r+   rK   ÚLagunaForCausalLM.forward§  sw  € ð. %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r.   )rÖ  r~  r½   r»   rÛ  rŸ  )	NNNNNNNNr   )rR   rS   rT   rU   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr1   r   r   rK   rV   rW   rX   rY   s   @@r+   rÕ  rÕ  “  sj   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HàÐ/Ð0€Jõ
ð Ø÷@
ñ @
ó ó ÷@
ð @
r.   rÕ  )rÈ   )rÉ   )Nr%   N)rÕ  r˜  r}  )FÚcollections.abcr   r3   Útorch.nn.functionalr   ré   rË   Ú r   r…  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   r   Úconfiguration_lagunar    r1  r#   r[   r¨   r¸   rÚ   rÿ   r  r!  r(  r=  r?  rg  r}  r˜  rÓ  rÕ  Ú__all__ry  r.   r+   Ú<module>r     s  ðõ* %ã ß Ð Ý å &Ý !ß .Ý )ß Sß RÝ BÝ 9ß Qß Kß FÝ &ß 5Ý 0ß [Ñ [ß EÝ .ñ ˜YÓ'ôJ�B—I‘Ió Jó (ðJô(G<˜BŸI™Iô G<ôT�—	‘	ô ô @�r—y‘yô @ð> ô$#�B—I‘Ió $#ó ð$#ôN˜2Ÿ9™9ô ò.(ô#õL	U÷%ô2U)�b—i‘iô U)ôp)Ð3ô )ðX ô$^˜Oó $^ó ð$^ðN ôP
Ð'ó P
ó ðP
÷fB&ðJ ôU
Ð-¨ó U
ó ðU
òp H‚r.   