+
    œ°³j‘w  ã                   óN  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^RIHtHtHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHt ^RIH t H!t! ^RI"H#t# ^RI$H%t%H&t&H't' ^RI(H)t) ^RI*H+t+H,t, ^RI-H.t.H/t/ ^RI0H1t1 ]! R4       ! R R]Pd                  4      4       t3 ! R R]Pd                  4      t4 ! R R]Pd                  4      t5R t6]! R4      R4R l4       t7R R lt8R5R  R! llt9]! ]74       ! R" R#]Pd                  4      4       t:] ! R$ R%]Pd                  4      4       t; ! R& R']Pd                  4      t< ! R( R)]Pd                  4      t= ! R* R+]4      t>]& ! R, R-]!4      4       t?]& ! R. R/]?4      4       t@R6R0 R1 lltA]& ! R2 R3]?]4      4       tB. R7OtCR# )8é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚOlmoeConfigÚRMSNormc                   óZ   a a€ ] tR t^+t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚOlmoeRMSNormc                ó   <€ V ^8„  d   QhRR/# )é   ÚreturnN© )ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/olmoe/modeling_olmoe.pyÚ__annotate__ÚOlmoeRMSNorm.__annotate__-   s   ø€ ÷ $ñ $°ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z+
OlmoeRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s   &&&€r)   r/   ÚOlmoeRMSNorm.__init__-   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör,   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r$   Úhidden_statesr%   ©r1   ÚTensor)r'   r(   s   "€r)   r*   r+   5   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r,   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r$   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor1   Úfloat32ÚpowÚmeanÚrsqrtr4   r3   )r5   r<   Úinput_dtypeÚvariances   &&  r)   ÚforwardÚOlmoeRMSNorm.forward5   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r,   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler3   Úshaper4   )r5   s   &r)   Ú
extra_reprÚOlmoeRMSNorm.extra_repr<   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr,   )r4   r3   )gñhãˆµøä>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r/   rJ   rO   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r8   r(   s   @@r)   r"   r"   +   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr,   r"   c                   óÎ   a a€ ] tR t^@t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚOlmoeRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r$   Úconfigr   )r'   r(   s   "€r)   r*   Ú!OlmoeRotaryEmbedding.__annotate__B   s   ø€ ÷ Oñ O™{ñ Or,   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r.   r/   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr_   Úrope_parametersrb   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r5   r_   r[   Úrope_init_fnrl   r8   s   &&&  €r)   r/   ÚOlmoeRotaryEmbedding.__init__A   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr,   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r$   r_   r%   )r   rM   r1   r>   Úfloat)r'   r(   s   "€r)   r*   r`   T   s.   ø€ ÷ 5ñ 5±ð 5ÑW\Ñ]b×]iÑ]iÑkpÐ]pÕWqñ 5r,   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©rB   )rh   Úgetattrr6   Únum_attention_headsr1   Úarangerr   rC   )r_   r[   ÚkwargsÚbaseÚdimÚattention_factorrl   s   &&,    r)   ri   Ú4OlmoeRotaryEmbedding.compute_default_rope_parametersR   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r,   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N©rB   r[   ÚmpsÚcpuF)Údevice_typeÚenabled©r|   rv   )NºNNNNrA   )r†   Nr†   )rl   ÚexpandrN   rC   r1   rr   r[   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrj   ÚsinrB   )
r5   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrƒ   ÚfreqsÚembr�   rŽ   s
   &&&       r)   rJ   ÚOlmoeRotaryEmbedding.forwardf   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rj   r_   rl   rf   rn   rg   rb   ©N)rQ   rR   rS   rT   r   r/   Ústaticmethodri   r1   Úno_gradr   rJ   rU   rV   rW   rX   s   @@r)   rZ   rZ   @   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r,   rZ   c                   ó8   a a€ ] tR t^yt oV 3R ltR tRtVtV ;t# )ÚOlmoeMLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©F©ÚbiasN)r.   r/   r_   r6   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r5   r_   r8   s   &&€r)   r/   ÚOlmoeMLP.__init__z   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr,   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r–   )r£   r¥   r¡   r¢   )r5   r�   r£   s   && r)   rJ   ÚOlmoeMLP.forward„   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr,   )r¥   r_   r£   r¡   r6   rŸ   r¢   ©	rQ   rR   rS   rT   r/   rJ   rU   rV   rW   rX   s   @@r)   rš   rš   y   s   ù‡ € õ0÷ò r,   rš   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr…   rA   )rN   r1   rŒ   )r�   Úx1Úx2s   &  r)   Úrotate_halfr®   ‰   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r,   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer®   )ÚqÚkr�   rŽ   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r)   Úapply_rotary_pos_embr·   �   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr,   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r$   r<   Ún_repr%   )r1   r>   Úint)r'   s   "r)   r*   r*   ª   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur,   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r†   r†   Nr†   r†   )rN   r‡   Úreshape)r<   r¹   ÚbatchÚnum_key_value_headsÚslenru   s   &&    r)   Ú	repeat_kvrÀ   ª   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr,   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r$   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrz   )r   ÚModuler1   r>   rr   r   r   )r'   s   "r)   r*   r*   ¶   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r,   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r$   )r|   rB   )ÚpÚtrainingrA   )rÀ   Únum_key_value_groupsr1   Úmatmulr‹   r   Ú
functionalÚsoftmaxrD   rC   rB   rÈ   rÌ   Ú
contiguous)rÂ   rÃ   rÄ   rÅ   rÆ   rÇ   rÈ   rz   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r)   Úeager_attention_forwardrÖ   ¶   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r,   c                   ó\   a a€ ] tR t^Ït oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )ÚOlmoeAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r$   r_   Ú	layer_idxN©r   rº   )r'   r(   s   "€r)   r*   ÚOlmoeAttention.__annotate__Ó   s   ø€ ÷ 
ñ 
™{ð 
±s¸Tµzñ 
r,   c                ó~  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        \)        VP
                  VP*                  R7      V n        \)        VP
                  VP                  ,          VP                  ,          VP*                  R7      V n        R# )ru   Tr�   ©r7   Ng      à¿)r.   r/   r_   rÚ   rw   r6   rx   ru   r¾   rÍ   rÇ   Úattention_dropoutÚ	is_causalr   r    Úattention_biasÚq_projÚk_projÚv_projÚo_projr"   Úrms_norm_epsÚq_normÚk_norm©r5   r_   rÚ   r8   s   &&&€r)   r/   ÚOlmoeAttention.__init__Ó   s™  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒÜ"Ø×Ñ 6×#=Ñ#=Õ=À×A[ÑA[Õ[Ðag×atÑatô
ˆŽr,   c                óR  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r$   r<   Úposition_embeddingsrÆ   NÚpast_key_valuesrz   r%   )r1   r>   rM   r   r   r   )r'   r(   s   "€r)   r*   rÜ   î   s—   ø€ ÷ /)ñ /)á—|‘|ð/)ñ #¡5§<¡<±·±Ð#=Õ>ð/)ñ Ÿ™ tÕ+ð	/)ñ
  �ð/)ñ Ñ+Õ,ð/)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ/)r,   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      4      pV P	                  V P                  V4      4      p	V P                  V4      p
V P                  P                  eµ   VP                  V P                  P                  ) V P                  P                  R7       V	P                  V P                  P                  ) V P                  P                  R7       V
P                  V P                  P                  ) V P                  P                  R7       VP                  ! V!  P                  ^^4      pV	P                  ! V!  P                  ^^4      p	V
P                  ! V!  P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                   ! V P                  P"                  \$        4      pT! V VV	V
V3RV P&                  '       g   RMV P(                  RV P*                  R\-        V P                  RR 4      /VB w  rïVP.                  ! . VORN5!  P1                  4       pV P3                  V4      pWï3# )N)ÚminÚmaxrÈ   ç        rÇ   Úsliding_windowrA   )rN   ru   rç   râ   rè   rã   rä   r_   Úclip_qkvÚclamp_Úviewr‹   r·   ÚupdaterÚ   r   Úget_interfaceÚ_attn_implementationrÖ   rÌ   rß   rÇ   rw   r¼   rÑ   rå   )r5   r<   rì   rÆ   rí   rz   Úinput_shapeÚhidden_shapeÚquery_statesrÒ   rÓ   r�   rŽ   Úattention_interfacerÕ   rÔ   s   &&&&&,          r)   rJ   ÚOlmoeAttention.forwardî   sI  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>ˆØ—[‘[ §¡¨]Ó!;Ó<ˆ
Ø—{‘{ =Ó1ˆà�;‰;×ÑÒ+Ø×Ñ T§[¡[×%9Ñ%9Ð$9¸t¿{¹{×?SÑ?SÐÔTØ×Ñ 4§;¡;×#7Ñ#7Ð"7¸T¿[¹[×=QÑ=QÐÔRØ×Ñ T§[¡[×%9Ñ%9Ð$9¸t¿{¹{×?SÑ?SÐÔTà#×(Ò(¨,Ñ7×AÑAÀ!ÀQÓGˆØ—_’_ lÑ3×=Ñ=¸aÀÓCˆ
Ø#×(Ò(¨,Ñ7×AÑAÀ!ÀQÓGˆØ&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ô # 4§;¡;Ð0@À$ÓGð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r,   )rß   r_   ru   rà   rè   rã   rÚ   rÍ   rå   rç   râ   rÇ   rä   r–   ©
rQ   rR   rS   rT   Ú__doc__r/   rJ   rU   rV   rW   rX   s   @@r)   rØ   rØ   Ï   s!   ù‡ € áG÷
õ 
÷6/)÷ /)ò /)r,   rØ   c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚOlmoeExpertsi   z2Collection of expert weights stored as 3D tensors.c                ó    <€ V ^8„  d   QhRS[ /# r^   r   )r'   r(   s   "€r)   r*   ÚOlmoeExperts.__annotate__$  s   ø€ ÷ 0ñ 0™{ñ 0r,   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        R# )r$   N)r.   r/   Únum_local_expertsÚnum_expertsr6   Ú
hidden_dimrŸ   Úintermediate_dimr   r0   r1   ÚemptyÚgate_up_projr£   r   r¤   r¥   r¦   s   &&€r)   r/   ÚOlmoeExperts.__init__$  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr,   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r$   r<   Útop_k_indexÚtop_k_weightsr%   r=   )r'   r(   s   "€r)   r*   r  -  sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r,   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesr…   NrA   )rA   éþÿÿÿ)r1   Ú
zeros_liker˜   r   rÏ   Úone_hotr  ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearr
  Úchunkr¥   r£   Ú
index_add_rC   rB   )r5   r<   r  r  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          r)   rJ   ÚOlmoeExperts.forward-  ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)r¥   r£   r
  r  r  r  rþ   rX   s   @@r)   r  r     s!   ù‡ € á<÷0ó 0÷#÷ #ð #r,   r  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚOlmoeTopKRouteriH  c                ó4  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  V P                  4      4      V n        R # r–   )r.   r/   Únum_experts_per_tokÚtop_kr  Únorm_topk_probr6   r  r   r0   r1   Úzerosr3   r¦   s   &&€r)   r/   ÚOlmoeTopKRouter.__init__I  si   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×-Ñ-ˆÔØ$×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŽr,   c                óÎ  € VP                  RV P                  4      p\        P                  ! WP                  4      p\
        P                  P                  P                  V\
        P                  RR7      p\
        P                  ! W0P                  RR7      w  rEV P                  '       d   WDP                  RRR7      ,          pVP                  VP                  4      pTpW&V3# )é   )rB   r|   r…   T)r|   r@   rA   )r¼   r  ÚFr  r3   r1   r   rÏ   rÐ   rr   Útopkr+  r,  r  rC   rB   )r5   r<   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress   &&     r)   rJ   ÚOlmoeTopKRouter.forwardQ  sµ   € Ø%×-Ñ-¨b°$·/±/ÓBˆÜŸš ·±Ó<ˆÜ—x‘x×*Ñ*×2Ñ2°=ÌÏÉÐY[Ð2Ó\ˆÜ+0¯:ª:°lÇJÁJÐTVÔ+WÑ(ÐØ××ÐØ× 4Ñ 4¸ÀTÐ 4Ó JÕJÐØ+×.Ñ.¨}×/BÑ/BÓCÐØ(ˆØ¨^Ð;Ð;r,   )r  r,  r  r+  r3   rª   rX   s   @@r)   r(  r(  H  s   ù‡ € õS÷	<ò 	<r,   r(  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚOlmoeSparseMoeBlocki]  c                ód   <€ \         SV `  4        \        V4      V n        \	        V4      V n        R # r–   )r.   r/   r(  r#  r  Úexpertsr¦   s   &&€r)   r/   ÚOlmoeSparseMoeBlock.__init__^  s&   ø€ Ü‰ÑÔÜ# FÓ+ˆŒ	Ü# FÓ+ˆŽr,   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r;   r=   )r'   r(   s   "€r)   r*   Ú OlmoeSparseMoeBlock.__annotate__c  s#   ø€ ÷ #ñ #¡U§\¡\ð #±e·l±lñ #r,   c                ó´   € VP                   w  r#pVP                  RV4      pV P                  V4      w  rVpV P                  WV4      P	                  W#V4      pV# )r0  rA   )rN   rõ   r#  r<  r¼   )	r5   r<   Ú
batch_sizeÚsequence_lengthr  Ú_r  r  r  s	   &&       r)   rJ   ÚOlmoeSparseMoeBlock.forwardc  s`   € Ø2?×2EÑ2EÑ/ˆ
 ZØ%×*Ñ*¨2¨zÓ:ˆØ(,¯	©	°-Ó(@Ñ%ˆ˜+Ø"Ÿl™l¨=À}ÓU×]Ñ]Ø¨ó
Ðð #Ð"r,   )r<  r#  rª   rX   s   @@r)   r:  r:  ]  s   ù‡ € õ,÷
#÷ #ð #r,   r:  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚOlmoeDecoderLayerim  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r$   r_   rÚ   rÛ   )r'   r(   s   "€r)   r*   ÚOlmoeDecoderLayer.__annotate__n  s   ø€ ÷ bñ b™{ð b±sñ br,   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))r_   rÚ   rÞ   N)r.   r/   r6   rØ   Ú	self_attnr:  Úmlpr"   ræ   Úinput_layernormÚpost_attention_layernormré   s   &&&€r)   r/   ÚOlmoeDecoderLayer.__init__n  sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ'¨vÔKˆŒÜ& vÓ.ˆŒÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÖ%r,   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r$   r<   rÆ   Nr�   rí   Ú	use_cacherì   rz   r%   )r1   r>   Ú
LongTensorr   ÚboolrM   r   r   )r'   r(   s   "€r)   r*   rH  v  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr,   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r<   rÆ   r�   rí   rP  rì   r&   )rL  rJ  rM  rK  )
r5   r<   rÆ   r�   rí   rP  rì   rz   ÚresidualrC  s
   &&&&&&&,  r)   rJ   ÚOlmoeDecoderLayer.forwardv  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr,   )r6   rL  rK  rM  rJ  )NNNFNrª   rX   s   @@r)   rF  rF  m  s    ù‡ € ÷bó b÷÷ ò r,   rF  c                   óž   a € ] tR tRt o RtRtR.tR.tRtRt	R]
! ]^ R7      R]R	]/tRtRt]P$                  ! 4       R
 4       tV 3R ltRtV tR# )ÚOlmoePreTrainedModeli–  ÚmodelTrF  rí   r3  )Úindexr<   Ú
attentionsc                óÜ  € \         P                  ! W4       \        V\        4      '       dq   \        P
                  ! VP                  R V P                  P                  R7       \        P
                  ! VP                  R V P                  P                  R7       R# \        V\        4      '       d:   \        P
                  ! VP                  R V P                  P                  R7       R# R# )rñ   )rF   ÚstdN)r   Ú_init_weightsrˆ   r  ÚinitÚnormal_r
  r_   Úinitializer_ranger£   r(  r3   )r5   rÂ   s   &&r)   r]  Ú"OlmoePreTrainedModel._init_weights¨  s�   € ä×%Ò% dÔ3Ü�fœl×+Ò+Ü�LŠL˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�LŠL˜×)Ñ)°¸¿¹×9VÑ9V×WÜ˜¤×0Ò0Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6S×Tñ 1r,   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r^   r   )r'   r(   s   "€r)   r*   Ú!OlmoePreTrainedModel.__annotate__–  s   ø‡ ‚ áÑò r,   r&   N)rQ   rR   rS   rT   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpar   r(  rF  rØ   Ú_can_record_outputsÚ_supports_attention_backendÚ_can_compile_fullgraphr1   r˜   r]  Ú__annotate_func__rU   rV   )r(   s   @r)   rW  rW  –  s~   ø‡ € ð  ÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€Nà™¨¸qÔAØÐ*Ø�nðÐð #'ÐØ!Ðà
‡]‚]ƒ_ñUó ðU÷' ƒ r,   rW  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
OlmoeModeli²  c                ó    <€ V ^8„  d   QhRS[ /# r^   r   )r'   r(   s   "€r)   r*   ÚOlmoeModel.__annotate__´  s   ø€ ÷ ñ ™{ñ r,   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )rÞ   ©r_   FN)r.   r/   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr6   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrF  Úlayersr"   ræ   ÚnormrZ   Ú
rotary_embÚgradient_checkpointingÚ	post_initré   s   && €r)   r/   ÚOlmoeModel.__init__´  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒÜŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜCHÈ×IaÑIaÔCbÓcÑCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŒØ&+ˆÔ#ð 	�‰Öùò ds   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r$   Ú	input_idsNrÆ   r�   rí   Úinputs_embedsrP  rz   r%   )	r1   rQ  r>   r   ÚFloatTensorrR  r   r   r   )r'   r(   s   "€r)   r*   rq  Æ  s“   ø€ ÷ 5
ñ 5
á×#Ñ# dÕ*ð5
ñ Ÿ™ tÕ+ð5
ñ ×&Ñ&¨Õ-ð	5
ñ
  �ð5
ñ ×(Ñ(¨4Õ/ð5
ñ ˜$•;ð5
ñ Ñ+Õ,ð5
ñ 
 ñ5
r,   c                ó†  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£4      pV P                  R V P                  P                    F  pV! V
3RVRV	RVRVR	V/VB p
K  	  V P                  V
4      p
\        V
VR
7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrs  )r[   )r_   r„  rÆ   rí   r�   rì   rÆ   r�   rí   rP  )Úlast_hidden_staterí   )Ú
ValueErrorr   r_   rx  Úget_seq_lengthr1   ry   rN   r[   r±   r   r~  r|  r{  r}  r   )r5   rƒ  rÆ   r�   rí   r„  rP  rz   Úpast_seen_tokensÚcausal_maskr<   rì   Údecoder_layers   &&&&&&&,     r)   rJ   ÚOlmoeModel.forwardÃ  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆð #Ÿo™o¨mÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà$7ðð  +ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r,   )rx  r  r|  r}  ru  r~  rv  )NNNNNN)rQ   rR   rS   rT   r/   r   r   r   rJ   rU   rV   rW   rX   s   @@r)   ro  ro  ²  s=   ù‡ € ÷ó ð  ØØ÷5
ñ 5
ó ó ó  ÷5
ð 5
r,   ro  c          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )r$   Úgate_logitsNr  rÆ   r%   )r1   r>   rM   rº   )r'   s   "r)   r*   r*   þ  sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&r,   c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
r€   rñ   )r[   rB   r…   )Ú	minlengthrA   )rˆ   rM   r[   r1   r-  rD   r¼   rC   r   rÏ   rÐ   r2  Úbincountrr   r  rN   Úscatter_add_Úrepeat_interleaver±   )r�  r  r+  rÆ   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsrC  Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r)   Úload_balancing_loss_funcr   þ  sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r,   c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚOlmoeForCausalLMiC  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr<   ÚlogitsÚkeep_full_weightc                óR  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        VP                  V n	        VP                  V n
        VP                  V n        V P                  4        R# rœ   )r.   r/   ro  rX  rv  r   r    r6   r£  Úrouter_aux_loss_coefr  r*  r€  r¦   s   &&€r)   r/   ÚOlmoeForCausalLM.__init__J  s   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Ör,   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r$   rƒ  NrÆ   r�   rí   r„  ÚlabelsrP  Úoutput_router_logitsÚlogits_to_keeprz   r%   )
r1   rQ  r>   r   r…  rR  rº   r   r   r   )r'   r(   s   "€r)   r*   ÚOlmoeForCausalLM.__annotate__X  sÖ   ø€ ÷ Q
ñ Q
á×#Ñ# dÕ*ðQ
ñ Ÿ™ tÕ+ðQ
ñ ×&Ñ&¨Õ-ð	Q
ñ
  �ðQ
ñ ×(Ñ(¨4Õ/ðQ
ñ × Ñ  4Õ'ðQ
ñ ˜$•;ðQ
ñ # T�kðQ
ñ ™eŸl™lÕ*ðQ
ñ Ñ+Õ,ðQ
ñ 
#ñQ
r,   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )u  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, OlmoeForCausalLM

>>> model = OlmoeForCausalLM.from_pretrained("allenai/OLMoE-1B-7B-0924")
>>> tokenizer = AutoTokenizer.from_pretrained("allenai/OLMoE-1B-7B-0924")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
'Hey, are you conscious? Can you talk to me?\nIâ€™m not sure if youâ€™re conscious of this, but Iâ€™m'
```
Nrƒ  rÆ   r�   rí   r„  rP  r¬  r†   )ÚlossÚaux_lossr¥  rí   r<   rZ  r3  r&   )r_   r¬  rX  r‡  rˆ   rº   Úslicer£  Úloss_functionrv  r   r3  r  r*  r¨  rC   r[   r   rí   r<   rZ  )r5   rƒ  rÆ   r�   rí   r„  r«  rP  r¬  r­  rz   Úoutputsr<   Úslice_indicesr¥  r°  r±  s   &&&&&&&&&&,      r)   rJ   ÚOlmoeForCausalLM.forwardV  sx  € ðP %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r,   )r£  rX  r  r*  r¨  rv  )	NNNNNNNNr   )rQ   rR   rS   rT   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr/   r   r   rJ   rU   rV   rW   rX   s   @@r)   r¢  r¢  C  sj   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõ
ð Ø÷Q
ñ Q
ó ó ÷Q
ð Q
r,   r¢  )r0  )rñ   )Nr$   N)r¢  ro  rW  )DÚcollections.abcr   r1   Útorch.nn.functionalr   rÏ   r1  Ú r   r^  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_olmoer   rÉ   r"   rZ   rš   r®   r·   rÀ   rÖ   rØ   r  r(  r:  rF  rW  ro  r   r¢  Ú__all__r&   r,   r)   Ú<module>rÎ     s¤  ðõ& %ã ß Ð Ý å &Ý !ß .Ý )ß hÑ hÝ /Ý 9ß Qß Kß FÝ &ß IÑ IÝ 0ß Gß EÝ ,ñ ˜YÓ'ôJ�2—9‘9ó Jó (ðJô(6<˜2Ÿ9™9ô 6<ôrˆr�y‰yô ò (ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ôM)�R—Y‘Yó M)ó +ðM)ð` ô$#�2—9‘9ó $#ó ð$#ôN<�b—i‘iô <ô*#˜"Ÿ)™)ô #ô &Ð2ô &ðR ôU˜?ó Uó ðUð6 ôH
Ð%ó H
ó ðH
÷VB&ðJ ôe
Ð+¨_ó e
ó ðe
òP E‚r,   