+
    œ°³j+  ã                   óF  € R t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIH	t	H
t
 ^RIHt ^RIHt ^RIHt ^R	IHtHt ^R
IHt ^RIHtHtHt ^RIHt ^RIHt ^RIHtH t H!t!H"t"H#t#H$t$ ^RI%H&t&H't'H(t( ^RI)H*t* ^RI+H,t, ]PZ                  ! ].4      t/ ! R R]!4      t0 ! R R]"4      t1 ! R R]4      t2 ! R R]4      t3 ! R R]&4      t4 ! R R]*4      t5 ! R R]Pl                  4      t7 ! R  R!] 4      t8] ! R" R#]4      4       t9] ! R$ R%](4      4       t: ! R& R']']4      t;. R(Ot<R# ))zPyTorch OLMoE model.)ÚCallableN)Únn)Úinitialization)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)ÚOutputRecorder)ÚGemmaMLP)ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚMixtralExpertsÚMixtralForCausalLMÚMixtralModel)ÚQwen2MoeTopKRouter©ÚOlmoeConfigc                   ó6   a a€ ] tR t^-t oRV 3R lltRtVtV ;t# )ÚOlmoeRMSNormc                ó&   <€ \         SV `  W4       R # ©N)ÚsuperÚ__init__)ÚselfÚhidden_sizeÚepsÚ	__class__s   &&&€Úh/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/olmoe/modular_olmoe.pyr#   ÚOlmoeRMSNorm.__init__.   s   ø€ Ü‰Ñ˜Ö*ó    © )gñhãˆµøä>©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r#   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r'   Ú__classdict__s   @@r(   r   r   -   s   ù‡ € ÷+÷ +r*   r   c                   ó   € ] tR t^2tRtR# )ÚOlmoeRotaryEmbeddingr+   N©r-   r.   r/   r0   r1   r+   r*   r(   r7   r7   2   ó   † Ûr*   r7   c                   ó   € ] tR t^6tRtR# )ÚOlmoeMLPr+   Nr8   r+   r*   r(   r;   r;   6   r9   r*   r;   c                   óX   a a€ ] tR t^:t oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚOlmoeAttentionc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )é   ÚconfigÚ	layer_idxN©r   Úint)Úformatr5   s   "€r(   Ú__annotate__ÚOlmoeAttention.__annotate__;   s   ø€ ÷ 
ñ 
™{ð 
±s¸Tµzñ 
r*   c                ó  <€ \         SV `  W4       \        VP                  VP                  R 7      V n        \        VP                  VP                  ,          VP                  ,          VP                  R 7      V n        R# )©r&   N)	r"   r#   r   r%   Úrms_norm_epsÚq_normÚnum_attention_headsÚnum_key_value_headsÚk_norm©r$   r@   rA   r'   s   &&&€r(   r#   ÚOlmoeAttention.__init__;   s^   ø€ Ü‰Ñ˜Ô+Ü" 6×#5Ñ#5¸6×;NÑ;NÔOˆŒÜ"Ø×Ñ 6×#=Ñ#=Õ=À×A[ÑA[Õ[Ðag×atÑatô
ˆŽr*   c                óR  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r?   Úhidden_statesÚposition_embeddingsÚattention_maskNÚpast_key_valuesÚkwargsÚreturn)ÚtorchÚTensorÚtupler   r   r   )rD   r5   s   "€r(   rE   rF   B   s—   ø€ ÷ /)ñ /)á—|‘|ð/)ñ #¡5§<¡<±·±Ð#=Õ>ð/)ñ Ÿ™ tÕ+ð	/)ñ
  �ð/)ñ Ñ+Õ,ð/)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ/)r*   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      4      pV P	                  V P                  V4      4      p	V P                  V4      p
V P                  P                  eµ   VP                  V P                  P                  ) V P                  P                  R7       V	P                  V P                  P                  ) V P                  P                  R7       V
P                  V P                  P                  ) V P                  P                  R7       VP                  ! V!  P                  ^^4      pV	P                  ! V!  P                  ^^4      p	V
P                  ! V!  P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                   ! V P                  P"                  \$        4      pT! V VV	V
V3RV P&                  '       g   RMV P(                  RV P*                  R\-        V P                  RR 4      /VB w  rïVP.                  ! . VORN5!  P1                  4       pV P3                  V4      pWï3# )N)ÚminÚmaxÚdropoutç        ÚscalingÚsliding_windowéÿÿÿÿ)ÚshapeÚhead_dimrJ   Úq_projrM   Úk_projÚv_projr@   Úclip_qkvÚclamp_ÚviewÚ	transposer   ÚupdaterA   r
   Úget_interfaceÚ_attn_implementationr   ÚtrainingÚattention_dropoutr_   ÚgetattrÚreshapeÚ
contiguousÚo_proj)r$   rQ   rR   rS   rT   rU   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss   &&&&&,          r(   ÚforwardÚOlmoeAttention.forwardB   sI  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>ˆØ—[‘[ §¡¨]Ó!;Ó<ˆ
Ø—{‘{ =Ó1ˆà�;‰;×ÑÒ+Ø×Ñ T§[¡[×%9Ñ%9Ð$9¸t¿{¹{×?SÑ?SÐÔTØ×Ñ 4§;¡;×#7Ñ#7Ð"7¸T¿[¹[×=QÑ=QÐÔRØ×Ñ T§[¡[×%9Ñ%9Ð$9¸t¿{¹{×?SÑ?SÐÔTà#×(Ò(¨,Ñ7×AÑAÀ!ÀQÓGˆØ—_’_ lÑ3×=Ñ=¸aÀÓCˆ
Ø#×(Ò(¨,Ñ7×AÑAÀ!ÀQÓGˆØ&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ô # 4§;¡;Ð0@À$ÓGð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r*   )rM   rJ   r!   ©	r-   r.   r/   r0   r#   r~   r1   r2   r3   r4   s   @@r(   r=   r=   :   s   ù‡ € ÷
õ 
÷/)÷ /)ò /)r*   r=   c                   ó   € ] tR t^ttRtR# )ÚOlmoeExpertsr+   Nr8   r+   r*   r(   r‚   r‚   t   r9   r*   r‚   c                   ó   € ] tR t^xtRtR# )ÚOlmoeTopKRouterr+   Nr8   r+   r*   r(   r„   r„   x   r9   r*   r„   c                   óD   a a€ ] tR t^|t oV 3R ltV3R lR ltRtVtV ;t# )ÚOlmoeSparseMoeBlockc                ód   <€ \         SV `  4        \        V4      V n        \	        V4      V n        R # r!   )r"   r#   r„   Úgater‚   Úexperts©r$   r@   r'   s   &&€r(   r#   ÚOlmoeSparseMoeBlock.__init__}   s&   ø€ Ü‰ÑÔÜ# FÓ+ˆŒ	Ü# FÓ+ˆŽr*   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r?   rQ   rV   )rW   rX   )rD   r5   s   "€r(   rE   Ú OlmoeSparseMoeBlock.__annotate__‚   s#   ø€ ÷ #ñ #¡U§\¡\ð #±e·l±lñ #r*   c                ó´   € VP                   w  r#pVP                  RV4      pV P                  V4      w  rVpV P                  WV4      P	                  W#V4      pV# )é   ra   )rb   ri   rˆ   r‰   rq   )	r$   rQ   Ú
batch_sizeÚsequence_lengthÚ
hidden_dimÚ_Útop_k_weightsÚtop_k_indexÚfinal_hidden_statess	   &&       r(   r~   ÚOlmoeSparseMoeBlock.forward‚   s`   € Ø2?×2EÑ2EÑ/ˆ
 ZØ%×*Ñ*¨2¨zÓ:ˆØ(,¯	©	°-Ó(@Ñ%ˆ˜+Ø"Ÿl™l¨=À}ÓU×]Ñ]Ø¨ó
Ðð #Ð"r*   )r‰   rˆ   r€   r4   s   @@r(   r†   r†   |   s   ù‡ € õ,÷
#÷ #ð #r*   r†   c                   ó>   a a€ ] tR t^Œt oV3R lV 3R lltRtVtV ;t# )ÚOlmoeDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r?   r@   rA   rB   )rD   r5   s   "€r(   rE   ÚOlmoeDecoderLayer.__annotate__�   s   ø€ ÷ bñ b™{ð b±sñ br*   c                ó"  <€ \         SV `  W4       VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))r@   rA   rH   N)r"   r#   r%   r=   Ú	self_attnr†   Úmlpr   rI   Úinput_layernormÚpost_attention_layernormrN   s   &&&€r(   r#   ÚOlmoeDecoderLayer.__init__�   sl   ø€ Ü‰Ñ˜Ô+Ø!×-Ñ-ˆÔÜ'¨vÔKˆŒÜ& vÓ.ˆŒÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÖ%r*   )r%   rŸ   rž   r    r�   r,   r4   s   @@r(   r™   r™   Œ   s   ù‡ € ÷b÷ bó br*   r™   c                   óž   a € ] tR t^–t o RtRtR.tR.tRtRt	R]
! ]^ R7      R]R]/tRtRt]P$                  ! 4       R	 4       tV 3R
 ltRtV tR# )ÚOlmoePreTrainedModelÚmodelTr™   rT   Úrouter_logits)ÚindexrQ   Ú
attentionsc                óÜ  € \         P                  ! W4       \        V\        4      '       dq   \        P
                  ! VP                  R V P                  P                  R7       \        P
                  ! VP                  R V P                  P                  R7       R# \        V\        4      '       d:   \        P
                  ! VP                  R V P                  P                  R7       R# R# )r^   )ÚmeanÚstdN)r   Ú_init_weightsÚ
isinstancer‚   ÚinitÚnormal_Úgate_up_projr@   Úinitializer_rangeÚ	down_projr„   Úweight)r$   Úmodules   &&r(   r«   Ú"OlmoePreTrainedModel._init_weights¨   s�   € ä×%Ò% dÔ3Ü�fœl×+Ò+Ü�LŠL˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�LŠL˜×)Ñ)°¸¿¹×9VÑ9V×WÜ˜¤×0Ò0Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6S×Tñ 1r*   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©r?   r@   r   )rD   r5   s   "€r(   rE   Ú!OlmoePreTrainedModel.__annotate__–   s   ø‡ ‚ áÑò r*   r+   N)r-   r.   r/   r0   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpar   r„   r™   r=   Ú_can_record_outputsÚ_supports_attention_backendÚ_can_compile_fullgraphrW   Úno_gradr«   Ú__annotate_func__r1   r2   )r5   s   @r(   r£   r£   –   s~   ø‡ € ð  ÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€Nà™¨¸qÔAØÐ*Ø�nðÐð #'ÐØ!Ðà
‡]‚]ƒ_ñUó ðU÷' ƒ r*   r£   c                   óT   a a€ ] tR t^²t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú
OlmoeModelc                ó    <€ V ^8„  d   QhRS[ /# r¶   r   )rD   r5   s   "€r(   rE   ÚOlmoeModel.__annotate__´   s   ø€ ÷ >ñ >™{ñ >r*   c           	     ó¢  <€ \         SV `  V4       \        P                  ! VP                  VP
                  V P                  4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP
                  VP                  R 7      V n        \!        VR7      V n        R# u upi )rH   ©r@   N)r"   r#   r   Ú	EmbeddingÚ
vocab_sizer%   Úpadding_idxÚembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr™   Úlayersr   rI   Únormr7   Ú
rotary_embrN   s   && €r(   r#   ÚOlmoeModel.__init__´   s›   ø€ Ü‰Ñ˜Ô ÜŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜCHÈ×IaÑIaÔCbÓcÑCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŽùò ds   Á3Cc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r?   Ú	input_idsNrS   Úposition_idsrT   Úinputs_embedsÚ	use_cacherU   rV   )	rW   Ú
LongTensorrX   r   ÚFloatTensorÚboolr   r   r	   )rD   r5   s   "€r(   rE   rÆ   ½   s“   ø€ ÷ 5
ñ 5
á×#Ñ# dÕ*ð5
ñ Ÿ™ tÕ+ð5
ñ ×&Ñ&¨Õ-ð	5
ñ
  �ð5
ñ ×(Ñ(¨4Õ/ð5
ñ ˜$•;ð5
ñ Ñ+Õ,ð5
ñ 
 ñ5
r*   c                ó†  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£4      pV P                  R V P                  P                    F  pV! V
3RVRV	RVRVR	V/VB p
K  	  V P                  V
4      p
\        V
VR
7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrÈ   )Údevice)r@   r×   rS   rT   rÖ   rR   rS   rÖ   rT   rØ   )Úlast_hidden_staterT   )Ú
ValueErrorr   r@   rÌ   Úget_seq_lengthrW   Úarangerb   rÝ   Ú	unsqueezer   rÒ   rÐ   rÏ   rÑ   r	   )r$   rÕ   rS   rÖ   rT   r×   rØ   rU   Úpast_seen_tokensÚcausal_maskrQ   rR   Údecoder_layers   &&&&&&&,     r(   r~   ÚOlmoeModel.forward½   se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆð #Ÿo™o¨mÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà$7ðð  +ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r*   )rÌ   rÐ   rÑ   rÒ   )NNNNNNr€   r4   s   @@r(   rÄ   rÄ   ²   s   ù‡ € ÷>ó >÷5
÷ 5
ò 5
r*   rÄ   c                   óF   a a€ ] tR t^õt oRR/tV 3R ltV 3R ltRtVtV ;t	# )ÚOlmoeForCausalLMzlm_head.weightzmodel.embed_tokens.weightc                óh   <€ \         SV `  V4       \        V4      V n        VP                  V n        R # r!   )r"   r#   rÄ   r¤   Únum_expertsrŠ   s   &&€r(   r#   ÚOlmoeForCausalLM.__init__ø   s*   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø!×-Ñ-ˆÖr*   c                ó$   <€ \         SV `  ! R/ VB # )u  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, OlmoeForCausalLM

>>> model = OlmoeForCausalLM.from_pretrained("allenai/OLMoE-1B-7B-0924")
>>> tokenizer = AutoTokenizer.from_pretrained("allenai/OLMoE-1B-7B-0924")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
'Hey, are you conscious? Can you talk to me?\nIâ€™m not sure if youâ€™re conscious of this, but Iâ€™m'
```
r+   )r"   r~   )r$   Úsuper_kwargsr'   s   &,€r(   r~   ÚOlmoeForCausalLM.forwardý   s   ø€ ô0 ‰wŠÑ. Ñ.Ð.r*   )r¤   rê   )
r-   r.   r/   r0   Ú_tied_weights_keysr#   r~   r1   r2   r3   r4   s   @@r(   rè   rè   õ   s    ù‡ € Ø*Ð,GÐHÐõ.÷
/õ /r*   rè   )rè   rÄ   r£   )=Ú__doc__Úcollections.abcr   rW   r   Ú r   r­   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr   Úmodeling_outputsr	   Úmodeling_utilsr
   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.output_capturingr   Úgemma.modeling_gemmar   Úllama.modeling_llamar   r   r   r   r   r   Úmixtral.modeling_mixtralr   r   r   Úqwen2_moe.modeling_qwen2_moer   Úconfiguration_olmoer   Ú
get_loggerr-   Úloggerr   r7   r;   r=   r‚   r„   ÚModuler†   r™   r£   rÄ   rè   Ú__all__r+   r*   r(   Ú<module>r     s  ðñ å $ã Ý å &ß .Ý )Ý /Ý 6ß FÝ &ß @Ñ @Ý 4Ý +÷÷ ÷ XÑ WÝ =Ý ,ð 
×	Ò	˜HÓ	%€ô+�<ô +ô
	Ð/ô 	ô	ˆxô 	ô7)�^ô 7)ôt	�>ô 	ô	Ð(ô 	ô#˜"Ÿ)™)ô #ô bÐ)ô bð ôU˜?ó Uó ðUð6 ô?
�ó ?
ó ð?
ôD /Ð)¨?ô  /òF E‚r*   