+
    œ°³j‡�  ã                   óÐ  € ^ RI Ht ^ RIHt ^ RIt^ RIHu Ht ^ RIHt ^RI	H
t ^RIHt ^RIHtHt ^RIHt ^R	IHtHtHt ^R
IHtHt ^RIHtHt ^RIHt ^RIH t H!t!H"t" ^RI#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+  ! R R]PX                  4      t- ! R R]PX                  4      t. ! R R]PX                  4      t/ ! R R]PX                  4      t0R R lt1 ! R R]PX                  4      t2 ! R R]PX                  4      t3R  t4R=R! lt5R>R" R# llt6 ! R$ R%]PX                  4      t7 ! R& R']PX                  4      t8 ! R( R)]4      t9]! ! R* R+]4      4       t:]!! R,R-7       ! R. R/]:4      4       t;]!! R0R-7       ! R1 R2]:4      4       t<]!! R3R-7      ] ! R4 R5]4      4       4       t= ! R6 R7]PX                  4      t> ! R8 R9]PX                  4      t?]!! R:R-7       ! R; R<]:4      4       t@. R?OtAR# )@é    )ÚCallable)Ú	dataclassN)Únn)Úinitialization)ÚACT2FN)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚtorch_compilable_check)Údeprecate_kwarg)Úcan_return_tupleÚmaybe_autocast)Úcapture_outputs©ÚNeoMMEConfigc                   óf   a a€ ] tR t^+t oRV3R lV 3R llltV3R lR ltV3R lR ltRtVtV ;t	# )	ÚNeoMMERMSNormc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )é   ÚdimÚepsÚ
with_scale)ÚintÚfloatÚbool)ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/neomme/modeling_neomme.pyÚ__annotate__ÚNeoMMERMSNorm.__annotate__,   s'   ø€ ÷ Lñ L™Cð L¡eð LÁñ Ló    c                óÆ   <€ \         SV `  4        W n        W0n        V P                  '       d4   \        P
                  ! \        P                  ! V4      R R7      V n        R# R# )T)Úrequires_gradN)	ÚsuperÚ__init__r!   r"   r   Ú	ParameterÚtorchÚonesÚweight)Úselfr    r!   r"   Ú	__class__s   &&&&€r(   r/   ÚNeoMMERMSNorm.__init__,   s@   ø€ Ü‰ÑÔØŒØ$Œà�?�?ˆ?ÜŸ,š,¤u§z¢z°#£ÀdÔKˆDŽKñ r+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   Úhidden_states©r1   ÚTensor)r&   r'   s   "€r(   r)   r*   4   s   ø€ ÷ =ñ =¡5§<¡<ñ =r+   c                ó¤   € VP                  ^4      P                  RRR7      V P                  ,           pV\        P                   ! VR4      ,          # )r   T©Úkeepdiméÿÿÿÿç      à¿)ÚpowÚmeanr!   r1   )r4   r8   Úmean_squareds   && r(   Ú_normÚNeoMMERMSNorm._norm4   sA   € Ø$×(Ñ(¨Ó+×0Ñ0°¸TÐ0ÓBÀTÇXÁXÕMˆàœuŸyšy¨°tÓ<Õ<Ð<r+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r8   Úreturnr9   )r&   r'   s   "€r(   r)   r*   9   s#   ø€ ÷ 4ñ 4¡U§\¡\ð 4±e·l±lñ 4r+   c                óÆ   € V P                  VP                  4       4      pV P                  '       d!   W P                  P                  4       ,          pVP	                  V4      # ©N)rC   r$   r"   r3   Útype_as)r4   r8   Únormed_outputs   && r(   ÚforwardÚNeoMMERMSNorm.forward9   sH   € ØŸ
™
 =×#6Ñ#6Ó#8Ó9ˆØ�?�?ˆ?Ø)¯K©K×,=Ñ,=Ó,?Õ?ˆMØ×$Ñ$ ]Ó3Ð3r+   )r!   r3   r"   )g�íµ ÷Æ°>T)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r/   rC   rK   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r5   r'   s   @@r(   r   r   +   s*   ù‡ € ÷Lõ L÷=ð =÷
4÷ 4ð 4r+   r   c                   óT   a a€ ] tR t^@t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚNeoMMEEmbeddingszZFactorized (ALBERT-style) token embeddings: `vocab_size -> embedding_rank -> hidden_size`.c                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfigr   )r&   r'   s   "€r(   r)   ÚNeoMMEEmbeddings.__annotate__C   s   ø€ ÷ eñ e™|ñ er+   c                óè   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R R7      V n	        R# ©F©ÚbiasN)
r.   r/   r   Ú	EmbeddingÚ
vocab_sizeÚembedding_rankÚword_embeddingsÚLinearÚhidden_sizeÚembedding_projection©r4   rY   r5   s   &&€r(   r/   ÚNeoMMEEmbeddings.__init__C   sO   ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?TÑ?TÓUˆÔÜ$&§I¢I¨f×.CÑ.CÀV×EWÑEWÐ^cÔ$dˆÖ!r+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Ú	input_idsrF   )r1   Ú
LongTensorr:   )r&   r'   s   "€r(   r)   rZ   H   s)   ø€ ÷ Jñ J¡×!1Ñ!1ð J±e·l±lñ Jr+   c                óB   € V P                  V P                  V4      4      # rH   ©re   rb   )r4   ri   s   &&r(   rK   ÚNeoMMEEmbeddings.forwardH   s   € Ø×(Ñ(¨×)=Ñ)=¸iÓ)HÓIÐIr+   rl   ©
rM   rN   rO   rP   Ú__doc__r/   rK   rQ   rR   rS   rT   s   @@r(   rV   rV   @   s&   ù‡ € Ùd÷eó e÷
J÷ Jð Jr+   rV   c                   óT   a a€ ] tR t^Lt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚNeoMMEPatchEmbeddingsz<Patch stem that maps flattened image patches to hidden size.c                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   Ú"NeoMMEPatchEmbeddings.__annotate__O   s   ø€ ÷ Zñ Z™|ñ Zr+   c                ó†  <€ \         SV `  4        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  ^,          RR7      V n        \        P                  ! 4       V n
        \        P                  ! VP                  ^,          VP                  RR7      V n        R# )r   Fr]   TN)r.   r/   r   Ú	LayerNormÚ	patch_dimÚnormrc   rd   Úup_projÚGELUÚact_fnÚ	down_projrf   s   &&€r(   r/   ÚNeoMMEPatchEmbeddings.__init__O   s}   ø€ Ü‰ÑÔÜ—L’L ×!1Ñ!1Ó2ˆŒ	Ü—y’y ×!1Ñ!1°6×3EÑ3EÈÕ3IÐPUÔVˆŒÜ—g’g“iˆŒÜŸš 6×#5Ñ#5¸Õ#9¸6×;MÑ;MÐTXÔYˆŽr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úpixel_valuesrF   r9   )r&   r'   s   "€r(   r)   rs   V   s#   ø€ ÷ :ñ :¡E§L¡Lð :±U·\±\ñ :r+   c                ó‚   € V P                  V P                  V4      4      pV P                  V P                  V4      4      # rH   )rx   rw   r{   rz   )r4   r~   r8   s   && r(   rK   ÚNeoMMEPatchEmbeddings.forwardV   s1   € ØŸ™ T§Y¡Y¨|Ó%<Ó=ˆØ�~‰~˜dŸk™k¨-Ó8Ó9Ð9r+   )rz   r{   rw   rx   rn   rT   s   @@r(   rq   rq   L   s#   ù‡ € ÙF÷Zó Z÷:÷ :ð :r+   rq   c                   óè   a a€ ] tR t^[t oRt]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]	P                  ! 4       ]RV3R	 lR
 ll4       4       tR tRtVtV ;t# )ÚNeoMMERotaryEmbeddingzBTwo-axis interleaved M-RoPE with per-layer-type frequency spectra.Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   Ú"NeoMMERotaryEmbedding.__annotate___   s   ø€ ÷ Uñ U™|ñ Ur+   c           
     óÞ  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n        / V n	        V P                   Fû  pV P
                  P                  V,          pVf   K&  VR,          V P                  V&   V P                  pV P                  V,          R8w  d   \        V P                  V,          ,          pV! V P
                  W#R7      w  rg\        W R2\        P                  ! VRR7      4       \        W R2\        P                  ! VP!                  4       RR7      4       \        W R2V4       Ký  	  R # )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r.   r/   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrY   ÚsortedÚsetÚlayer_typesrˆ   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr   ÚBufferÚclone)	r4   rY   rƒ   r‹   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr5   s	   &&&     €r(   r/   ÚNeoMMERotaryEmbedding.__init__^   s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà)4°[Õ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÕ)¨YÔ6Ü2°4·>±>À*Õ3MÕN�Ù4@ÀÇÁÈfÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó +r+   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r   rY   r‹   NrF   )r   ÚstrÚtupler1   r:   r$   )r&   r'   s   "€r(   r)   r†   v   s7   ø€ ÷ 5ñ 5Ùð5Ù7:¸Tµzð5á	‰u�|‰|™UÐ"Õ	#ñ5r+   c           	     ó¦  € V P                   V,          R,          pV P                   V,          P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        We,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          p	V	P                  V4      V3# )a9  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`, *optional*):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorç      ð?Úhead_dimN)Údtype)
r–   ÚgetÚgetattrrd   Únum_attention_headsr#   r1   Úaranger$   Úto)
rY   rƒ   r‹   ÚkwargsÚbaser¥   r§   r    Úattention_factorÚinv_freqs
   &&&,      r(   r—   Ú5NeoMMERotaryEmbedding.compute_default_rope_parameterst   s²   € ð" ×%Ñ% jÕ1°,Õ?ˆà &× 6Ñ 6°zÕ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐð ˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r+   c          
      ó¦   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[R,          RS[S[ P                  S[ P                  3,          /# )r   ÚxÚposition_idsr‹   NrF   )r1   r:   rj   r¡   r¢   )r&   r'   s   "€r(   r)   r†   “   sN   ø€ ÷ 0ñ 0Ù—‘ð0Ù-2×-=Ñ-=ð0ÙKNÐQUÍ:ð0á	‰u�|‰|™UŸ\™\Ð)Õ	*ñ0r+   c                ó4  € \        W R 24      p\        W R24      pVR,          P                  4       P                  ^VP                  ^,          R^4      pVR	,          P                  4       p\	        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	V	P                  4       V,          p
V	P                  4       V,          pRRR4       V P                  X
4      p
V P                  X4      pV
P                  VP                  4      VP                  VP                  4      3#   + '       g   i     Lh; i)
rŒ   r�   NÚmpsÚcpuF)Údevice_typeÚenabled)NNºNNNNr>   )r»   r»   Nr»   )rª   r$   ÚexpandÚshapeÚ
isinstancerƒ   Útyper¡   r   Ú	transposeÚcosÚsinÚrecomposition_frequenciesr­   r¨   )r4   r´   rµ   r‹   r±   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr¹   ÚfreqsrÁ   rÂ   s   &&&&        r(   rK   ÚNeoMMERotaryEmbedding.forward‘   s=  € ô
 ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐà$Ð%8Õ9×?Ñ?ÓA×HÑHÈÈL×L^ÑL^Ð_`ÕLaÐceÐghÓiÐØ ,¨]Õ ;× AÑ AÓ CÐä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCà&Õ>×IÑIÈ!ÈQÓOˆEØ—)‘)“+Ð 1Õ1ˆCØ—)‘)“+Ð 1Õ1ˆC÷	 Dð ×,Ñ,¨SÓ1ˆØ×,Ñ,¨SÓ1ˆØ�v‰v�a—g‘g‹ §¡ q§w¡w£Ð/Ð/÷ D×Cús   Ã AFÆF	c                óÂ   € V^ ,          R,          V^,          R,          r2\         P                  ! W#.RR7      P                  R4      p\         P                  ! WD.RR7      # )zM
Recompose the frequencies into the final spatial layout used per each grid.
©r    ).:r   Nr   ).:é   Nr   r>   éþÿÿÿ)r1   ÚstackÚflattenÚcat)r4   ÚfreqÚfreq_rowÚfreq_colÚangless   &&   r(   rÃ   Ú/NeoMMERotaryEmbedding.recomposition_frequencies§   sO   € ð " !�W YÕ/°°aµ¸Õ1C�(Ü—’˜hÐ1°rÔ:×BÑBÀ2ÓFˆÜ�yŠy˜&Ð)¨rÔ2Ð2r+   )rY   r•   r‘   r’   rˆ   rH   ©NN)rM   rN   rO   rP   ro   r   r/   Ústaticmethodr—   r1   Úno_gradr   rK   rÃ   rQ   rR   rS   rT   s   @@r(   r‚   r‚   [   s‚   ù‡ € ÙLá�X vÔ.÷Uô Uó /ðUð* Ù�X vÔ.÷5ñ 5ó /ó ð5ð6 ‡]‚]ƒ_Ø÷0ñ 0ó ó ð0÷(3ò 3r+   r‚   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r   r8   Ún_reprF   ©r1   r:   r#   )r&   s   "r(   r)   r)   ²   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur+   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r»   r»   Nr»   r»   )r½   r¼   Úreshape)r8   rÙ   ÚbatchÚnum_key_value_headsÚslenr§   s   &&    r(   Ú	repeat_kvrà   ²   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr+   c                   óP   a a€ ] tR t^¾t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚNeoMMEExclusiveSelfAttentionc                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   Ú)NeoMMEExclusiveSelfAttention.__annotate__¿   s   ø€ ÷ Kñ K™|ñ Kr+   c                óÚ   <€ \         SV `  4        VP                  VP                  ,          V n        \
        P                  ! \        P                  ! VP                  4      4      V n	        R # rH   )
r.   r/   r«   rÞ   Únum_key_value_groupsr   r0   r1   ÚzerosÚalpharf   s   &&€r(   r/   Ú%NeoMMEExclusiveSelfAttention.__init__¿   sF   ø€ Ü‰ÑÔØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ü—\’\¤%§+¢+¨f×.HÑ.HÓ"IÓJˆŽ
r+   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   Úattn_outputÚvalue_statesrF   r9   )r&   r'   s   "€r(   r)   rä   Ä   s1   ø€ ÷ ?ñ ?¡5§<¡<ð ?¹u¿|¹|ð ?ÑPU×P\ÑP\ñ ?r+   c                ó¾  € \        W P                  4      P                  ^^4      p\        P                  ! VP                  4       RR7      P                  VP                  4      pW,          P                  RRR7      p\        P                  ! V P                  4      P                  VP                  4      P                  ^^R^4      pWV,          V,          ,
          # )rË   rÊ   Tr<   r>   )rà   ræ   rÀ   ÚFÚ	normalizer$   r­   r¨   Úsumr1   Útanhrè   Úview)r4   rë   rì   Ú
value_unitÚ
projectionÚscales   &&&   r(   rK   Ú$NeoMMEExclusiveSelfAttention.forwardÄ   sª   € Ü  ×/HÑ/HÓI×SÑSÐTUÐWXÓYˆÜ—[’[ ×!3Ñ!3Ó!5¸2Ô>×AÑAÀ+×BSÑBSÓTˆ
Ø!Õ.×3Ñ3°BÀÐ3ÓEˆ
Ü—
’
˜4Ÿ:™:Ó&×)Ñ)¨+×*;Ñ*;Ó<×AÑAÀ!ÀQÈÈAÓNˆØ jÕ0°JÕ>Õ>Ð>r+   )rè   ræ   ©	rM   rN   rO   rP   r/   rK   rQ   rR   rS   rT   s   @@r(   râ   râ   ¾   s    ù‡ € ÷Kó K÷
?÷ ?ð ?r+   râ   c                   óP   a a€ ] tR t^Ìt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚNeoMMESigmoidGatedProjectionc                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   Ú)NeoMMESigmoidGatedProjection.__annotate__Í   s   ø€ ÷ añ a™|ñ ar+   c                ó  <€ \         SV `  4        VP                  VP                  ,          p\        P
                  ! VP                  VR R7      V n        \        P
                  ! W!P                  VP                  R7      V n	        R# r\   )
r.   r/   r«   r§   r   rc   rd   Ú	gate_projÚattention_biasÚo_proj)r4   rY   Úprojection_sizer5   s   && €r(   r/   Ú%NeoMMESigmoidGatedProjection.__init__Í   sZ   ø€ Ü‰ÑÔØ ×4Ñ4°v·±ÕFˆÜŸš 6×#5Ñ#5°ÈUÔSˆŒÜ—i’i ×1CÑ1CÈ&×J_ÑJ_Ô`ˆŽr+   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   rë   r8   rF   r9   )r&   r'   s   "€r(   r)   rû   Ó   s6   ø€ ÷ Wñ W¡5§<¡<ð WÁÇÁð WÑQV×Q]ÑQ]ñ Wr+   c                óx   € V P                  V\        P                  ! V P                  V4      4      ,          4      # rH   )rÿ   r1   Úsigmoidrý   )r4   rë   r8   s   &&&r(   rK   Ú$NeoMMESigmoidGatedProjection.forwardÓ   s(   € Ø�{‰{˜;¬¯ª°t·~±~ÀmÓ7TÓ)UÕUÓVÐVr+   )rý   rÿ   r÷   rT   s   @@r(   rù   rù   Ì   s#   ù‡ € ÷aó a÷W÷ Wð Wr+   rù   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..NrÊ   r>   )r½   r1   rÏ   )r´   Úx1Úx2s   &  r(   Úrotate_halfr	  ×   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r+   c                ó¶  € VP                  V4      pVP                  V4      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜Wb,          \        V4      V,          ,           p
W‚,          \        V4      V,          ,           p\        P                  ! W§.RR7      p
\        P                  ! W¹.RR7      pW«3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.NrÊ   r>   )Ú	unsqueezer½   r	  r1   rÏ   )ÚqÚkrÁ   rÂ   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds   &&&&&       r(   Úapply_rotary_pos_embr  Þ   sÐ   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð —‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ð �{œ{¨5Ó1°CÕ7Õ8€GØ�{œ{¨5Ó1°CÕ7Õ8€Gô �iŠi˜Ð)¨rÔ2€GÜ�iŠi˜Ð)¨rÔ2€GØÐÐr+   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutr®   )r   ÚModuler1   r:   r$   r   r   )r&   s   "r(   r)   r)     sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r+   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r   )r    r¨   )ÚpÚtrainingr>   )rà   ræ   r1   ÚmatmulrÀ   r   Ú
functionalÚsoftmaxÚfloat32r­   r¨   r  r"  Ú
contiguous)r  r  r  r  r  r  r  r®   Ú
key_statesrì   Úattn_weightsrë   s   &&&&&&&,    r(   Úeager_attention_forwardr*    sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r+   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚNeoMMEAttentioni  zéBidirectional grouped-query attention with QK-norm, M-RoPE, and a sigmoid output gate.

QK-norm runs before rotary embedding, value embeddings are added after rotation, and
exclusive self-attention is applied before the output gate.
c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   rY   Ú	layer_idx©r   r#   )r&   r'   s   "€r(   r)   ÚNeoMMEAttention.__annotate__"  s   ø€ ÷ "Fñ "F™|ð "F¹ñ "Fr+   c                óæ  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        VP&                  V,          R8H  V n        V P(                  '       d%   VP*                  V,          P,                  ^,           MRV n        VP&                  V,          V n        VP                  V n        \1        VP                  VP2                  RR7      V n        \1        VP                  VP2                  RR7      V n        \9        V4      V n        \=        V4      V n        R# )r§   Fr]   Úsliding_attentionN©r"   r?   ) r.   r/   rY   r/  rª   rd   r«   r§   rÞ   ræ   r  Úattention_dropoutÚ	is_causalr   rc   rþ   Úq_projÚk_projÚv_projr•   Úis_local_attentionÚper_layer_configÚsliding_windowÚattention_typer   Únorm_epsÚq_normÚk_normrâ   Úexclusive_self_attentionrù   Úoutput_projection©r4   rY   r/  r5   s   &&&€r(   r/   ÚNeoMMEAttention.__init__"  sÑ  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒð
 #)×"4Ñ"4°YÕ"?ÐCVÑ"VˆÔð
 FJ×E\×E\ÐE\ˆF×#Ñ# IÕ.×=Ñ=ÀÖAÐbfð 	Ôð %×0Ñ0°Õ;ˆÔØ#)×#=Ñ#=ˆÔ Ü# F§O¡O°V·_±_ÐQVÔWˆŒÜ# F§O¡O°V·_±_ÐQVÔWˆŒÜ(DÀVÓ(LˆÔ%Ü!=¸fÓ!EˆÖr+   c                ó0  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r   r8   Úposition_embeddingsr  NÚvalue_embedsr®   rF   ©r1   r:   r¢   r   r   )r&   r'   s   "€r(   r)   r1  F  sŒ   ø€ ÷ *)ñ *)á—|‘|ð*)ñ #¡5§<¡<±·±Ð#=Õ>ð*)ñ Ÿ™ tÕ+ð	*)ñ
 —l‘l TÕ)ð*)ñ Ñ+Õ,ð*)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ*)r+   c           	     ó„  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      pV P                  V	4      p	Vw  r¼\        W‰W¼4      w  r‰Ve(   W¤P                  V4      P	                  ^^4      ,           p
\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       d   V P                   MRRV P"                  RV P$                  /VB w  rïV P'                  Wê4      pVP(                  ! . VORN5!  pV P+                  Wá4      pWï3# )Nr  ç        r  r<  r>   )r½   r§   r7  rò   rÀ   r8  r9  r?  r@  r  r   Úget_interfacerY   Ú_attn_implementationr*  r"  r5  r  r<  rA  rÜ   rB  )r4   r8   rF  r  rG  r®   Úinput_shapeÚhidden_shapeÚquery_statesr(  rì   rÁ   rÂ   Úattention_interfacerë   r)  s   &&&&&,          r(   rK   ÚNeoMMEAttention.forwardF  sÂ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆØÒ#Ø'×*;Ñ*;¸LÓ*I×*SÑ*SÐTUÐWXÓ*YÕYˆLä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð /3¯m¯m¨m�D×*Ò*Àð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð ×3Ñ3°KÓNˆØ!×)Ò)Ð;¨;Ð;¸Ó;ˆØ×,Ñ,¨[ÓHˆØÐ(Ð(r+   )r5  r=  rY   rA  r§   r6  r:  r@  r8  r/  r«   ræ   rB  r?  r7  r  r<  r9  rÕ   rn   rT   s   @@r(   r,  r,    s&   ù‡ € ñ÷"Fó "F÷H*)÷ *)ò *)r+   r,  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )Ú	NeoMMEMLPis  c                óœ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  VP                  R 7      V n        \
        P                  ! V P                  V P                  VP                  R 7      V n	        \        VP                  ,          V n        R# )r]   N)r.   r/   rY   rd   Úintermediate_sizer   rc   Úmlp_biasrx   r{   r   Ú
hidden_actrz   rf   s   &&€r(   r/   ÚNeoMMEMLP.__init__t  s�   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Õ/ˆŽr+   c                ó`   € V P                  V P                  V P                  V4      4      4      # rH   )r{   rz   rx   )r4   r´   s   &&r(   rK   ÚNeoMMEMLP.forward}  s"   € Ø�~‰~˜dŸk™k¨$¯,©,°q«/Ó:Ó;Ð;r+   )rz   rY   r{   rd   rU  rx   r÷   rT   s   @@r(   rS  rS  s  s   ù‡ € õ0÷<ò <r+   rS  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚNeoMMEEncoderLayeri�  zGPre-norm encoder layer with initial-state mixing and muP depth scaling.c                ó&   <€ V ^8„  d   QhRS[ RS[/# r.  r0  )r&   r'   s   "€r(   r)   ÚNeoMMEEncoderLayer.__annotate__„  s   ø€ ÷ <ñ <™|ð <¹ñ <r+   c                ó´  <€ \         SV `  4        \        W4      V n        \	        V4      V n        \        P                  ! \        P                  ! R R.4      4      V n
        \        VP                  VP                  RR7      V n        \        VP                  VP                  RR7      V n        VP                   V n        VP"                  V,          V n        R# )r¦   rJ  Fr4  N)r.   r/   r,  Ú	self_attnrS  Úmlpr   r0   r1   ÚtensorÚlambdasr   rd   r>  Úinput_layernormÚpost_attention_layernormÚresidual_multiplierr•   r=  rC  s   &&&€r(   r/   ÚNeoMMEEncoderLayer.__init__„  sœ   ø€ Ü‰ÑÔÜ(¨Ó;ˆŒÜ˜VÓ$ˆŒÜ—|’|¤E§L¢L°#°s°Ó$<Ó=ˆŒÜ,¨V×-?Ñ-?ÀÇÁÐ]bÔcˆÔÜ(5°f×6HÑ6HÈ&Ï/É/ÐfkÔ(lˆÔ%Ø#)×#=Ñ#=ˆÔ Ø$×0Ñ0°Õ;ˆÖr+   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[S[,          RS[ P                  /# )	r   r8   Úinitial_hidden_statesrG  NrF  r  r®   rF   rH  )r&   r'   s   "€r(   r)   r^  Ž  s•   ø€ ÷ Eñ Eá—|‘|ðEñ  %Ÿ|™|ðEñ —l‘l TÕ)ð	Eñ
 #¡5§<¡<±·±Ð#=Õ>ÀÕEðEñ Ÿ™ tÕ+ðEñ Ñ+Õ,ðEñ 
�‰ñEr+   c           	     ót  € V P                   ^ ,          V,          V P                   ^,          V,          ,           pV P                  V4      pV P                  ! V3RVRVRV/VB w  ršWP                  V	,          ,           pV P	                  V4      pV P                  V4      pWP                  V,          ,           # )r   rF  r  rG  )rc  rd  r`  rf  re  ra  )r4   r8   ri  rG  rF  r  r®   Úmixed_statesÚnormed_statesrë   Ú_Ú
mlp_outputs   &&&&&&,     r(   rK   ÚNeoMMEEncoderLayer.forwardŽ  s¹   € ð —|‘| A•¨Õ6¸¿¹Àa½ÐK`Õ9`Õ`ˆØ×,Ñ,¨\Ó:ˆØŸšØñ
à 3ð
ð *ð
ð &ð	
ð
 ñ
‰ˆð &×(@Ñ(@À;Õ(NÕNˆà×5Ñ5°mÓDˆØ—X‘X˜mÓ,ˆ
Ø×7Ñ7¸*ÕDÕDÐDr+   )r=  rd  rc  ra  re  rf  r`  ©NNNrn   rT   s   @@r(   r\  r\  �  s$   ù‡ € ÙQ÷<ó <÷E÷ Eò Er+   r\  c                   óê   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/tRtV3R lR	 ltV3R
 lR lt]P(                  ! 4       V3R lV 3R ll4       tRV3R lV 3R llltV3R ltRtVtV ;t# )ÚNeoMMEPreTrainedModeli§  ÚmodelTr\  Úpast_key_valuesr8   Ú
attentionsc                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rF   ©r   r_   )r&   r'   s   "€r(   r)   Ú"NeoMMEPreTrainedModel.__annotate__º  s   ø€ ÷ 5ñ 5¡b§l¡lñ 5r+   c                óZ   € \        W P                  V 4      pVP                  P                  # rH   ©rª   Úbase_model_prefixÚembed_tokensrb   )r4   Úbackbones   & r(   Úget_input_embeddingsÚ*NeoMMEPreTrainedModel.get_input_embeddingsº  s&   € Ü˜4×!7Ñ!7¸Ó>ˆØ×$Ñ$×4Ñ4Ð4r+   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# )r   r  rF   Nrw  )r&   r'   s   "€r(   r)   rx  ¾  s   ø€ ÷ 6ñ 6©"¯,©,ð 6¸4ñ 6r+   c                óR   € \        W P                  V 4      pWP                  n        R # rH   rz  )r4   r  r}  s   && r(   Úset_input_embeddingsÚ*NeoMMEPreTrainedModel.set_input_embeddings¾  s    € Ü˜4×!7Ñ!7¸Ó>ˆØ05×ÑÖ-r+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r  )r   r  )r&   r'   s   "€r(   r)   rx  Ã  s   ø€ ÷ Yñ Y¡B§I¡Iñ Yr+   c                óÒ  <€ \         SV `  V4       \        V\        4      '       dK   \        P
                  ! VP                  P                  R V P                  P                  R,          R7       R# \        V\        4      '       d#   \        P                  ! VP                  4       R# \        V\        4      '       d-   \        P                  ! VP                  P                  4       R# \        V\        4      '       d-   \        P                  ! VP                   P                  4       R# \        V\"        4      '       d:   \        P$                  ! VP&                  \(        P*                  ! RR .4      4       R# \        V\,        4      '       d-   \        P                  ! VP.                  P                  4       R# \        V\0        4      '       dK   \        P
                  ! VP2                  P                  R V P                  P                  R,          R7       R# \        V\4        4      '       d´   VP6                   F¡  pVP8                  pVP:                  V,          R8w  d   \<        VP:                  V,          ,          pV! VP                  VR7      w  rE\        P$                  ! \?        W R24      V4       \        P$                  ! \?        W R24      V4       K£  	  R# R# )	rJ  )rA   Ústdr¦   r‰   rŠ   rŒ   rŽ   Nr?   ) r.   Ú_init_weightsr¾   rV   ÚinitÚnormal_rb   r3   rY   ra   râ   Úzeros_rè   rù   rÿ   rS  r{   r\  Úcopy_rc  r1   rb  ÚNeoMMEModelÚvalue_embeddingsÚNeoMMEForMaskedLMÚlm_headr‚   r•   r—   rˆ   r   rª   )r4   r  r‹   rœ   r±   rm  r5   s   &&    €r(   r‡  Ú#NeoMMEPreTrainedModel._init_weightsÂ  sÚ  ø€ ô 	‰Ñ˜fÔ%ä�fÔ.×/Ò/Ü�LŠL˜×/Ñ/×6Ñ6¸SÀdÇkÁk×F`ÑF`ÐbfÕFf×gÜ˜Ô <×=Ò=Ü�KŠK˜Ÿ™Ö%Ü˜Ô <×=Ò=ä�KŠK˜Ÿ™×,Ñ,Ö-Ü˜¤	×*Ò*Ü�KŠK˜×(Ñ(×/Ñ/Ö0Ü˜Ô 2×3Ò3Ü�JŠJ�v—~‘~¤u§|¢|°S¸#°JÓ'?Ö@Ü˜¤×,Ò,Ü�KŠK˜×/Ñ/×6Ñ6Ö7Ü˜Ô 1×2Ò2Ü�LŠL˜Ÿ™×.Ñ.°S¸d¿k¹k×>XÑ>XÐZ^Õ>^×_Ü˜Ô 5×6Ò6Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ*¨6¯=©=ÀZÔP‘�Ü—
’
œ7 6¨\¸Ð+CÓDÀhÔOÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈxÖXó 1ñ 7r+   c                óT   <€ V ^8„  d   QhRS[ RS[ R,          RS[RS[P                  /# )r   Únew_num_tokensÚpad_to_multiple_ofNÚmean_resizingrF   )r#   r%   r   r_   )r&   r'   s   "€r(   r)   rx  ß  s5   ø€ ÷ ñ Ù!ðÙ7:¸TµzðÙY]ðá	�‰ñr+   c                ó\  <€ \         SV `  WV4      p\        W P                  V 4      pV P	                  VP
                  VP                  P                  ^ ,          VR7      p\        P                  ! VP                  VP                  4      Vn        VP                  VP
                  n        V# )z0Resize word and value embedding tables together.)r”  )r.   Ú_resize_token_embeddingsrª   r{  Ú_get_resized_embeddingsr�  r3   r½   r   r_   Únum_embeddingsÚembedding_dim)r4   r’  r“  r”  rb   r}  Úresizedr5   s   &&&&   €r(   r–  Ú.NeoMMEPreTrainedModel._resize_token_embeddingsß  s–   ø€ ô  ™'Ñ:¸>Ð_lÓmˆÜ˜4×!7Ñ!7¸Ó>ˆØ×.Ñ.Ø×%Ñ% ×'=Ñ'=×'CÑ'CÀAÕ'FÐVcð /ó 
ˆô %'§L¢L°×1GÑ1GÈ×I^ÑI^Ó$_ˆÔ!Ø+2¯>©>ˆ×!Ñ!Ô(ØÐr+   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rX   r   )r&   r'   s   "€r(   r)   rx  §  s   ø‡ ‚ áÑò r+   © )ÚimageÚtext)NT)rM   rN   rO   rP   r{  Úsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr\  r,  Ú_can_record_outputsÚinput_modalitiesr~  r‚  r1   r×   r‡  r–  Ú__annotate_func__rQ   rR   rS   rT   s   @@r(   rr  rr  §  s¥   ù‡ € ð  ÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ+Ø�oðÐð )Ð÷5ð 5÷6ð 6ð ‡]‚]ƒ_÷Yò Yó ðY÷8õ ÷q … r+   rr  zu
    The bare NeoMME model. It encodes text tokens and image patches with one bidirectional Transformer encoder.
    ©Úcustom_introc                   ó®   a a€ ] tR tRt oV3R lV 3R llt]]! RR7      V3R lR l4       4       tV3R lR	 lt]	]RV3R
 lR ll4       4       t
RtVtV ;t# )rŒ  ií  c                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   ÚNeoMMEModel.__annotate__ó  s   ø€ ÷ ñ ™|ñ r+   c           	     ó  <€ \         SV `  V4       \        V4      V n        \	        V4      V n        \        V4      V n        \        VP                  VP                  R R7      V n        \        VP                  VP                  R R7      V n        \        P                  ! \        VP                   4       Uu. uF  p\#        W4      NK  	  up4      V n        \'        VP(                  4       UUu. uF  w  r4VR8X  g   K  VNK  	  ppp\        P*                  ! VP,                  VP.                  VP0                  ,          4      V n        V^ ,          VR,          0V n        R V n        V P9                  4        R# u upi u uppi )Fr4  Úfull_attentionNr>   )r.   r/   rV   r|  rq   Úpatch_embeddingsr‚   Ú
rotary_embr   rd   r>  Úembedding_normÚ
final_normr   Ú
ModuleListÚrangeÚnum_hidden_layersr\  ÚlayersÚ	enumerater•   r_   r`   rÞ   r§   r�  Úvalue_embedding_layersÚgradient_checkpointingÚ	post_init)r4   rY   r/  Úir‹   Úglobal_layersr5   s   &&    €r(   r/   ÚNeoMMEModel.__init__ó  s2  ø€ Ü‰Ñ˜Ô Ü,¨VÓ4ˆÔÜ 5°fÓ =ˆÔÜ/°Ó7ˆŒÜ+¨F×,>Ñ,>ÀÇÁÐ\aÔbˆÔÜ'¨×(:Ñ(:¸F¿O¹OÐX]Ô^ˆŒÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô 1:¸&×:LÑ:LÔ0MÔpÑ0M™}˜qÐQ[Ð_oÑQoŸ˜Ñ0MˆÑpÜ "§¢¨V×->Ñ->À×@ZÑ@ZÐ]c×]lÑ]lÕ@lÓ mˆÔØ'4°QÕ'7¸ÀrÕ9JÐ&KˆÔ#Ø&+ˆÔ#Ø�‰Öùò eùó qs   Â6E8Ã.E=Ã>E=z=Projects flattened image patches into the model hidden space.r«  c                óP   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[/# )r   r~   r®   rF   )r1   r:   r   r   r   )r&   r'   s   "€r(   r)   r¯    s4   ø€ ÷ 	jñ 	jÙ!ŸL™Lð	jÙ4:Ñ;MÕ4Nð	já	#ñ	jr+   c           	     óž  € VP                   R,          V P                  P                  8w  dO   \        RVP                   R,           RV P                  P                   RV P                  P                   R24      hV P                  VP                  V P
                  P                  P                  P                  4      4      p\        W3R7      # )rË   zpixel_values has patch width z but the model expects z( (= 3 * patch_size ** 2 with patch_size=Ú))Úlast_hidden_stateÚpooler_outputr>   )r½   rY   rv   Ú
ValueErrorÚ
patch_sizer²  r­   rw   r3   r¨   r   )r4   r~   r®   Úimage_featuress   &&, r(   Úget_image_featuresÚNeoMMEModel.get_image_features  s°   € ð
 ×Ñ˜bÕ! T§[¡[×%:Ñ%:Ô:ÜØ/°×0BÑ0BÀ2Õ0FÐ/GÐG^Ø—;‘;×(Ñ(Ð)Ð)QÐRV×R]ÑR]×RhÑRhÐQiÐijðlóð ð ×.Ñ.¨|¯©¸t×?TÑ?T×?YÑ?Y×?`Ñ?`×?fÑ?fÓ/gÓhˆÜ)¸NÔiÐir+   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   ri   rÈ  rF   )r1   rj   r:   Ú
BoolTensor)r&   r'   s   "€r(   r)   r¯    s;   ø€ ÷ 
Qñ 
Q©e×.>Ñ.>ð 
QÑPU×P\ÑP\ð 
QÑaf×aqÑaqñ 
Qr+   c                óÄ  aa€ \         P                  ! VRRR13,          RV P                  P                  ;'       g    ^ R7      pWP                  P                  8H  W0P                  P
                  8g  ,          pVP                  4       o\        SSP                  ^ ,          8H  VV3R l4       VP                  R4      P                  RRSP                  R,          4      # )zPFind patch placeholders and validate that every image feature has a destination.r»   N)r  c                  óJ   <€ R S P                   ^ ,           R\        S4       R2# )zGot z image patches for z image placeholder tokens)r½   r#   )rÈ  Únum_image_tokenss   €€r(   Ú<lambda>Ú2NeoMMEModel.get_placeholder_mask.<locals>.<lambda>  s)   ø€ �d˜>×/Ñ/°Õ2Ð3Ð3FÄsÐK[ÓG\ÐF]Ð]vÑwr+   r>   )rË   r   )rî   ÚpadrY   Úpad_token_idÚimage_token_idÚdocument_token_idrð   r   r½   r  r¼   )r4   ri   rÈ  Úprevious_idsÚ
image_maskrÏ  s   &&f  @r(   Úget_placeholder_maskÚ NeoMMEModel.get_placeholder_mask  s³   ù€ ä—u’u˜Y q¨#¨2¨# vÕ.°¸d¿k¹k×>VÑ>V×>[Ð>[ÐZ[Ô\ˆØ§;¡;×#=Ñ#=Ñ=À,×R]ÑR]×RoÑRoÑBoÕpˆ
à%Ÿ>™>Ó+ÐÜØ × 4Ñ 4°QÕ 7Ñ7Ýwô	
ð ×#Ñ# BÓ'×.Ñ.¨r°2°~×7KÑ7KÈBÕ7OÓPÐPr+   c                óÈ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )r   ri   r  Nrµ   r~   r®   rF   )r1   rj   r:   r   r   r   )r&   r'   s   "€r(   r)   r¯    s€   ø€ ÷ C@ñ C@á×#Ñ#ðC@ñ Ÿ™ tÕ+ðC@ñ ×&Ñ&¨Õ-ð	C@ñ
 —l‘l TÕ)ðC@ñ Ñ+Õ,ðC@ñ 
ñC@r+   c           	     ó”  € V P                  V4      pVeJ   V P                  VRR7      pV P                  WP                  4      pVP	                  W‡P                  4      pVP
                  R,          w  ršVf2   \        P                  ! W¦P                  R7      P                  V	R4      pVP                  ^8X  d#   VP                  ^ 4      P                  ^RR4      pV P                  V4      ;rk\        T;p\        4      '       gz   / pRVRV/p\        V P                   P"                  4       FN  pV P                   P$                  V,          pVP&                  e   \)        RRV/VB WÎ&   K?  \+        RRV/VB WÎ&   KP  	  \-        V P                   P.                  4       Uu/ uF  pVV P1                  WcV4      bK  	  ppV P3                  V4      p\5        V P6                  4       F@  w  ppT! TTVV P8                  9   d   TMR3R	VVP:                  ,          RVV,          /VB pKB  	  V P=                  V4      p\?        VR
7      # u upi )aþ  
position_ids (`torch.LongTensor` of shape `(2, batch_size, sequence_length)` or `(batch_size, sequence_length)`, *optional*):
    Positions for the input tokens. [`NeoMMEProcessor`] returns two-axis positions for document images. A
    one-axis position tensor is used for text inputs.
pixel_values (`torch.Tensor` of shape `(num_patches, 3 * patch_size ** 2)`, *optional*):
    Flattened image patches returned by [`NeoMMEProcessor`]. The model places these patches at image
    placeholders in `input_ids`.
NT)Úreturn_dictºNr   N)rƒ   Úinputs_embedsr  rY   rF  )rÄ  r>   r�  ) r|  rÉ  rØ  rÅ  Úmasked_scatterr½   r1   r¬   rƒ   r¼   Úndimr  r´  r¾   Údictr·  rY   r¸  r;  r<  r	   r   r”   r•   r³  r�  rº  r¹  r»  r=  rµ  r   )r4   ri   r  rµ   r~   r®   r8   Úimage_outputsr×  Ú
batch_sizeÚseq_lenri  Úattention_mask_mappingÚmask_kwargsÚlayer_idr;  r‹   rF  rG  r/  Úencoder_layers   &&&&&,               r(   rK   ÚNeoMMEModel.forward  sV  € ð$ ×)Ñ)¨)Ó4ˆØÒ#Ø ×3Ñ3°LÈdÐ3ÓSˆMØ×2Ñ2°9×>YÑ>YÓZˆJØ)×8Ñ8¸×E`ÑE`ÓaˆMà+×1Ñ1°"Õ5Ñˆ
àÒÜ Ÿ<š<¨×8LÑ8LÔM×TÑTÐU_ÐacÓdˆLØ×Ñ Ô!Ø'×1Ñ1°!Ó4×;Ñ;¸A¸rÀ2ÓFˆLð 15×0CÑ0CÀMÓ0RÐRˆä°NÐBÐ0ÄD×IÒIØEGÐ"Ø*¨MÐ;KÈ^Ð\ˆKÜ! $§+¡+×"?Ñ"?Ö@�Ø#'§;¡;×#?Ñ#?ÀÕ#IÐ Ø#×2Ñ2Ò>Ü7_ñ 8Ø/ð8à%ñ8Ð*Ó4ô
 8Qñ 8Ø/ð8à%ñ8Ð*Ó4ñ Aô " $§+¡+×"9Ñ"9Ô:ó
á:�
ð ˜Ÿ™¨ÀZÓPÒPÙ:ð 	ð 
ð
 ×,Ñ,¨YÓ7ˆä(1°$·+±+Ö(>Ñ$ˆI�}á)ØØ%Ø )¨T×-HÑ-HÔ H‘Èdñð %8¸×8TÑ8TÕ$Uð	ð
  6°iÕ@ðð ñŠMñ )?ð Ÿ™¨Ó6ˆÜ°Ô?Ð?ùò'
s   Æ I)	r|  r´  rµ  r¼  r¹  r²  r³  r»  r�  rp  )rM   rN   rO   rP   r/   r   r   rÉ  rØ  r   rK   rQ   rR   rS   rT   s   @@r(   rŒ  rŒ  í  sn   ù‡ € ÷ó ð" ÙÐ!`Ôa÷	jó bó ð	j÷
Qð 
Qð Ø÷C@ñ C@ó ó ÷C@ð C@r+   rŒ  zB
    The NeoMME model with a factorized masked token decoder.
    c                   ót   a a€ ] tR tRt oRRRR/tV3R lV 3R llt]]RV3R lR	 ll4       4       tR
t	Vt
V ;t# )rŽ  ie  zlm_head.weightz)model.embed_tokens.word_embeddings.weightzunembedding_projection.weightz.model.embed_tokens.embedding_projection.weightc                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   ÚNeoMMEForMaskedLM.__annotate__p  ó   ø€ ÷ ñ ™|ñ r+   c                ó.  <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        \        P
                  ! VP                  VP                  R R7      V n
        V P                  4        R# r\   )r.   r/   rŒ  rs  r   rc   ra   rd   Úunembedding_projectionr`   r�  r½  rf   s   &&€r(   r/   ÚNeoMMEForMaskedLM.__init__p  si   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ü&(§i¢i°×0EÑ0EÀv×GYÑGYÐ`eÔ&fˆÔ#Ü—y’y ×!6Ñ!6¸×8IÑ8IÐPUÔVˆŒØ�‰Ör+   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )	r   ri   Nr  rµ   r~   Úlabelsr®   rF   )r1   rj   r:   r   r   r   )r&   r'   s   "€r(   r)   rì  y  sŠ   ø€ ÷ "
ñ "
á×#Ñ# dÕ*ð"
ñ Ÿ™ tÕ+ð"
ñ ×&Ñ&¨Õ-ð	"
ñ
 —l‘l TÕ)ð"
ñ × Ñ  4Õ'ð"
ñ Ñ+Õ,ð"
ñ 
ñ"
r+   c                óL  € V P                   ! RRVRVRVRV/VB pVP                  pW€P                  P                  ,          pV P	                  V4      p	Rp
Ve*   V P
                  ! W•3RV P                  P                  /VB p
\        V
V	VP                  VP                  R7      # )	a  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for the masked-language-modeling loss. Indices should be in `[0, ..., config.vocab_size - 1]`
    or `-100`; only tokens with a label different from `-100` contribute.
ri   r  rµ   r~   Nr`   )ÚlossÚlogitsr8   ru  r�  )rs  rÄ  rï  r3   r�  Úloss_functionrY   r`   r   r8   ru  )r4   ri   r  rµ   r~   rò  r®   Úoutputsr8   rõ  rô  s   &&&&&&,    r(   rK   ÚNeoMMEForMaskedLM.forwardw  sÄ   € ð  $(§:¢:ñ $
Øð$
à)ð$
ð &ð$
ð &ð	$
ð
 ñ$
ˆð  ×1Ñ1ˆØ%×(CÑ(C×(JÑ(JÕJˆØ—‘˜mÓ,ˆàˆØÒØ×%Ò% fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   )r�  rs  rï  )NNNNN)rM   rN   rO   rP   Ú_tied_weights_keysr/   r   r   rK   rQ   rR   rS   rT   s   @@r(   rŽ  rŽ  e  sK   ù‡ € ð 	ÐEØ'Ð)YðÐ÷
ó ð Ø÷"
ñ "
ó ó ÷"
ð "
r+   rŽ  z1
    Output type for [`NeoMMEForRetrieval`].
    c                   ó<   a € ] tR tRt o RtRtRtRtV 3R ltRt	V t
R# )ÚNeoMMEForRetrievalOutputiž  aU  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*):
    Retrieval loss. This value is always `None`.
embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, embedding_dim)`, *optional*):
    Normalized token embeddings for late-interaction retrieval. Padding rows are zeroed. Score them with MeanMaxSim.
dense_embeddings (`torch.FloatTensor` of shape `(batch_size, hidden_size)` or `(batch_size, dense_dim)`, *optional*):
    A normalized mean-pooled embedding for each input. When `dense_dim` is set, the last dimension is `dense_dim`.
    Score them with cosine similarity.
Nc                ó¤   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   # )r   Nrô  Ú
embeddingsÚdense_embeddings)r1   ÚFloatTensor)r&   r'   s   "€r(   r)   Ú%NeoMMEForRetrievalOutput.__annotate__ž  sM   ø‡ ‚ ñ" ×
Ñ
˜dÕ
"Ñ)ñ# ñ$ ×!Ñ! DÕ(Ñ/ñ% ñ& ×'Ñ'¨$Õ.Ñ5ò' r+   r�  )rM   rN   rO   rP   ro   rô  rý  rþ  rª  rQ   rR   ©r'   s   @r(   rû  rû  ž  s#   ø‡ € ñð &*€DØ+/€JØ15Ð÷' ƒ r+   rû  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚNeoMMEMultiVectorHeadi´  c                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   Ú"NeoMMEMultiVectorHead.__annotate__µ  s   ø€ ÷ Tñ T™|ñ Tr+   c                óˆ   <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        R# r\   )r.   r/   r   rc   rd   r™  Úprojrf   s   &&€r(   r/   ÚNeoMMEMultiVectorHead.__init__µ  s.   ø€ Ü‰ÑÔÜ—I’I˜f×0Ñ0°&×2FÑ2FÈUÔSˆŽ	r+   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   r8   r  rF   r9   )r&   r'   s   "€r(   r)   r  ¹  s6   ø€ ÷ Qñ Q¡U§\¡\ð QÁ5Ç<Á<ð QÑTY×T`ÑT`ñ Qr+   c                ó  € V P                  VP                  V P                   P                  P                  4      4      p\        P
                  ! VRR7      pVP                  VP                  4       P                  R4      ( R4      # )rË   rÊ   rJ  r>   )	r  r­   r3   r¨   rî   rï   Úmasked_fillr%   r  )r4   r8   r  rý  s   &&& r(   rK   ÚNeoMMEMultiVectorHead.forward¹  se   € Ø—Y‘Y˜}×/Ñ/°·	±	×0@Ñ0@×0FÑ0FÓGÓHˆ
Ü—[’[ °Ô4ˆ
à×%Ñ% ~×':Ñ':Ó'<×'FÑ'FÀrÓ'JÐ&JÈCÓPÐPr+   )r  r÷   rT   s   @@r(   r  r  ´  s#   ù‡ € ÷Tó T÷Q÷ Qð Qr+   r  c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )ÚNeoMMEDenseHeadiÀ  Nc                ó|   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[R,          RS[ P                  /# )r   r8   r  Ú	dense_dimNrF   rÚ   )r&   r'   s   "€r(   r)   ÚNeoMMEDenseHead.__annotate__Á  s?   ø€ ÷ 
<ñ 
<Ù"Ÿ\™\ð
<Ù;@¿<¹<ð
<ÙTWÐZ^ÕT^ð
<á	�‰ñ
<r+   c                ó  € VP                  R4      P                  VP                  4      P                  VP                  4      pW,          P                  ^4      VP                  ^4      P                  R4      ,          pVf   \        P                  ! VRR7      # ^ Tu;8  d   VP                  R,          8:  g%   M \        RVP                  R,           RV 24      h\        P                  ! VRRV13,          RR7      # )rË   g•Ö&è.>NrÊ   zdense_dim must be in 1..z (the pooled width), got .r>   )
r  r¼   r½   r­   r¨   rð   Ú	clamp_minrî   rï   rÆ  )r4   r8   r  r  Úexpanded_maskÚpooleds   &&&&  r(   rK   ÚNeoMMEDenseHead.forwardÁ  s×   € ð '×0Ñ0°Ó4×;Ñ;¸M×<OÑ<OÓP×SÑSÐTa×TgÑTgÓhˆØÕ/×4Ñ4°QÓ7¸-×:KÑ:KÈAÓ:N×:XÑ:XÐY]Ó:^Õ^ˆØÒÜ—;’;˜v¨2Ô.Ð.à�9Ö0 §¡¨RÕ 0Ö0ÜÐ7¸¿¹ÀRÕ8HÐ7IÐIbÐclÐbmÐnÓoÐoÜ�{Š{˜6 # z¨	 z /Õ2¸Ô;Ð;r+   r�  rH   )rM   rN   rO   rP   rK   rQ   rR   r  s   @r(   r  r  À  s   ø‡ € ÷
<÷ 
<ð 
<r+   r  z½
    The NeoMME model with multi-vector and dense retrieval heads. One forward pass can return token embeddings for
    MaxSim scoring and mean-pooled embeddings for cosine similarity.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚNeoMMEForRetrievaliÎ  c                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   ÚNeoMMEForRetrieval.__annotate__Õ  rí  r+   c                ó¤   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        \        4       V n        V P                  4        R # rH   )	r.   r/   rŒ  rs  r  Úmulti_vector_headr  Ú
dense_headr½  rf   s   &&€r(   r/   ÚNeoMMEForRetrieval.__init__Õ  s=   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ü!6°vÓ!>ˆÔÜ)Ó+ˆŒØ�‰Ör+   c                óö   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[RS[RS[R,          R	S[S[,          R
S[/	# )r   ri   Nr  rµ   r~   Úoutput_multivectorÚoutput_denser  r®   rF   )r1   rj   r:   r%   r#   r   r   rû  )r&   r'   s   "€r(   r)   r  Þ  s—   ø€ ÷ *
ñ *
á×#Ñ# dÕ*ð*
ñ Ÿ™ tÕ+ð*
ñ ×&Ñ&¨Õ-ð	*
ñ
 —l‘l TÕ)ð*
ñ !ð*
ñ ð*
ñ ˜•:ð*
ñ Ñ+Õ,ð*
ñ 
"ñ*
r+   c                óÆ  € V'       g   V'       g   \        R4      hV P                  ! R
RVRVRVRV/VB p	V	P                  p
VfC   \        P                  ! V
P
                  R,          \        P                  V
P                  R7      pV'       d   V P                  W¢4      MRpV'       d   V P                  W¢V4      MRp\        VVV
V	P                  V	P                  R	7      # )a   
output_multivector (`bool`, *optional*, defaults to `True`):
    Whether to return token embeddings for late-interaction retrieval.
output_dense (`bool`, *optional*, defaults to `True`):
    Whether to return one mean-pooled dense embedding per input.
dense_dim (`int`, *optional*):
    Width of the Matryoshka prefix to return for dense embeddings. The model truncates the pooled vector
    before normalizing it.
zCAt least one of `output_multivector` or `output_dense` must be Trueri   r  rµ   r~   NrÝ  )r¨   rƒ   )rý  rþ  rÄ  r8   ru  r�  )rÆ  rs  rÄ  r1   r2   r½   r%   rƒ   r  r  rû  r8   ru  )r4   ri   r  rµ   r~   r   r!  r  r®   r÷  r8   rý  rþ  s   &&&&&&&&,    r(   rK   ÚNeoMMEForRetrieval.forwardÜ  sß   € ÷, #§lÜÐbÓcÐcà#'§:¢:ñ $
Øð$
à)ð$
ð &ð$
ð &ð	$
ð
 ñ$
ˆð  ×1Ñ1ˆØÒ!Ü"ŸZšZ¨×(;Ñ(;¸BÕ(?ÄuÇzÁzÐZg×ZnÑZnÔoˆNçN`�T×+Ñ+¨MÔJÐfjˆ
ßXd˜4Ÿ?™?¨=È)ÔTÐjnÐÜ'Ø!Ø-Ø+Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r+   )r  rs  r  )NNNNTTN)rM   rN   rO   rP   r/   r   r   rK   rQ   rR   rS   rT   s   @@r(   r  r  Î  s5   ù‡ € ÷ó ð Ø÷*
ñ *
ó ó ÷*
ð *
r+   r  )rË   )rJ  )rŽ  r  rŒ  rr  )BÚcollections.abcr   Údataclassesr   r1   Útorch.nn.functionalr   r$  rî   Ú r   rˆ  Úactivationsr   Úmasking_utilsr   r	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_neommer   r  r   rV   rq   r‚   rà   râ   rù   r	  r  r*  r,  rS  r\  rr  rŒ  rŽ  rû  r  r  r  Ú__all__r�  r+   r(   Ú<module>r5     sé  ðõ* %Ý !ã ß Ð Ý å &Ý !ß `Ý 9ß [Ñ [ß Kß FÝ &ß OÑ OÝ 0ß =Ý 5Ý .ô4�B—I‘Iô 4ô*	J�r—y‘yô 	Jô:˜BŸI™Iô :ôT3˜BŸI™Iô T3õn	Uô? 2§9¡9ô ?ôW 2§9¡9ô Wò(ô!÷H%ô2U)�b—i‘iô U)ôp<�—	‘	ô <ô#EÐ3ô #EðL ôB˜Oó Bó ðBñJ ðôô
p@Ð'ó p@óð
p@ñf ðôô
1
Ð-ó 1
óð
1
ñh ðôð
 ô6˜ó 6ó óð6ô 	Q˜BŸI™Iô 	Qô<�b—i‘iô <ñ ðôô4
Ð.ó 4
óð4
òn ^‚r+   