+
    œ°³j“  ã                   óÄ  € ^ RI t ^ RIHt ^ RIHt ^ RIt^ RIHt ^ RIHt ^ RI	H
t
HtHt ^RIHt ^RIHt ^R	IHt ^R
IHtHt ^RIHtHt ^RIHt ^RIHtHtHtH t H!t!H"t" ^RI#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+H,t,H-t- ^RI.H/t/H0t0 ^RI1H2t2 ^RI3H4t4 ^RI5H6t6H7t7 ]-Pp                  ! ]94      t:],! RR7      ] ! R R]4      4       4       t; ! R R]Px                  4      t= ! R R]Px                  4      t> ! R R]64      t?]! R 4      R;R! l4       t@]! ]@4       ! R" R#]Px                  4      4       tA ! R$ R%]4      tB], ! R& R']'4      4       tC], ! R( R)]C4      4       tD ! R* R+]Px                  4      tE],! R,R-7       ! R. R/]C4      4       tF],! R0R-7       ! R1 R2]C4      4       tG],! R3R-7       ! R4 R5]C4      4       tH], ! R6 R7]C4      4       tI],! R8R-7       ! R9 R:]C4      4       tJ. R<OtKR# )=é    N)ÚCallable)ÚLiteral)Ústrict)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)ÚPreTrainedConfig)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs)Úeager_attention_forward)ÚGemma3RotaryEmbeddingÚrotate_halfzanswerdotai/ModernBERT-base)Ú
checkpointc                   ó"  a a€ ] tR t^4t oRtRtR.tRRRR/tRtR	t	R
t
^t^tRtRtRtRtRtRtRtRtRtRtRtRtRtRtRt^€tRtRtRtRt Rt!Rt"Rt#Rt$Rt%Rt&Rt'Rt(V 3R lt)R t*V 3R lt+],R 4       t-]-P\                  R 4       t-V3R lt/Rt0Vt1V ;t2# ) ÚModernBertConfigaŸ  
initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
    The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
norm_eps (`float`, *optional*, defaults to 1e-05):
    The epsilon used by the rms normalization layers.
norm_bias (`bool`, *optional*, defaults to `False`):
    Whether to use bias in the normalization layers.
local_attention (`int`, *optional*, defaults to 128):
    The window size for local attention.
mlp_dropout (`float`, *optional*, defaults to 0.0):
    The dropout ratio for the MLP layers.
decoder_bias (`bool`, *optional*, defaults to `True`):
    Whether to use bias in the decoder layers.
classifier_pooling (`str`, *optional*, defaults to `"cls"`):
    The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
    CLS token doesn't attend to all tokens on long sequences.
classifier_bias (`bool`, *optional*, defaults to `False`):
    Whether to use bias in the classifier.
classifier_activation (`str`, *optional*, defaults to `"gelu"`):
    The activation function for the classifier.
deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
    Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
sparse_prediction (`bool`, *optional*, defaults to `False`):
    Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
    The index to ignore for the sparse prediction.

Examples:

```python
>>> from transformers import ModernBertModel, ModernBertConfig

>>> # Initializing a ModernBert style configuration
>>> configuration = ModernBertConfig()

>>> # Initializing a model from the modernbert-base style configuration
>>> model = ModernBertModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```Ú
modernbertÚpast_key_valuesÚglobalg     ˆAÚlocalg     ˆÃ@iÀÄ  i   i€  Úgelui    g{®Gáz”?ç       @gñhãˆµøä>FikÄ  ijÄ  iiÄ  ç        NTÚclsc                óø   <€ VP                  R ^4      pV P                  fD   \        V P                  4       Uu. uF  p\	        W2,          4      '       d   RMRNK   	  upV n        \
        SV `  ! R/ VB  R# u upi )Úglobal_attn_every_n_layersNÚsliding_attentionÚfull_attention© )ÚgetÚlayer_typesÚrangeÚnum_hidden_layersÚboolÚsuperÚ__post_init__)ÚselfÚkwargsr1   ÚiÚ	__class__s   &,  €Úr/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/modernbert/modular_modernbert.pyr;   ÚModernBertConfig.__post_init__‡   sy   ø€ à%+§Z¡ZÐ0LÈaÓ%PÐ"Ø×ÑÒ#ô ˜t×5Ñ5Ô6ó á6�Aô (,¨AÕ,J×'KÒ'KÑ#ÐQaÒaÙ6ñ ˆDÔô
 	‰ÒÑ' Ô'ùò s   ¹$A7c                óø  € VP                  R R4      pRRR/RRR//pV P                  e   V P                  MTV n        VeE   V P                  R,          P                  V4       V P                  R,          P                  V4       V P                  P                  R4      f   RR/V P                  R&   V P                  R,          P	                  RVP                  RV P
                  R,          4      4       V P                  P                  R4      f   RR/V P                  R&   V P                  R,          P	                  RVP                  R	V P
                  R
,          4      4       V P                  4        V# )Úrope_scalingNr2   Ú	rope_typeÚdefaultr3   Ú
rope_thetaÚglobal_rope_thetar*   Úlocal_rope_thetar+   )ÚpopÚrope_parametersÚupdater5   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)r<   r=   rC   Údefault_rope_paramss   &,  r@   Úconvert_rope_params_to_dictÚ,ModernBertConfig.convert_rope_params_to_dict’   sg  € Ø—z‘z .°$Ó7ˆð
   +¨yÐ!9Ø˜{¨IÐ6ð
Ðð 8<×7KÑ7KÒ7W˜t×3Ò3Ð]pˆÔØÒ#Ø× Ñ Ð!1Õ2×9Ñ9¸,ÔGØ× Ñ Ð!4Õ5×<Ñ<¸\ÔJð ×Ñ×#Ñ#Ð$4Ó5Ò=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Õ.×9Ñ9Ø˜&Ÿ*™*Ð%8¸$×:LÑ:LÈXÕ:VÓWô	
ð ×Ñ×#Ñ#Ð$7Ó8Ò@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Õ1×<Ñ<Ø˜&Ÿ*™*Ð%7¸×9KÑ9KÈGÕ9TÓUô	
ð
 	×$Ñ$Ô&Øˆó    c                óH   <€ \         SV `  4       pVP                  R R4       V# )Úreference_compileN)r:   Úto_dictrI   )r<   Úoutputr?   s   & €r@   rU   ÚModernBertConfig.to_dict°   s#   ø€ Ü‘‘Ó"ˆØ�
‰
Ð&¨Ô-ØˆrR   c                ó(   € V P                   ^,          # )zKHalf-window size: `local_attention` is the total window, so we divide by 2.©Úlocal_attention©r<   s   &r@   Úsliding_windowÚModernBertConfig.sliding_windowµ   s   € ð ×#Ñ# qÕ(Ð(rR   c                ó"   € V^,          V n         R# )z<Set sliding_window by updating local_attention to 2 * value.NrY   ©r<   Úvalues   &&r@   r\   r]   º   s   € ð  % q�yˆÖrR   c                óª  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[ ;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[ R,          ;R&   S[ S[S[ ,          ,          R,          ;R&   S[ R,          ;R&   S[ R,          ;R&   S[ R,          ;R&   S[;R&   S[S[ ,          ;R&   S[S[,          R,          ;R&   S[S[R#,          S[3,          R,          ;R&   S[ ;R&   S[S[ ,          ;R&   S[;R&   S[S[ ,          ;R&   S[;R&   S[R$,          ;R&   S[S[ ,          ;R&   S[;R&   S[;R&   S[;R&   S[;R &   S[ ;R!&   S[;R"&   # )%é   Ú
vocab_sizeÚhidden_sizeÚintermediate_sizer8   Únum_attention_headsÚhidden_activationÚmax_position_embeddingsÚinitializer_rangeÚinitializer_cutoff_factorÚnorm_epsÚ	norm_biasNÚpad_token_idÚeos_token_idÚbos_token_idÚcls_token_idÚsep_token_idÚattention_biasÚattention_dropoutr6   rJ   rZ   Úembedding_dropoutÚmlp_biasÚmlp_dropoutÚdecoder_biasÚclassifier_poolingÚclassifier_dropoutÚclassifier_biasÚclassifier_activationÚdeterministic_flash_attnÚsparse_predictionÚsparse_pred_ignore_indexÚtie_word_embeddings)r3   r2   )r/   Úmean)ÚintÚstrÚfloatr9   ÚlistÚdictr   )ÚformatÚ__classdict__s   "€r@   Ú__annotate__ÚModernBertConfig.__annotate__4   sE  ø‡ ‚ ñb Ññc ñd Ññe ñf Ñ!ñg ñh Ññi ñj Ñ!ñk ñl Ñ#ñm ñn !Ñ'ño ñp Ñ#ñq ñr  %Ñ*ñs ñt Ññu ñv Ññw ñx ˜•*Ñ$ñy ñz ™™S�	•/ DÕ(Ñ0ñ{ ñ| ˜•*Ñ$ñ} ñ~ ˜•*Ñ$ñ ñ@ ˜•*Ñ$ñA ñB Ñ ñC ñD ™s•{Ñ(ñE ñF ‘c•˜TÕ!Ñ(ñG ñH ™'Ð"GÕHÉ$ÐNÕOÐRVÕVÑ]ñI ñJ ÑñK ñL ™s•{Ñ(ñM ñN ÑñO ñP ™•Ñ"ñQ ñR ÑñS ñT   Õ.Ñ6ñU ñV ¡�Ñ)ñW ñX Ñ!ñY ñZ Ñ'ñ[ ñ\ #Ñ*ñ] ñ^ Ñ#ñ_ ñ` "Ñ(ña ñb Ñ$òc rR   )r6   rZ   rJ   iœÿÿÿ)3Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencerM   rc   rd   re   r8   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   r6   rJ   rZ   rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r;   rP   rU   Úpropertyr\   ÚsetterÚ__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r?   r‡   s   @@r@   r'   r'   4   s#  ù‡ € ñ(ðT €JØ#4Ð"5ÐØ˜y¨'°8Ð<€Mà€JØ€KØ!ÐØÐØ!ÐØ#ÐØ#'ÐØ#ÐØ'*ÐØ€HØ€IØ$€LØ+0€LØ$€LØ$€LØ$€LØ €NØ%(ÐØ$(€KØY]€OØ€OØ%(ÐØ€HØ"€KØ€LØ16ÐØ&)ÐØ!€OØ!'ÐØ%*ÐØ#ÐØ$(ÐØ $Ðõ	(òõ<ð
 ñ)ó ð)ð ×Ññ)ó ð)÷O … rR   r'   c                   óX   a a€ ] tR t^Àt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚModernBertEmbeddingszN
Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
c                ó    <€ V ^8„  d   QhRS[ /# ©rb   Úconfig©r'   )r†   r‡   s   "€r@   rˆ   Ú!ModernBertEmbeddings.__annotate__Å   s   ø€ ÷ 9ñ 9Ñ/ñ 9rR   c                ój  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# ))Úpadding_idx©ÚepsÚbiasN)r:   Ú__init__rœ   r   Ú	Embeddingrc   rd   rm   Útok_embeddingsÚ	LayerNormrk   rl   ÚnormÚDropoutrt   Údrop©r<   rœ   r?   s   &&€r@   r¤   ÚModernBertEmbeddings.__init__Å   su   ø€ Ü‰ÑÔØŒÜ Ÿlšl¨6×+<Ñ+<¸f×>PÑ>PÐ^d×^qÑ^qÔrˆÔÜ—L’L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ô`ˆŒ	Ü—J’J˜v×7Ñ7Ó8ˆŽ	rR   c                ó„   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  /# )rb   Ú	input_idsNÚinputs_embedsÚreturn)ÚtorchÚ
LongTensorÚTensor)r†   r‡   s   "€r@   rˆ   rž   Ì   s>   ø€ ÷ ñ Ù×)Ñ)¨DÕ0ðÙHMÏÉÐW[ÕH[ðá	�‰ñrR   c                ó°   € Ve#   V P                  V P                  V4      4      pV# V P                  V P                  V P                  V4      4      4      pV# ©N)rª   r¨   r¦   )r<   r®   r¯   Úhidden_statess   &&& r@   ÚforwardÚModernBertEmbeddings.forwardÌ   sS   € ð Ò$Ø ŸI™I d§i¡i°Ó&>Ó?ˆMð Ðð !ŸI™I d§i¡i°×0CÑ0CÀIÓ0NÓ&OÓPˆMØÐrR   )rœ   rª   r¨   r¦   ©NN©
rŠ   r‹   rŒ   r�   rŽ   r¤   r·   r”   r•   r–   r—   s   @@r@   r™   r™   À   s#   ù‡ € ñ÷9ó 9÷÷ ò rR   r™   c                   óT   a a€ ] tR t^Öt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚModernBertMLPa*  Applies the GLU at the end of each ModernBERT layer.

Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
c                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   ÚModernBertMLP.__annotate__Ý   s   ø€ ÷ `ñ `Ñ/ñ `rR   c                óÂ  <€ \         SV `  4        Wn        \        P                  ! VP
                  \        VP                  4      ^,          VP                  R7      V n	        \        VP                  ,          V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP
                  VP                  R7      V n        R# )rb   ©r£   N)r:   r¤   rœ   r   ÚLinearrd   r�   re   ru   ÚWir   rg   Úactr©   rv   rª   ÚWor«   s   &&€r@   r¤   ÚModernBertMLP.__init__Ý   s‘   ø€ Ü‰ÑÔØŒÜ—)’)˜F×.Ñ.´°F×4LÑ4LÓ0MÐPQÕ0QÐX^×XgÑXgÔhˆŒÜ˜&×2Ñ2Õ3ˆŒÜ—J’J˜v×1Ñ1Ó2ˆŒ	Ü—)’)˜F×4Ñ4°f×6HÑ6HÈvÏÉÔ_ˆŽrR   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rb   r¶   r°   ©r±   r³   )r†   r‡   s   "€r@   rˆ   r¾   å   s#   ø€ ÷ :ñ :¡U§\¡\ð :±e·l±lñ :rR   c                ó¶   € V P                  V4      P                  ^RR7      w  r#V P                  V P                  V P	                  V4      V,          4      4      # )rb   ©Údiméÿÿÿÿ)rÂ   ÚchunkrÄ   rª   rÃ   )r<   r¶   ÚinputÚgates   &&  r@   r·   ÚModernBertMLP.forwardå   sG   € Ø—g‘g˜mÓ,×2Ñ2°1¸"Ð2Ó=‰ˆØ�w‰w�t—y‘y §¡¨%£°4Õ!7Ó8Ó9Ð9rR   )rÂ   rÄ   rÃ   rœ   rª   rº   r—   s   @@r@   r¼   r¼   Ö   s%   ù‡ € ñ÷`ó `÷:÷ :ð :rR   r¼   c                   ó^   a a€ ] tR t^êt oRV3R lV 3R llltRV3R lV 3R llltRtVtV ;t# )ÚModernBertRotaryEmbeddingc                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   Ú&ModernBertRotaryEmbedding.__annotate__ë   s   ø€ ÷ !ñ !Ñ/ñ !rR   c                ó&   <€ \         SV `  V4       R # rµ   )r:   r¤   )r<   rœ   Údevicer?   s   &&&€r@   r¤   Ú"ModernBertRotaryEmbedding.__init__ë   s   ø€ Ü‰Ñ˜Ö rR   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )rb   rœ   Ú
layer_typeNr°   )r'   r‚   Útupler±   r³   rƒ   )r†   r‡   s   "€r@   rˆ   rÔ   î   s<   ø€ ÷ Kñ KÙ ðKÙ;>À½:ðKá	‰u�|‰|™UÐ"Õ	#ñKrR   c                ó"   <€ \         SV `  W4      # rµ   )r:   Úcompute_default_rope_parameters)rœ   rÖ   rÙ   r=   r?   s   &&&,€r@   rÜ   Ú9ModernBertRotaryEmbedding.compute_default_rope_parametersî   s   ø€ ô ‰wÑ6°vÓJÐJrR   r4   rµ   r¹   )	rŠ   r‹   rŒ   r�   r¤   rÜ   r”   r•   r–   r—   s   @@r@   rÒ   rÒ   ê   s!   ù‡ € ÷!õ !÷K÷ Kõ KrR   rÒ   Úrotary_pos_embc                ó’  € V P                   pVP                  V4      pVP                  V4      pV P                  4       V,          \        V P                  4       4      V,          ,           pVP                  4       V,          \        VP                  4       4      V,          ,           pVP	                  V4      VP	                  V4      3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)ÚdtypeÚ	unsqueezerƒ   r$   Úto)ÚqÚkÚcosÚsinÚunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds   &&&&&   r@   Úapply_rotary_pos_embrë   ô   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3�¤;¨q¯w©w«yÓ#9¸CÕ#?Õ@€GØ�w‰w‹y˜3�¤;¨q¯w©w«yÓ#9¸CÕ#?Õ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐArR   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚModernBertAttentioni  an  Performs multi-headed self attention on a batch of unpadded sequences.

If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
which requires padding and unpadding inputs, adding some overhead.

See `forward` method for additional details.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©rb   rœ   Ú	layer_idxN©r'   r�   )r†   r‡   s   "€r@   rˆ   Ú ModernBertAttention.__annotate__  s$   ø€ ÷ rñ rÑ/ð r¹CÀ$½Jñ rrR   c                ó–  <€ \         SV `  4        Wn        W n        VP                  VP
                  ,          ^ 8w  d'   \        RVP                   RVP
                   R24      hVP                  V n        VP                  V n        VP                  VP
                  ,          V n	        \        P                  ! VP                  ^V P                  ,          VP
                  ,          VP                  R7      V n        VP                  V,          R8X  d   VP                  ^,           V n        MRV n        RV n        \        P                  ! VP                  VP                  VP                  R7      V n        VP                  R8”  d(   \        P$                  ! VP                  4      V n        R# \        P&                  ! 4       V n        R# )	r   zThe hidden size (z6) is not a multiple of the number of attention heads (Ú)rÀ   r2   NFr.   )r:   r¤   rœ   rð   rd   rf   Ú
ValueErrorrs   r|   Úhead_dimr   rÁ   rr   ÚWqkvr6   r\   Ú	is_causalrÄ   r©   ÚIdentityÚout_drop©r<   rœ   rð   r?   s   &&&€r@   r¤   ÚModernBertAttention.__init__  sv  ø€ Ü‰ÑÔØŒØ"Œà×Ñ × :Ñ :Õ:¸aÔ?ÜØ# F×$6Ñ$6Ð#7Ð7mÐnt÷  oIñ  oIð  nJð  JKð  Lóð ð "(×!9Ñ!9ˆÔØ(.×(GÑ(GˆÔ%Ø×*Ñ*¨f×.HÑ.HÕHˆŒÜ—I’IØ×Ñ  D§M¡MÕ 1°F×4NÑ4NÕ NÐU[×UjÑUjô
ˆŒ	ð ×Ñ˜iÕ(Ð,?Ô?ð #)×"7Ñ"7¸!Õ";ˆDÕà"&ˆDÔàˆŒä—)’)˜F×.Ñ.°×0BÑ0BÈ×I^ÑI^Ô_ˆŒØ@F×@XÑ@XÐ[^Ô@^œŸ
š
 6×#;Ñ#;Ó<ˆŽÔdf×doÒdoÓdqˆŽrR   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )rb   r¶   Úposition_embeddingsNÚattention_maskr=   r°   )r±   r³   rÚ   r   r   )r†   r‡   s   "€r@   rˆ   rò   7  s~   ø€ ÷ ')ñ ')á—|‘|ð')ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð')ñ Ÿ™ tÕ+ð	')ñ
 Ñ+Õ,ð')ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ')rR   c                óä  € VP                   R R pV P                  V4      pVP                  ! . VO^NRNV P                  N5!  pVP	                  R	R7      w  rxp	VP                  ^^4      pVP                  ^^4      pV	P                  ^^4      p	Vw  r«\        WxW«^R7      w  rx\        P                  ! V P                  P                  \        4      pT! V VVV	V3RV P                  '       d   V P                  MRRV P                  R
,          RV P                  RV P                  /VB w  rÞVP                   ! . VORN5!  P#                  4       pV P%                  V P'                  V4      4      pWÞ3# )NrÊ   )rç   Údropoutr.   Úscalingr\   ÚdeterministicrÌ   éýÿÿÿç      à¿)Úshaper÷   Úviewrö   ÚunbindÚ	transposerë   r   Úget_interfacerœ   Ú_attn_implementationr"   Útrainingrs   r\   r|   ÚreshapeÚ
contiguousrú   rÄ   )r<   r¶   rþ   rÿ   r=   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesrå   ræ   Úattention_interfaceÚattn_outputÚattn_weightss   &&&&,          r@   r·   ÚModernBertAttention.forward7  s†  € ð $×)Ñ)¨#¨2Ð.ˆà�i‰i˜Ó&ˆØ�hŠhÐ:˜Ð: QÐ:¨Ð:¨D¯M©MÓ:ˆØ14·±À°Ó1CÑ.ˆ ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÐjkÔ#lÑ ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð /3¯m¯m¨m�D×*Ò*Àð%
ð —M‘M 4Õ'ð%
ð  ×.Ñ.ð%
ð ×7Ñ7ð%
ð ñ%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—m‘m D§G¡G¨KÓ$8Ó9ˆØÐ(Ð(rR   )
rÄ   r÷   rs   rœ   r|   rö   rø   rð   rú   r\   rµ   r¹   rº   r—   s   @@r@   rí   rí     s%   ù‡ € ñ÷rõ r÷:')÷ ')ò ')rR   rí   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚModernBertEncoderLayeria  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# rï   rñ   )r†   r‡   s   "€r@   rˆ   Ú#ModernBertEncoderLayer.__annotate__b  s    ø€ ÷ <ñ <Ñ/ð <¹CÀ$½Jñ <rR   c                óâ  <€ \         SV `  4        Wn        W n        V^ 8X  d   \        P
                  ! 4       V n        M<\        P                  ! VP                  VP                  VP                  R7      V n        \        WR7      V n        \        P                  ! VP                  VP                  VP                  R7      V n        \        V4      V n        VP                   V,          V n        R# )r   r¡   )rœ   rð   N)r:   r¤   rœ   rð   r   rù   Ú	attn_normr§   rd   rk   rl   rí   ÚattnÚmlp_normr¼   Úmlpr6   Úattention_typerû   s   &&&€r@   r¤   ÚModernBertEncoderLayer.__init__b  s¡   ø€ Ü‰ÑÔØŒØ"ŒØ˜Œ>ÜŸ[š[›]ˆD�NäŸ\š\¨&×*<Ñ*<À&Ç/Á/ÐX^×XhÑXhÔiˆDŒNÜ'¨vÔKˆŒ	ÜŸš V×%7Ñ%7¸V¿_¹_ÐSY×ScÑScÔdˆŒÜ  Ó(ˆŒØ$×0Ñ0°Õ;ˆÖrR   c          
      ó´   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[ P                  /# )rb   r¶   rÿ   Nrþ   r=   r°   )r±   r³   r   r   )r†   r‡   s   "€r@   rˆ   r  o  s]   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ #Ÿ\™\¨DÕ0ð	ñ
 Ñ+Õ,ðñ 
�‰ñrR   c                ó²   € V P                   ! V P                  V4      3R VRV/VB w  rVW,           pWP                  V P                  V4      4      ,           pV# )rþ   rÿ   )r  r  r   r  )r<   r¶   rÿ   rþ   r=   r  Ú_s   &&&&,  r@   r·   ÚModernBertEncoderLayer.forwardo  sf   € ð ŸšØ�N‰N˜=Ó)ñ
à 3ð
ð *ð
ð ñ	
‰ˆð &Õ3ˆØ%¯©°·±¸}Ó1MÓ(NÕNˆØÐrR   )r!  r  r  rœ   rð   r   r  rµ   r¹   ©	rŠ   r‹   rŒ   r�   r¤   r·   r”   r•   r–   r—   s   @@r@   r  r  a  s   ù‡ € ÷<õ <÷÷ ò rR   r  c                   óž   a a€ ] tR tRt oRtRtRR.tRtRtRt	Rt
R]R]/t]P                  ! 4       V3R lV 3R	 ll4       tV3R
 ltRtVtV ;t# )ÚModernBertPreTrainedModeli�  ÚmodelTr™   r  r¶   Ú
attentionsc                ó4   <€ V ^8„  d   QhRS[ P                  /# )rb   Úmodule)r   ÚModule)r†   r‡   s   "€r@   rˆ   Ú&ModernBertPreTrainedModel.__annotate__’  s   ø€ ÷ 7^ñ 7^¡B§I¡Iñ 7^rR   c           	     ó  <a€ \         S	V `  V4       V P                  P                  oSf   ^oR V3R llpRV P                  P                  RV P                  P                  \
        P                  ! RV P                  P                  ,          4      ,          RV P                  P                  RV P                  P                  R,          /p\        V\        4      '       d   V! VP                  VR,          4       R # \        V\        4      '       d7   V! VP                  VR,          4       V! VP                  VR,          4       R # \        V\        4      '       d7   V! VP                   VR,          4       V! VP                  VR,          4       R # \        V\"        4      '       d   V! VP$                  VR,          4       R # \        V\&        4      '       d   V! VP(                  VR,          4       R # \        V\*        \,        \.        \0        34      '       d   V! VP2                  VR,          4       R # \        V\4        4      '       d´   VP6                   F¡  pVP8                  pVP:                  V,          R8w  d   \<        VP:                  V,          ,          pV! VP                  VR	7      w  rg\>        P@                  ! \C        W R
24      V4       \>        P@                  ! \C        W R24      V4       K£  	  R # R # )Nc                óD   € V ^8„  d   QhR\         P                  R\        /# )rb   r-  Ústd)r   r.  rƒ   )r†   s   "r@   rˆ   Ú=ModernBertPreTrainedModel._init_weights.<locals>.__annotate__˜  s   € ÷ 	-ñ 	-¤§	¡	ð 	-´ñ 	-rR   c                 ó  <€ \         P                  ! V P                  R VS) V,          SV,          R7       \        V \        P
                  4      '       d3   V P                  e#   \         P                  ! V P                  4       R# R# R# )r.   )r€   r2  ÚaÚbN)ÚinitÚtrunc_normal_ÚweightÚ
isinstancer   rÁ   r£   Úzeros_)r-  r2  Úcutoff_factors   &&€r@   Úinit_weightÚ<ModernBertPreTrainedModel._init_weights.<locals>.init_weight˜  sg   ø€ Ü×ÒØ—‘ØØØ �. 3Õ&Ø #Õ%õô ˜&¤"§)¡)×,Ò,Ø—;‘;Ò*Ü—K’K §¡Ö,ñ +ñ -rR   ÚinÚoutr-   Ú	embeddingÚ	final_outrE   )rÙ   Ú	_inv_freqÚ_original_inv_freqr  )"r:   Ú_init_weightsrœ   rj   ri   ÚmathÚsqrtr8   rd   r:  r™   r¦   r¼   rÂ   rÄ   rí   r÷   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierrÒ   r6   rÜ   rD   r   r7  Úcopy_Úgetattr)
r<   r-  r=  ÚstdsrÙ   Úrope_init_fnÚcurr_inv_freqr%  r<  r?   s
   &&      @€r@   rE  Ú'ModernBertPreTrainedModel._init_weights‘  s9  ù€ ä‰Ñ˜fÔ%ØŸ™×=Ñ=ˆØÒ ØˆM÷	-ð 	-ð �$—+‘+×/Ñ/Ø�4—;‘;×0Ñ0´4·9²9¸SÀ4Ç;Á;×C`ÑC`Õ=`Ó3aÕaØ˜Ÿ™×6Ñ6Ø˜Ÿ™×0Ñ0°$Õ6ð	
ˆô �fÔ2×3Ò3Ù˜×-Ñ-¨t°KÕ/@ÖAÜ˜¤×.Ò.Ù˜Ÿ	™	 4¨¥:Ô.Ù˜Ÿ	™	 4¨¥;Ö/Ü˜Ô 3×4Ò4Ù˜Ÿ™ T¨$¥ZÔ0Ù˜Ÿ	™	 4¨¥;Ö/Ü˜Ô 8×9Ò9Ù˜Ÿ™ d¨5¥kÖ2Ü˜Ô 5×6Ò6Ù˜Ÿ™¨¨U­Ö4ÜØä3Ü+Ü0Ü.ð	÷
ò 
ñ ˜×)Ñ)¨4°Õ+<Ö=Ü˜Ô 9×:Ò:Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó 1ñ ;rR   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r›   r�   )r†   r‡   s   "€r@   rˆ   r/  �  s   ø‡ ‚ áÑò rR   r4   )rŠ   r‹   rŒ   r�   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  rí   Ú_can_record_outputsr±   Úno_gradrE  r“   r”   r•   r–   r—   s   @@r@   r)  r)  �  su   ù‡ € ð  ÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 	Ð/ØÐ)ðÐð
 ‡]‚]ƒ_÷7^ò 7^ó ð7^÷# … rR   r)  c                   ó~   a a€ ] tR tRt oV3R lV 3R lltR tR t]]]	R	V3R lR ll4       4       4       t
RtVtV ;t# )
ÚModernBertModeliÌ  c                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   ÚModernBertModel.__annotate__Î  ó   ø€ ÷ 
ñ 
Ñ/ñ 
rR   c           	     ó²  <€ \         SV `  V4       Wn        \        V4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n
        \
        P                  ! VP                  VP                  VP                  R 7      V n        \!        VR7      V n        RV n        V P'                  4        R# u upi )r¡   )rœ   FN)r:   r¤   rœ   r™   Ú
embeddingsr   Ú
ModuleListr7   r8   r  Úlayersr§   rd   rk   rl   Ú
final_normrÒ   Ú
rotary_embÚgradient_checkpointingÚ	post_initrû   s   && €r@   r¤   ÚModernBertModel.__init__Î  s¡   ø€ Ü‰Ñ˜Ô ØŒÜ.¨vÓ6ˆŒÜ—m’mÜHMÈf×NfÑNfÔHgÓhÑHg¸9Ô# FÖ6ÑHgÑhó
ˆŒô Ÿ,š, v×'9Ñ'9¸v¿¹ÐU[×UeÑUeÔfˆŒÜ3¸6ÔBˆŒØ&+ˆÔ#Ø�‰Öùò is   ÁCc                ó.   € V P                   P                  # rµ   ©rg  r¦   r[   s   &r@   Úget_input_embeddingsÚ$ModernBertModel.get_input_embeddingsÚ  s   € Ø�‰×-Ñ-Ð-rR   c                ó&   € WP                   n        R # rµ   rp  r_   s   &&r@   Úset_input_embeddingsÚ$ModernBertModel.set_input_embeddingsÝ  s   € Ø).�‰Ö&rR   c                óÖ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )rb   r®   Nrÿ   Úposition_idsr¯   r=   r°   )r±   r²   r³   r   r   r   )r†   r‡   s   "€r@   rˆ   rd  ã  s}   ø€ ÷ ,@ñ ,@á×#Ñ# dÕ*ð,@ñ Ÿ™ tÕ+ð,@ñ ×&Ñ&¨Õ-ð	,@ñ
 —|‘| dÕ*ð,@ñ Ñ+Õ,ð,@ñ 
ñ,@rR   c           	     óø  € VR J VR J,          '       d   \        R4      hVe   VP                  ^,          MVP                  ^,          pVe   VP                  MVP                  pVf'   \        P                  ! WgR7      P                  ^ 4      pV P                  WR7      p\        T;p	\        4      '       g+   RV P                  RVRV/p
R\        R/ V
B R\        R/ V
B /p	/ p\        V P                  P                  4       F  pV P                  WƒV4      W¼&   K  	  V P                   F0  pV! V3RW�P                   ,          R	W½P                   ,          /VB pK2  	  V P#                  V4      p\%        VR
7      # )Nz:You must specify exactly one of input_ids or inputs_embeds©rÖ   )r®   r¯   rœ   r¯   rÿ   r3   r2   rþ   )Úlast_hidden_stater4   )rõ   r  rÖ   r±   Úarangerá   rg  r:  r…   rœ   r   r   Úsetr6   rk  ri  r!  rj  r   )r<   r®   rÿ   rw  r¯   r=   Úseq_lenrÖ   r¶   Úattention_mask_mappingÚmask_kwargsrþ   rÙ   Úencoder_layers   &&&&&,        r@   r·   ÚModernBertModel.forwardà  s}  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZà,9Ò,E�-×%Ñ% aÖ(È9Ï?É?Ð[\ÕK]ˆØ%.Ò%:�×!Ò!À×@TÑ@TˆàÒÜ Ÿ<š<¨Ô?×IÑIÈ!ÓLˆLàŸ™°)˜ÓYˆä°NÐBÐ0ÄD×IÒIà˜$Ÿ+™+Ø Ø  .ðˆKð !Ô";Ñ"J¸kÑ"JØ#Ô%MÑ%\ÐP[Ñ%\ð&Ð"ð
 !ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ð "Ÿ[œ[ˆMÙ)Øñà5×6RÑ6RÕSðð %8×8TÑ8TÕ$Uðð ñ	ŠMñ )ð Ÿ™¨Ó6ˆä°Ô?Ð?rR   )rœ   rg  rj  rl  ri  rk  )NNNN)rŠ   r‹   rŒ   r�   r¤   rq  rt  r    r!   r   r·   r”   r•   r–   r—   s   @@r@   rb  rb  Ì  sK   ù‡ € ÷
ó 
ò.ò/ð  ØØ÷,@ñ ,@ó ó ó  ÷,@ð ,@rR   rb  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )rH  i  c                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   Ú%ModernBertPredictionHead.__annotate__  s   ø€ ÷ añ aÑ/ñ arR   c                óV  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP
                  VP                  4      V n        \        VP                  ,          V n
        \        P                  ! VP
                  VP                  VP                  R 7      V n        R# )r¡   N)r:   r¤   rœ   r   rÁ   rd   rz   rI  r   r{   rÃ   r§   rk   rl   r¨   r«   s   &&€r@   r¤   Ú!ModernBertPredictionHead.__init__  so   ø€ Ü‰ÑÔØŒÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÀv×G]ÑG]Ó^ˆŒ
Ü˜&×6Ñ6Õ7ˆŒÜ—L’L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ô`ˆŽ	rR   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÇ   rÈ   )r†   r‡   s   "€r@   rˆ   r„    s#   ø€ ÷ >ñ >¡U§\¡\ð >±e·l±lñ >rR   c                ó`   € V P                  V P                  V P                  V4      4      4      # rµ   )r¨   rÃ   rI  )r<   r¶   s   &&r@   r·   Ú ModernBertPredictionHead.forward  s#   € Ø�y‰y˜Ÿ™ $§*¡*¨]Ó";Ó<Ó=Ð=rR   )rÃ   rœ   rI  r¨   r'  r—   s   @@r@   rH  rH    s    ù‡ € ÷aó a÷>÷ >ð >rR   rH  zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   óˆ   a a€ ] tR tRt oRR/tV3R lV 3R lltR tV3R lR lt]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )rJ  i  zdecoder.weightz&model.embeddings.tok_embeddings.weightc                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   Ú"ModernBertForMaskedLM.__annotate__&  ó   ø€ ÷ ñ Ñ/ñ rR   c                óv  <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        \        P                  ! VP                  VP                  VP                  R 7      V n        V P                  P                  V n        V P                  P                  V n        V P                  4        R# )rÀ   N)r:   r¤   rœ   rb  r*  rH  Úheadr   rÁ   rd   rc   rw   rK  r}   r~   rm  r«   s   &&€r@   r¤   ÚModernBertForMaskedLM.__init__&  sƒ   ø€ Ü‰Ñ˜Ô ØŒÜ$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—y’y ×!3Ñ!3°V×5FÑ5FÈV×M`ÑM`ÔaˆŒà!%§¡×!>Ñ!>ˆÔØ(,¯©×(LÑ(LˆÔ%ð 	�‰ÖrR   c                ó   € V P                   # rµ   ©rK  r[   s   &r@   Úget_output_embeddingsÚ+ModernBertForMaskedLM.get_output_embeddings3  s   € Ø�|‰|ÐrR   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )rb   Únew_embeddings)r   rÁ   )r†   r‡   s   "€r@   rˆ   r�  6  s   ø€ ÷ &ñ &±B·I±Iñ &rR   c                ó   € Wn         R # rµ   r“  )r<   r—  s   &&r@   Úset_output_embeddingsÚ+ModernBertForMaskedLM.set_output_embeddings6  s   € Ø%ŽrR   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# ©	rb   r®   Nrÿ   rw  r¯   Úlabelsr=   r°   )r±   r²   r³   r   r   rÚ   r   )r†   r‡   s   "€r@   rˆ   r�  ;  s”   ø€ ÷ '
ñ '
á×#Ñ# dÕ*ð'
ñ Ÿ™ tÕ+ð'
ñ —l‘l TÕ)ð	'
ñ
 —|‘| dÕ*ð'
ñ —‘˜tÕ#ð'
ñ Ñ+Õ,ð'
ñ 
‰u�|‰|Õ	™~Õ	-ñ'
rR   c                óü  € V P                   ! RR VRVRVRV/VB pV^ ,          pV P                  '       dW   VeS   VP                  R4      pVP                  VP                  ^ ,          R4      pWPP                  8g  p	W‰,          pWY,          pV P                  V P                  V4      4      p
RpVe*   V P                  ! W¥3RV P                  P                  /VB p\        VV
VP                  VP                  R7      # )	r®   rÿ   rw  r¯   Nrc   ©ÚlossÚlogitsr¶   r+  r4   rÌ   )r*  r}   r  r  r~   rK  r�  Úloss_functionrœ   rc   r   r¶   r+  )r<   r®   rÿ   rw  r¯   r�  r=   Úoutputsrz  Úmask_tokensr¡  r   s   &&&&&&,     r@   r·   ÚModernBertForMaskedLM.forward9  s  € ð —*’*ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð $ A�JÐà×!×!Ð! fÒ&8à—[‘[ “_ˆFØ 1× 6Ñ 6°v·|±|ÀAµÈÓ KÐð !×$AÑ$AÑAˆKØ 1Õ >ÐØÕ(ˆFà—‘˜dŸi™iÐ(9Ó:Ó;ˆàˆØÒØ×%Ò% fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rR   )rœ   rK  r�  r*  r~   r}   ©NNNNN)rŠ   r‹   rŒ   r�   Ú_tied_weights_keysr¤   r”  r™  r   r   r·   r”   r•   r–   r—   s   @@r@   rJ  rJ    sR   ù‡ € ð +Ð,TÐUÐ÷ó ò÷&ð &ð Ø÷'
ñ '
ó ó ÷'
ð '
rR   rJ  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )rL  ie  c                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   Ú0ModernBertForSequenceClassification.__annotate__k  rŽ  rR   c                óp  <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        \        V4      V n        \        P                  P                  VP                  4      V n        \        P                  ! VP                  VP                  4      V n        V P!                  4        R # rµ   )r:   r¤   Ú
num_labelsrœ   rb  r*  rH  r�  r±   r   r©   ry   rª   rÁ   rd   rP  rm  r«   s   &&€r@   r¤   Ú,ModernBertForSequenceClassification.__init__k  s€   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrR   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# rœ  )r±   r²   r³   r   r   rÚ   r   )r†   r‡   s   "€r@   rˆ   rª  z  sž   ø€ ÷ C
ñ C
á×#Ñ# dÕ*ðC
ñ Ÿ™ tÕ+ðC
ñ —l‘l TÕ)ð	C
ñ
 —|‘| dÕ*ðC
ñ —‘˜tÕ#ðC
ñ Ñ+Õ,ðC
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñC
rR   c                óº  € V P                   ! RRVRVRVRV/VB pV^ ,          pV P                  P                  R8X  d   VR,          pM V P                  P                  R8X  d†   VfC   \        P                  ! VP
                  R,          VP                  \        P                  R	7      pW‚P                  R4      ,          P                  ^R
7      VP                  ^RR7      ,          pV P                  V4      p	V P                  V	4      p	V P                  V	4      p
RpVEe˜   V P                  P                  f–   V P                  ^8X  d   RV P                  n        MrV P                  ^8”  dQ   VP                  \        P                   8X  g    VP                  \        P"                  8X  d   RV P                  n        MRV P                  n        V P                  P                  R8X  dK   \%        4       pV P                  ^8X  d'   V! V
P'                  4       VP'                  4       4      pMŽV! W¥4      pM…V P                  P                  R8X  d>   \)        4       pV! V
P+                  RV P                  4      VP+                  R4      4      pM-V P                  P                  R8X  d   \-        4       pV! W¥4      p\/        VV
VP0                  VP2                  R7      # )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r®   rÿ   rw  r¯   r/   r€   N:Nrb   N)rÖ   rà   rÊ   T©rË   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrŸ  r4   ):NNNr   rÌ   )r*  rœ   rx   r±   Úonesr  rÖ   r9   rá   Úsumr�  rª   rP  Úproblem_typer¬  rà   Úlongr�   r	   Úsqueezer   r  r   r   r¶   r+  )r<   r®   rÿ   rw  r¯   r�  r=   r£  rz  Úpooled_outputr¡  r   Úloss_fcts   &&&&&&,      r@   r·   Ú+ModernBertForSequenceClassification.forwardx  sj  € ð" —*’*ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð $ A�JÐà�;‰;×)Ñ)¨UÔ2Ø 1°$Õ 7ÑØ�[‰[×+Ñ+¨vÔ5ØÒ%Ü!&§¢Ø%×+Ñ+¨BÕ/Ð8I×8PÑ8PÔX]×XbÑXbô"�ð "3×5MÑ5MÈbÓ5QÕ!Q× VÑ VÐ[\Ð VÓ ]Ð`n×`rÑ`rØ˜tð asó aõ !Ðð Ÿ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rR   )rP  rœ   rª   r�  r*  r¬  r¦  ©rŠ   r‹   rŒ   r�   r¤   r   r   r·   r”   r•   r–   r—   s   @@r@   rL  rL  e  s9   ù‡ € ÷ó ð Ø÷C
ñ C
ó ó ÷C
ð C
rR   rL  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )rN  iÀ  c                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   Ú-ModernBertForTokenClassification.__annotate__Æ  re  rR   c                ód  <€ \         SV `  V4       VP                  V n        \        V4      V n        \        V4      V n        \        P                  P                  VP                  4      V n        \        P                  ! VP                  VP                  4      V n        V P                  4        R # rµ   ©r:   r¤   r¬  rb  r*  rH  r�  r±   r   r©   ry   rª   rÁ   rd   rP  rm  r«   s   &&€r@   r¤   Ú)ModernBertForTokenClassification.__init__Æ  s{   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrR   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# rœ  )r±   r²   r³   r   r   rÚ   r   )r†   r‡   s   "€r@   rˆ   rÀ  Ô  s•   ø€ ÷ $
ñ $
á×#Ñ# dÕ*ð$
ñ Ÿ™ tÕ+ð$
ñ —l‘l TÕ)ð	$
ñ
 —|‘| dÕ*ð$
ñ —‘˜tÕ#ð$
ñ Ñ+Õ,ð$
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñ$
rR   c                óx  € V P                   ! RRVRVRVRV/VB pV^ ,          pV P                  V4      pV P                  V4      pV P                  V4      p	Rp
Ve=   \	        4       pV! V	P                  RV P                  4      VP                  R4      4      p
\        V
V	VP                  VP                  R7      # )	zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
r®   rÿ   rw  r¯   NrŸ  r4   rÌ   )
r*  r�  rª   rP  r   r  r¬  r   r¶   r+  )r<   r®   rÿ   rw  r¯   r�  r=   r£  rz  r¡  r   r»  s   &&&&&&,     r@   r·   Ú(ModernBertForTokenClassification.forwardÒ  sÖ   € ð —*’*ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð $ A�JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rR   ©rP  rª   r�  r*  r¬  r¦  r½  r—   s   @@r@   rN  rN  À  s5   ù‡ € ÷
ó 
ð Ø÷$
ñ $
ó ó ÷$
ð $
rR   rN  c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )rO  iû  c                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   Ú+ModernBertForQuestionAnswering.__annotate__ý  s   ø€ ÷ 	ñ 	Ñ/ñ 	rR   c                ód  <€ \         SV `  V4       VP                  V n        \        V4      V n        \        V4      V n        \        P                  P                  VP                  4      V n        \        P                  ! VP                  VP                  4      V n        V P                  4        R # rµ   rÂ  r«   s   &&€r@   r¤   Ú'ModernBertForQuestionAnswering.__init__ý  sy   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰ÖrR   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# )	rb   r®   Nrÿ   rw  Ústart_positionsÚend_positionsr=   r°   )r±   r³   r   r   rÚ   r   )r†   r‡   s   "€r@   rˆ   rÊ  
  s“   ø€ ÷ #
ñ #
á—<‘< $Õ&ð#
ñ Ÿ™ tÕ+ð#
ñ —l‘l TÕ)ð	#
ñ
 Ÿ™¨Õ,ð#
ñ —|‘| dÕ*ð#
ñ Ñ+Õ,ð#
ñ 
‰u�|‰|Õ	Ñ;Õ	;ñ#
rR   c                óÒ  € V P                   ! V3R VRV/VB pV^ ,          pV P                  V4      pV P                  V4      pV P                  V4      p	V	P	                  ^RR7      w  r«V
P                  R4      P                  4       p
VP                  R4      P                  4       pRpVe   Ve   V P                  ! W«WE3/ VB p\        VV
VVP                  VP                  R7      # )rÿ   rw  rÊ   N)r   Ústart_logitsÚ
end_logitsr¶   r+  rÌ   )r*  r�  rª   rP  Úsplitr¹  r  r¢  r   r¶   r+  )r<   r®   rÿ   rw  rÎ  rÏ  r=   r£  rz  r¡  rÑ  rÒ  r   s   &&&&&&,      r@   r·   Ú&ModernBertForQuestionAnswering.forward  s  € ð —*’*Øñ
à)ð
ð &ð
ð ñ	
ˆð $ A�JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆà#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆØÒ&¨=Ò+DØ×%Ò% lÀÑiÐbhÑiˆDä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rR   rÇ  r¦  r½  r—   s   @@r@   rO  rO  û  s5   ù‡ € ÷	ó 	ð Ø÷#
ñ #
ó ó ÷#
ð #
rR   rO  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )rM  i0  c                ó    <€ V ^8„  d   QhRS[ /# r›   r�   )r†   r‡   s   "€r@   rˆ   Ú(ModernBertForMultipleChoice.__annotate__6  re  rR   c                ó:  <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        \        P                  P                  VP                  4      V n        \        P                  ! VP                  ^4      V n        V P                  4        R# )é   N)r:   r¤   rœ   rb  r*  rH  r�  r±   r   r©   ry   rª   rÁ   rd   rP  rm  r«   s   &&€r@   r¤   Ú$ModernBertForMultipleChoice.__init__6  sm   ø€ Ü‰Ñ˜Ô ØŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÖrR   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# rœ  )r±   r²   r³   r   r   rÚ   r   )r†   r‡   s   "€r@   rˆ   r×  D  sž   ø€ ÷ C
ñ C
á×#Ñ# dÕ*ðC
ñ Ÿ™ tÕ+ðC
ñ —l‘l TÕ)ð	C
ñ
 —|‘| dÕ*ðC
ñ —‘˜tÕ#ðC
ñ Ñ+Õ,ðC
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñC
rR   c                ó8  € Ve   VP                   ^,          MVP                   ^,          pVe"   VP                  RVP                  R4      4      MRpVe"   VP                  RVP                  R4      4      MRpVe"   VP                  RVP                  R4      4      MRpVe2   VP                  RVP                  R4      VP                  R4      4      MRpV P                  ! RRVRVRVRV/VB pV^ ,          p	V P                  P
                  R8X  d¡   \        P                  ! V	P                   ^ ,          V	P                  R7      p
Ve-   VP                  RR7      P                  V	P                  4      pM2\        P                  ! R^ \        P                  V	P                  R	7      pWšV3,          p	M\V P                  P
                  R
8X  dB   VP                  ^RR7      pW’P                  R4      ,          P                  ^R7      V,          p	V P                  V	4      pV P!                  V4      pV P#                  V4      pVP                  RV4      pRpVe   \$        P&                  ! 4       pV! Wõ4      p\)        VVVP*                  VP,                  R7      # )a  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
Nr®   rÿ   rw  r¯   r/   ry  rÊ   )rà   rÖ   r€   Tr°  rŸ  rÌ   éþÿÿÿr4   )r  r  Úsizer*  rœ   rx   r±   r{  rÖ   Úargmaxrâ   Úfullr¸  r¶  rá   r�  rª   rP  r   r   r   r¶   r+  )r<   r®   rÿ   rw  r¯   r�  r=   Únum_choicesr£  rz  Ú	indices_0Úcls_maskÚnum_non_pad_tokensrº  r¡  Úreshaped_logitsr   r»  s   &&&&&&,           r@   r·   Ú#ModernBertForMultipleChoice.forwardB  s{  € ð  -6Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*’*ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð $ A�JÐð �;‰;×)Ñ)¨UÔ2ÜŸšÐ%6×%<Ñ%<¸QÕ%?ÐHY×H`ÑH`ÔaˆIàÒ)Ø)×0Ñ0°RÐ0Ó8×;Ñ;Ð<M×<TÑ<TÓU‘ô !Ÿ:š: b¨!´5·:±:ÐFW×F^ÑF^Ô_�à 1¸XÐ2EÕ FÑð �[‰[×+Ñ+¨vÔ5Ø!/×!3Ñ!3¸À4Ð!3Ó!HÐØ!2×5MÑ5MÈbÓ5QÕ!Q× VÑ VÐ[\Ð VÓ ]Ð`rÕ rÐàŸ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÒÜ×*Ò*Ó,ˆHÙ˜OÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rR   )rP  rœ   rª   r�  r*  r¦  r½  r—   s   @@r@   rM  rM  0  s9   ù‡ € ÷
ó 
ð Ø÷C
ñ C
ó ó ÷C
ð C
rR   rM  )rÙ  )r'   rb  r)  rJ  rL  rN  rO  rM  )LrF  Úcollections.abcr   Útypingr   r±   Úhuggingface_hub.dataclassesr   r   Útorch.nnr   r   r	   Ú r
   r7  Úactivationsr   Úconfiguration_utilsr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r    Úutils.output_capturingr!   Úalign.modeling_alignr"   Úgemma3.modeling_gemma3r#   r$   Ú
get_loggerrŠ   Úloggerr'   r.  r™   r¼   rÒ   rë   rí   r  r)  rb  rH  rJ  rL  rN  rO  rM  Ú__all__r4   rR   r@   Ú<module>rý     s5  ðó  Ý $Ý ã Ý .Ý ß AÑ Aå &Ý !Ý 3ß Lß `Ý 9÷÷ õ 7ß FÝ &ß @Ñ @ß IÝ 5Ý :ß Gð 
×	Ò	˜HÓ	%€ñ Ð8Ô9ØôG)Ð'ó G)ó ó :ðG)ôT˜2Ÿ9™9ô ô,:�B—I‘Iô :ô(KÐ 5ô Kñ Ð-Ó.óBó /ðBñ4 Ð)Ó*ôN)˜"Ÿ)™)ó N)ó +ðN)ôbÐ7ô ð@ ôG^ ó G^ó ðG^ðT ôB@Ð/ó B@ó ðB@ôJ	>˜rŸy™yô 	>ñ ðôô
?
Ð5ó ?
óð
?
ñD ðôô
S
Ð*Có S
óð
S
ñl ðôô
3
Ð'@ó 3
óð
3
ðl ô1
Ð%>ó 1
ó ð1
ñh ðôô
R
Ð";ó R
óð
R
òj	‚rR   