+
    œ°³jŽ  ã                   ón  € ^ RI t ^ RIHt ^ RIt^ RIHt ^ RIHtHtHt ^RI	H
t ^RIHt ^RIHtHt ^RIHtHt ^R	IHt ^R
IHtHtHtHtHtHt ^RIHtHt ^RI H!t!H"t" ^RI#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+H,t,H-t- ^RI.H/t/ ^RI0H1t1  ! R R]Pd                  4      t3 ! R R]Pd                  4      t4 ! R R]Pd                  4      t5R7R R llt6R t7]! R4      R8R l4       t8]! ]84       ! R R]Pd                  4      4       t9 ! R  R!]4      t:]' ! R" R#]"4      4       t;]' ! R$ R%];4      4       t< ! R& R']Pd                  4      t=]'! R(R)7       ! R* R+];4      4       t>]'! R,R)7       ! R- R.];4      4       t?]'! R/R)7       ! R0 R1];4      4       t@]' ! R2 R3];4      4       tA]'! R4R)7       ! R5 R6];4      4       tB. R9OtCR# ):é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Údeprecate_kwarg)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚModernBertConfigc                   óX   a a€ ] tR t^4t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚModernBertEmbeddingszN
Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
c                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr!   )ÚformatÚ__classdict__s   "€Ús/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/modernbert/modeling_modernbert.pyÚ__annotate__Ú!ModernBertEmbeddings.__annotate__9   s   ø€ ÷ 9ñ 9Ñ/ñ 9ó    c                ój  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# ))Úpadding_idx©ÚepsÚbiasN)ÚsuperÚ__init__r(   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚtok_embeddingsÚ	LayerNormÚnorm_epsÚ	norm_biasÚnormÚDropoutÚembedding_dropoutÚdrop©Úselfr(   Ú	__class__s   &&€r+   r5   ÚModernBertEmbeddings.__init__9   su   ø€ Ü‰ÑÔØŒÜ Ÿlšl¨6×+<Ñ+<¸f×>PÑ>PÐ^d×^qÑ^qÔrˆÔÜ—L’L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ô`ˆŒ	Ü—J’J˜v×7Ñ7Ó8ˆŽ	r.   c                ó„   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  /# )r'   Ú	input_idsNÚinputs_embedsÚreturn)ÚtorchÚ
LongTensorÚTensor)r)   r*   s   "€r+   r,   r-   @   s>   ø€ ÷ ñ Ù×)Ñ)¨DÕ0ðÙHMÏÉÐW[ÕH[ðá	�‰ñr.   c                ó°   € Ve#   V P                  V P                  V4      4      pV# V P                  V P                  V P                  V4      4      4      pV# ©N)rA   r>   r:   )rC   rG   rH   Úhidden_statess   &&& r+   ÚforwardÚModernBertEmbeddings.forward@   sS   € ð Ò$Ø ŸI™I d§i¡i°Ó&>Ó?ˆMð Ðð !ŸI™I d§i¡i°×0CÑ0CÀIÓ0NÓ&OÓPˆMØÐr.   )r(   rA   r>   r:   ©NN©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r5   rP   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rD   r*   s   @@r+   r$   r$   4   s#   ù‡ € ñ÷9ó 9÷÷ ò r.   r$   c                   óT   a a€ ] tR t^Jt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚModernBertMLPa*  Applies the GLU at the end of each ModernBERT layer.

Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
c                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   ÚModernBertMLP.__annotate__Q   s   ø€ ÷ `ñ `Ñ/ñ `r.   c                óÂ  <€ \         SV `  4        Wn        \        P                  ! VP
                  \        VP                  4      ^,          VP                  R7      V n	        \        VP                  ,          V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP
                  VP                  R7      V n        R# )r'   ©r3   N)r4   r5   r(   r   ÚLinearr8   ÚintÚintermediate_sizeÚmlp_biasÚWir	   Úhidden_activationÚactr?   Úmlp_dropoutrA   ÚWorB   s   &&€r+   r5   ÚModernBertMLP.__init__Q   s‘   ø€ Ü‰ÑÔØŒÜ—)’)˜F×.Ñ.´°F×4LÑ4LÓ0MÐPQÕ0QÐX^×XgÑXgÔhˆŒÜ˜&×2Ñ2Õ3ˆŒÜ—J’J˜v×1Ñ1Ó2ˆŒ	Ü—)’)˜F×4Ñ4°f×6HÑ6HÈvÏÉÔ_ˆŽr.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r'   rO   rI   ©rJ   rL   )r)   r*   s   "€r+   r,   r`   Y   s#   ø€ ÷ :ñ :¡U§\¡\ð :±e·l±lñ :r.   c                ó¶   € V P                  V4      P                  ^RR7      w  r#V P                  V P                  V P	                  V4      V,          4      4      # )r'   ©Údiméÿÿÿÿ)rg   Úchunkrk   rA   ri   )rC   rO   ÚinputÚgates   &&  r+   rP   ÚModernBertMLP.forwardY   sG   € Ø—g‘g˜mÓ,×2Ñ2°1¸"Ð2Ó=‰ˆØ�w‰w�t—y‘y §¡¨%£°4Õ!7Ó8Ó9Ð9r.   )rg   rk   ri   r(   rA   rS   r\   s   @@r+   r^   r^   J   s%   ù‡ € ñ÷`ó `÷:÷ :ð :r.   r^   c                   óÎ   a a€ ] tR t^^t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚModernBertRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   Ú&ModernBertRotaryEmbedding.__annotate__`   s   ø€ ÷ Uñ UÑ/ñ Ur.   c           
     óÞ  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n        / V n	        V P                   Fû  pV P
                  P                  V,          pVf   K&  VR,          V P                  V&   V P                  pV P                  V,          R8w  d   \        V P                  V,          ,          pV! V P
                  W#R7      w  rg\        W R2\        P                  ! VRR7      4       \        W R2\        P                  ! VP!                  4       RR7      4       \        W R2V4       Ký  	  R # )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r4   r5   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr(   ÚsortedÚsetÚlayer_typesr   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr   ÚBufferÚclone)	rC   r(   rz   r‚   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingrD   s	   &&&     €r+   r5   Ú"ModernBertRotaryEmbedding.__init___   s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà)4°[Õ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÕ)¨YÔ6Ü2°4·>±>À*Õ3MÕN�Ù4@ÀÇÁÈfÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó +r.   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r'   r(   r‚   NrI   )r"   ÚstrÚtuplerJ   rL   Úfloat)r)   r*   s   "€r+   r,   r}   w   s7   ø€ ÷ 5ñ 5Ù ð5Ù;>À½:ð5á	‰u�|‰|™UÐ"Õ	#ñ5r.   c           	     ó>  € V P                   V,          R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©Údtype)r�   Úgetattrr8   Únum_attention_headsrJ   Úarangerš   Úto)r(   rz   r‚   ÚkwargsÚbaserr   Úattention_factorÚinv_freqs   &&&,    r+   rŽ   Ú9ModernBertRotaryEmbedding.compute_default_rope_parametersu   sƒ   € ð& ×%Ñ% jÕ1°,Õ?ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r.   c                óZ  € \        W R 24      p\        W R24      pVR
,          P                  VP                  ^ ,          R^4      P                  \        P
                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	\        P                  ! W™3RR7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP                  VP                  R	7      XP                  VP                  R	7      3#   + '       g   i     LH; i)rƒ   r†   N©rŸ   rz   ÚmpsÚcpuF)Údevice_typeÚenabledrq   rž   )NºNNNNrs   )r¯   Nr¯   )r    ÚexpandÚshaper£   rJ   rš   rz   Ú
isinstanceÚtyper˜   r   Ú	transposeÚcatÚcosÚsinrŸ   )rC   ÚxÚposition_idsr‚   r§   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr­   ÚfreqsÚembr¶   r·   s   &&&&         r+   rP   Ú!ModernBertRotaryEmbedding.forward�   sJ  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã:A FÆF*	)r(   rŒ   rˆ   r‰   r   rN   rR   )rT   rU   rV   rW   r   r5   ÚstaticmethodrŽ   rJ   Úno_gradr   rP   rY   rZ   r[   r\   s   @@r+   ry   ry   ^   su   ù‡ € Ù�X vÔ.÷Uô Uó /ðUð* Ù�X vÔ.÷5ñ 5ó /ó ð5ð2 ‡]‚]ƒ_Øñ<ó ó ÷<ð <r.   ry   c                óÞ   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        /# )	r'   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropout)r   ÚModulerJ   rL   rš   )r)   s   "r+   r,   r,   ¦   sg   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ñ%r.   c                 óÎ  € \         P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P                  P                  VR\         P                  R7      P                  VP                  4      p\        P                  P                  W†V P                  R7      p\         P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )r'   )rr   rŸ   )ÚpÚtrainingrs   )rJ   Úmatmulr´   r   Ú
functionalÚsoftmaxÚfloat32r£   rŸ   rÉ   rÍ   Ú
contiguous)
rÃ   rÄ   rÅ   rÆ   rÇ   rÈ   rÉ   r¤   Úattn_weightsÚattn_outputs
   &&&&&&&,  r+   Úeager_attention_forwardrÕ   ¦   s°   € ô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$r.   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrq   rs   )r±   rJ   rµ   )r¸   Úx1Úx2s   &  r+   Úrotate_halfrÙ   ¼   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r.   Úrotary_pos_embc                ó’  € V P                   pVP                  V4      pVP                  V4      pV P                  4       V,          \        V P                  4       4      V,          ,           pVP                  4       V,          \        VP                  4       4      V,          ,           pVP	                  V4      VP	                  V4      3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)rŸ   Ú	unsqueezerš   rÙ   r£   )ÚqÚkr¶   r·   Úunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds   &&&&&   r+   Úapply_rotary_pos_embrã   Ã   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3�¤;¨q¯w©w«yÓ#9¸CÕ#?Õ@€GØ�w‰w‹y˜3�¤;¨q¯w©w«yÓ#9¸CÕ#?Õ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐAr.   c                   ó\   a a€ ] tR t^Þt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚModernBertAttentionan  Performs multi-headed self attention on a batch of unpadded sequences.

If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
which requires padding and unpadding inputs, adding some overhead.

See `forward` method for additional details.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r'   r(   Ú	layer_idxN©r"   rd   )r)   r*   s   "€r+   r,   Ú ModernBertAttention.__annotate__é   s$   ø€ ÷ rñ rÑ/ð r¹CÀ$½Jñ rr.   c                ó–  <€ \         SV `  4        Wn        W n        VP                  VP
                  ,          ^ 8w  d'   \        RVP                   RVP
                   R24      hVP                  V n        VP                  V n        VP                  VP
                  ,          V n	        \        P                  ! VP                  ^V P                  ,          VP
                  ,          VP                  R7      V n        VP                  V,          R8X  d   VP                  ^,           V n        MRV n        RV n        \        P                  ! VP                  VP                  VP                  R7      V n        VP                  R8”  d(   \        P$                  ! VP                  4      V n        R# \        P&                  ! 4       V n        R# )	r   zThe hidden size (z6) is not a multiple of the number of attention heads (Ú)rb   Úsliding_attentionNFç        )r4   r5   r(   rè   r8   r¡   Ú
ValueErrorÚattention_dropoutÚdeterministic_flash_attnr�   r   rc   Úattention_biasÚWqkvrŒ   Úsliding_windowÚ	is_causalrk   r?   ÚIdentityÚout_drop©rC   r(   rè   rD   s   &&&€r+   r5   ÚModernBertAttention.__init__é   sv  ø€ Ü‰ÑÔØŒØ"Œà×Ñ × :Ñ :Õ:¸aÔ?ÜØ# F×$6Ñ$6Ð#7Ð7mÐnt÷  oIñ  oIð  nJð  JKð  Lóð ð "(×!9Ñ!9ˆÔØ(.×(GÑ(GˆÔ%Ø×*Ñ*¨f×.HÑ.HÕHˆŒÜ—I’IØ×Ñ  D§M¡MÕ 1°F×4NÑ4NÕ NÐU[×UjÑUjô
ˆŒ	ð ×Ñ˜iÕ(Ð,?Ô?ð #)×"7Ñ"7¸!Õ";ˆDÕà"&ˆDÔàˆŒä—)’)˜F×.Ñ.°×0BÑ0BÈ×I^ÑI^Ô_ˆŒØ@F×@XÑ@XÐ[^Ô@^œŸ
š
 6×#;Ñ#;Ó<ˆŽÔdf×doÒdoÓdqˆŽr.   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r'   rO   Úposition_embeddingsNrÇ   r¤   rI   )rJ   rL   r™   r   r   )r)   r*   s   "€r+   r,   rê     s~   ø€ ÷ ')ñ ')á—|‘|ð')ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð')ñ Ÿ™ tÕ+ð	')ñ
 Ñ+Õ,ð')ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ')r.   c                óä  € VP                   R R pV P                  V4      pVP                  ! . VO^NRNV P                  N5!  pVP	                  R	R7      w  rxp	VP                  ^^4      pVP                  ^^4      pV	P                  ^^4      p	Vw  r«\        WxW«^R7      w  rx\        P                  ! V P                  P                  \        4      pT! V VVV	V3RV P                  '       d   V P                  MRRV P                  R
,          RV P                  RV P                  /VB w  rÞVP                   ! . VORN5!  P#                  4       pV P%                  V P'                  V4      4      pWÞ3# )Nrq   )rß   rÉ   rî   rÈ   rô   Údeterministicrs   éýÿÿÿç      à¿)r±   ró   Úviewr�   Úunbindr´   rã   r   Úget_interfacer(   Ú_attn_implementationrÕ   rÍ   rð   rô   rñ   ÚreshaperÒ   r÷   rk   )rC   rO   rû   rÇ   r¤   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesr¶   r·   Úattention_interfacerÔ   rÓ   s   &&&&,          r+   rP   ÚModernBertAttention.forward  s†  € ð $×)Ñ)¨#¨2Ð.ˆà�i‰i˜Ó&ˆØ�hŠhÐ:˜Ð: QÐ:¨Ð:¨D¯M©MÓ:ˆØ14·±À°Ó1CÑ.ˆ ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÐjkÔ#lÑ ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð /3¯m¯m¨m�D×*Ò*Àð%
ð —M‘M 4Õ'ð%
ð  ×.Ñ.ð%
ð ×7Ñ7ð%
ð ñ%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—m‘m D§G¡G¨KÓ$8Ó9ˆØÐ(Ð(r.   )
rk   ró   rð   r(   rñ   r�   rõ   rè   r÷   rô   rN   rR   rS   r\   s   @@r+   rå   rå   Þ   s%   ù‡ € ñ÷rõ r÷:')÷ ')ò ')r.   rå   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚModernBertEncoderLayeri0  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# rç   ré   )r)   r*   s   "€r+   r,   Ú#ModernBertEncoderLayer.__annotate__1  s    ø€ ÷ <ñ <Ñ/ð <¹CÀ$½Jñ <r.   c                óâ  <€ \         SV `  4        Wn        W n        V^ 8X  d   \        P
                  ! 4       V n        M<\        P                  ! VP                  VP                  VP                  R7      V n        \        WR7      V n        \        P                  ! VP                  VP                  VP                  R7      V n        \        V4      V n        VP                   V,          V n        R# )r   r1   )r(   rè   N)r4   r5   r(   rè   r   rö   Ú	attn_normr;   r8   r<   r=   rå   ÚattnÚmlp_normr^   ÚmlprŒ   Úattention_typerø   s   &&&€r+   r5   ÚModernBertEncoderLayer.__init__1  s¡   ø€ Ü‰ÑÔØŒØ"ŒØ˜Œ>ÜŸ[š[›]ˆD�NäŸ\š\¨&×*<Ñ*<À&Ç/Á/ÐX^×XhÑXhÔiˆDŒNÜ'¨vÔKˆŒ	ÜŸš V×%7Ñ%7¸V¿_¹_ÐSY×ScÑScÔdˆŒÜ  Ó(ˆŒØ$×0Ñ0°Õ;ˆÖr.   c          
      ó´   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[ P                  /# )r'   rO   rÇ   Nrû   r¤   rI   )rJ   rL   r   r   )r)   r*   s   "€r+   r,   r  >  s]   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ #Ÿ\™\¨DÕ0ð	ñ
 Ñ+Õ,ðñ 
�‰ñr.   c                ó²   € V P                   ! V P                  V4      3R VRV/VB w  rVW,           pWP                  V P                  V4      4      ,           pV# )rû   rÇ   )r  r  r  r  )rC   rO   rÇ   rû   r¤   rÔ   Ú_s   &&&&,  r+   rP   ÚModernBertEncoderLayer.forward>  sf   € ð ŸšØ�N‰N˜=Ó)ñ
à 3ð
ð *ð
ð ñ	
‰ˆð &Õ3ˆØ%¯©°·±¸}Ó1MÓ(NÕNˆØÐr.   )r  r  r  r(   rè   r  r  rN   rR   ©	rT   rU   rV   rW   r5   rP   rY   rZ   r[   r\   s   @@r+   r  r  0  s   ù‡ € ÷<õ <÷÷ ò r.   r  c                   óž   a a€ ] tR tRt oRtRtRR.tRtRtRt	Rt
R]R]/t]P                  ! 4       V3R lV 3R	 ll4       tV3R
 ltRtVtV ;t# )ÚModernBertPreTrainedModeliP  ÚmodelTr$   r  rO   Ú
attentionsc                ó4   <€ V ^8„  d   QhRS[ P                  /# )r'   rÃ   )r   rÊ   )r)   r*   s   "€r+   r,   Ú&ModernBertPreTrainedModel.__annotate__a  s   ø€ ÷ 7^ñ 7^¡B§I¡Iñ 7^r.   c           	     ó  <a€ \         S	V `  V4       V P                  P                  oSf   ^oR V3R llpRV P                  P                  RV P                  P                  \
        P                  ! RV P                  P                  ,          4      ,          RV P                  P                  RV P                  P                  R,          /p\        V\        4      '       d   V! VP                  VR,          4       R # \        V\        4      '       d7   V! VP                  VR,          4       V! VP                  VR,          4       R # \        V\        4      '       d7   V! VP                   VR,          4       V! VP                  VR,          4       R # \        V\"        4      '       d   V! VP$                  VR,          4       R # \        V\&        4      '       d   V! VP(                  VR,          4       R # \        V\*        \,        \.        \0        34      '       d   V! VP2                  VR,          4       R # \        V\4        4      '       d´   VP6                   F¡  pVP8                  pVP:                  V,          R8w  d   \<        VP:                  V,          ,          pV! VP                  VR	7      w  rg\>        P@                  ! \C        W R
24      V4       \>        P@                  ! \C        W R24      V4       K£  	  R # R # )Nc                óD   € V ^8„  d   QhR\         P                  R\        /# )r'   rÃ   Ústd)r   rÊ   rš   )r)   s   "r+   r,   Ú=ModernBertPreTrainedModel._init_weights.<locals>.__annotate__g  s   € ÷ 	-ñ 	-¤§	¡	ð 	-´ñ 	-r.   c                 ó  <€ \         P                  ! V P                  R VS) V,          SV,          R7       \        V \        P
                  4      '       d3   V P                  e#   \         P                  ! V P                  4       R# R# R# )rî   )Úmeanr$  ÚaÚbN)ÚinitÚtrunc_normal_Úweightr²   r   rc   r3   Úzeros_)rÃ   r$  Úcutoff_factors   &&€r+   Úinit_weightÚ<ModernBertPreTrainedModel._init_weights.<locals>.init_weightg  sg   ø€ Ü×ÒØ—‘ØØØ �. 3Õ&Ø #Õ%õô ˜&¤"§)¡)×,Ò,Ø—;‘;Ò*Ü—K’K §¡Ö,ñ +ñ -r.   ÚinÚoutg       @Ú	embeddingÚ	final_outr€   r�   rƒ   r…   rÿ   )"r4   Ú_init_weightsr(   Úinitializer_cutoff_factorÚinitializer_rangeÚmathÚsqrtÚnum_hidden_layersr8   r²   r$   r:   r^   rg   rk   rå   ró   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierry   rŒ   rŽ   r   r   r*  Úcopy_r    )
rC   rÃ   r/  Ústdsr‚   r“   r”   r  r.  rD   s
   &&      @€r+   r5  Ú'ModernBertPreTrainedModel._init_weights`  s9  ù€ ä‰Ñ˜fÔ%ØŸ™×=Ñ=ˆØÒ ØˆM÷	-ð 	-ð �$—+‘+×/Ñ/Ø�4—;‘;×0Ñ0´4·9²9¸SÀ4Ç;Á;×C`ÑC`Õ=`Ó3aÕaØ˜Ÿ™×6Ñ6Ø˜Ÿ™×0Ñ0°$Õ6ð	
ˆô �fÔ2×3Ò3Ù˜×-Ñ-¨t°KÕ/@ÖAÜ˜¤×.Ò.Ù˜Ÿ	™	 4¨¥:Ô.Ù˜Ÿ	™	 4¨¥;Ö/Ü˜Ô 3×4Ò4Ù˜Ÿ™ T¨$¥ZÔ0Ù˜Ÿ	™	 4¨¥;Ö/Ü˜Ô 8×9Ò9Ù˜Ÿ™ d¨5¥kÖ2Ü˜Ô 5×6Ò6Ù˜Ÿ™¨¨U­Ö4ÜØä3Ü+Ü0Ü.ð	÷
ò 
ñ ˜×)Ñ)¨4°Õ+<Ö=Ü˜Ô 9×:Ò:Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó 1ñ ;r.   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r&   r!   )r)   r*   s   "€r+   r,   r!  P  s   ø‡ ‚ áÑò r.   © )rT   rU   rV   rW   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  rå   Ú_can_record_outputsrJ   rÁ   r5  Ú__annotate_func__rY   rZ   r[   r\   s   @@r+   r  r  P  su   ù‡ € ð  ÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 	Ð/ØÐ)ðÐð
 ‡]‚]ƒ_÷7^ò 7^ó ð7^÷# … r.   r  c                   ó~   a a€ ] tR tRt oV3R lV 3R lltR tR t]]]	R	V3R lR ll4       4       4       t
RtVtV ;t# )
ÚModernBertModeli›  c                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   ÚModernBertModel.__annotate__�  ó   ø€ ÷ 
ñ 
Ñ/ñ 
r.   c           	     ó²  <€ \         SV `  V4       Wn        \        V4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n
        \
        P                  ! VP                  VP                  VP                  R 7      V n        \!        VR7      V n        RV n        V P'                  4        R# u upi )r1   )r(   FN)r4   r5   r(   r$   Ú
embeddingsr   Ú
ModuleListÚranger:  r  Úlayersr;   r8   r<   r=   Ú
final_normry   Ú
rotary_embÚgradient_checkpointingÚ	post_initrø   s   && €r+   r5   ÚModernBertModel.__init__�  s¡   ø€ Ü‰Ñ˜Ô ØŒÜ.¨vÓ6ˆŒÜ—m’mÜHMÈf×NfÑNfÔHgÓhÑHg¸9Ô# FÖ6ÑHgÑhó
ˆŒô Ÿ,š, v×'9Ñ'9¸v¿¹ÐU[×UeÑUeÔfˆŒÜ3¸6ÔBˆŒØ&+ˆÔ#Ø�‰Öùò is   ÁCc                ó.   € V P                   P                  # rN   ©rX  r:   ©rC   s   &r+   Úget_input_embeddingsÚ$ModernBertModel.get_input_embeddings©  s   € Ø�‰×-Ñ-Ð-r.   c                ó&   € WP                   n        R # rN   rb  )rC   rÆ   s   &&r+   Úset_input_embeddingsÚ$ModernBertModel.set_input_embeddings¬  s   € Ø).�‰Ö&r.   c                óÖ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )r'   rG   NrÇ   r¹   rH   r¤   rI   )rJ   rK   rL   r   r   r   )r)   r*   s   "€r+   r,   rU  ²  s}   ø€ ÷ ,@ñ ,@á×#Ñ# dÕ*ð,@ñ Ÿ™ tÕ+ð,@ñ ×&Ñ&¨Õ-ð	,@ñ
 —|‘| dÕ*ð,@ñ Ñ+Õ,ð,@ñ 
ñ,@r.   c           	     óø  € VR J VR J,          '       d   \        R4      hVe   VP                  ^,          MVP                  ^,          pVe   VP                  MVP                  pVf'   \        P                  ! WgR7      P                  ^ 4      pV P                  WR7      p\        T;p	\        4      '       g+   RV P                  RVRV/p
R\        R/ V
B R\        R/ V
B /p	/ p\        V P                  P                  4       F  pV P                  WƒV4      W¼&   K  	  V P                   F0  pV! V3RW�P                   ,          R	W½P                   ,          /VB pK2  	  V P#                  V4      p\%        VR
7      # )Nz:You must specify exactly one of input_ids or inputs_embeds©rz   )rG   rH   r(   rH   rÇ   Úfull_attentionrí   rû   )Úlast_hidden_staterH  )rï   r±   rz   rJ   r¢   rÜ   rX  r²   Údictr(   r   r   r‹   rŒ   r]  r[  r  r\  r   )rC   rG   rÇ   r¹   rH   r¤   Úseq_lenrz   rO   Úattention_mask_mappingÚmask_kwargsrû   r‚   Úencoder_layers   &&&&&,        r+   rP   ÚModernBertModel.forward¯  s}  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZà,9Ò,E�-×%Ñ% aÖ(È9Ï?É?Ð[\ÕK]ˆØ%.Ò%:�×!Ò!À×@TÑ@TˆàÒÜ Ÿ<š<¨Ô?×IÑIÈ!ÓLˆLàŸ™°)˜ÓYˆä°NÐBÐ0ÄD×IÒIà˜$Ÿ+™+Ø Ø  .ðˆKð !Ô";Ñ"J¸kÑ"JØ#Ô%MÑ%\ÐP[Ñ%\ð&Ð"ð
 !ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ð "Ÿ[œ[ˆMÙ)Øñà5×6RÑ6RÕSðð %8×8TÑ8TÕ$Uðð ñ	ŠMñ )ð Ÿ™¨Ó6ˆä°Ô?Ð?r.   )r(   rX  r\  r^  r[  r]  )NNNN)rT   rU   rV   rW   r5   rd  rg  r   r    r   rP   rY   rZ   r[   r\   s   @@r+   rS  rS  ›  sK   ù‡ € ÷
ó 
ò.ò/ð  ØØ÷,@ñ ,@ó ó ó  ÷,@ð ,@r.   rS  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )r;  iá  c                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   Ú%ModernBertPredictionHead.__annotate__â  s   ø€ ÷ añ aÑ/ñ ar.   c                óV  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP
                  VP                  4      V n        \        VP                  ,          V n
        \        P                  ! VP
                  VP                  VP                  R 7      V n        R# )r1   N)r4   r5   r(   r   rc   r8   Úclassifier_biasr<  r	   Úclassifier_activationri   r;   r<   r=   r>   rB   s   &&€r+   r5   Ú!ModernBertPredictionHead.__init__â  so   ø€ Ü‰ÑÔØŒÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÀv×G]ÑG]Ó^ˆŒ
Ü˜&×6Ñ6Õ7ˆŒÜ—L’L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ô`ˆŽ	r.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rn   ro   )r)   r*   s   "€r+   r,   rv  é  s#   ø€ ÷ >ñ >¡U§\¡\ð >±e·l±lñ >r.   c                ó`   € V P                  V P                  V P                  V4      4      4      # rN   )r>   ri   r<  )rC   rO   s   &&r+   rP   Ú ModernBertPredictionHead.forwardé  s#   € Ø�y‰y˜Ÿ™ $§*¡*¨]Ó";Ó<Ó=Ð=r.   )ri   r(   r<  r>   r  r\   s   @@r+   r;  r;  á  s    ù‡ € ÷aó a÷>÷ >ð >r.   r;  zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   óˆ   a a€ ] tR tRt oRR/tV3R lV 3R lltR tV3R lR lt]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )r=  ií  zdecoder.weightz&model.embeddings.tok_embeddings.weightc                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   Ú"ModernBertForMaskedLM.__annotate__õ  ó   ø€ ÷ ñ Ñ/ñ r.   c                óv  <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        \        P                  ! VP                  VP                  VP                  R 7      V n        V P                  P                  V n        V P                  P                  V n        V P                  4        R# )rb   N)r4   r5   r(   rS  r  r;  Úheadr   rc   r8   r7   Údecoder_biasr>  Úsparse_predictionÚsparse_pred_ignore_indexr_  rB   s   &&€r+   r5   ÚModernBertForMaskedLM.__init__õ  sƒ   ø€ Ü‰Ñ˜Ô ØŒÜ$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—y’y ×!3Ñ!3°V×5FÑ5FÈV×M`ÑM`ÔaˆŒà!%§¡×!>Ñ!>ˆÔØ(,¯©×(LÑ(LˆÔ%ð 	�‰Ör.   c                ó   € V P                   # rN   ©r>  rc  s   &r+   Úget_output_embeddingsÚ+ModernBertForMaskedLM.get_output_embeddings  s   € Ø�|‰|Ðr.   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r'   Únew_embeddings)r   rc   )r)   r*   s   "€r+   r,   r�    s   ø€ ÷ &ñ &±B·I±Iñ &r.   c                ó   € Wn         R # rN   rŠ  )rC   rŽ  s   &&r+   Úset_output_embeddingsÚ+ModernBertForMaskedLM.set_output_embeddings  s   € Ø%Žr.   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# ©	r'   rG   NrÇ   r¹   rH   Úlabelsr¤   rI   )rJ   rK   rL   r   r   r™   r   )r)   r*   s   "€r+   r,   r�  
  s”   ø€ ÷ '
ñ '
á×#Ñ# dÕ*ð'
ñ Ÿ™ tÕ+ð'
ñ —l‘l TÕ)ð	'
ñ
 —|‘| dÕ*ð'
ñ —‘˜tÕ#ð'
ñ Ñ+Õ,ð'
ñ 
‰u�|‰|Õ	™~Õ	-ñ'
r.   c                óü  € V P                   ! RR VRVRVRV/VB pV^ ,          pV P                  '       dW   VeS   VP                  R4      pVP                  VP                  ^ ,          R4      pWPP                  8g  p	W‰,          pWY,          pV P                  V P                  V4      4      p
RpVe*   V P                  ! W¥3RV P                  P                  /VB p\        VV
VP                  VP                  R7      # )	rG   rÇ   r¹   rH   Nr7   ©ÚlossÚlogitsrO   r  rH  rs   )r  r†  r   r±   r‡  r>  r„  Úloss_functionr(   r7   r   rO   r  )rC   rG   rÇ   r¹   rH   r”  r¤   Úoutputsrm  Úmask_tokensr˜  r—  s   &&&&&&,     r+   rP   ÚModernBertForMaskedLM.forward  s  € ð —*’*ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð $ A�JÐà×!×!Ð! fÒ&8à—[‘[ “_ˆFØ 1× 6Ñ 6°v·|±|ÀAµÈÓ KÐð !×$AÑ$AÑAˆKØ 1Õ >ÐØÕ(ˆFà—‘˜dŸi™iÐ(9Ó:Ó;ˆàˆØÒØ×%Ò% fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r.   )r(   r>  r„  r  r‡  r†  ©NNNNN)rT   rU   rV   rW   Ú_tied_weights_keysr5   r‹  r�  r   r   rP   rY   rZ   r[   r\   s   @@r+   r=  r=  í  sR   ù‡ € ð +Ð,TÐUÐ÷ó ò÷&ð &ð Ø÷'
ñ '
ó ó ÷'
ð '
r.   r=  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )r?  i4  c                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   Ú0ModernBertForSequenceClassification.__annotate__:  r‚  r.   c                óp  <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        \        V4      V n        \        P                  P                  VP                  4      V n        \        P                  ! VP                  VP                  4      V n        V P!                  4        R # rN   )r4   r5   Ú
num_labelsr(   rS  r  r;  r„  rJ   r   r?   Úclassifier_dropoutrA   rc   r8   rC  r_  rB   s   &&€r+   r5   Ú,ModernBertForSequenceClassification.__init__:  s€   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör.   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# r“  )rJ   rK   rL   r   r   r™   r   )r)   r*   s   "€r+   r,   r¡  I  sž   ø€ ÷ C
ñ C
á×#Ñ# dÕ*ðC
ñ Ÿ™ tÕ+ðC
ñ —l‘l TÕ)ð	C
ñ
 —|‘| dÕ*ðC
ñ —‘˜tÕ#ðC
ñ Ñ+Õ,ðC
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñC
r.   c                óº  € V P                   ! RRVRVRVRV/VB pV^ ,          pV P                  P                  R8X  d   VR,          pM V P                  P                  R8X  d†   VfC   \        P                  ! VP
                  R,          VP                  \        P                  R	7      pW‚P                  R4      ,          P                  ^R
7      VP                  ^RR7      ,          pV P                  V4      p	V P                  V	4      p	V P                  V	4      p
RpVEe˜   V P                  P                  f–   V P                  ^8X  d   RV P                  n        MrV P                  ^8”  dQ   VP                  \        P                   8X  g    VP                  \        P"                  8X  d   RV P                  n        MRV P                  n        V P                  P                  R8X  dK   \%        4       pV P                  ^8X  d'   V! V
P'                  4       VP'                  4       4      pMŽV! W¥4      pM…V P                  P                  R8X  d>   \)        4       pV! V
P+                  RV P                  4      VP+                  R4      4      pM-V P                  P                  R8X  d   \-        4       pV! W¥4      p\/        VV
VP0                  VP2                  R7      # )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
rG   rÇ   r¹   rH   Úclsr'  N:Nr'   N)rz   rŸ   rq   T©rr   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationr–  rH  )r¯   r   rs   )r  r(   Úclassifier_poolingrJ   Úonesr±   rz   ÚboolrÜ   Úsumr„  rA   rC  Úproblem_typer£  rŸ   Úlongrd   r   Úsqueezer   r   r   r   rO   r  )rC   rG   rÇ   r¹   rH   r”  r¤   rš  rm  Úpooled_outputr˜  r—  Úloss_fcts   &&&&&&,      r+   rP   Ú+ModernBertForSequenceClassification.forwardG  sj  € ð" —*’*ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð $ A�JÐà�;‰;×)Ñ)¨UÔ2Ø 1°$Õ 7ÑØ�[‰[×+Ñ+¨vÔ5ØÒ%Ü!&§¢Ø%×+Ñ+¨BÕ/Ð8I×8PÑ8PÔX]×XbÑXbô"�ð "3×5MÑ5MÈbÓ5QÕ!Q× VÑ VÐ[\Ð VÓ ]Ð`n×`rÑ`rØ˜tð asó aõ !Ðð Ÿ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r.   )rC  r(   rA   r„  r  r£  r�  ©rT   rU   rV   rW   r5   r   r   rP   rY   rZ   r[   r\   s   @@r+   r?  r?  4  s9   ù‡ € ÷ó ð Ø÷C
ñ C
ó ó ÷C
ð C
r.   r?  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )rA  i�  c                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   Ú-ModernBertForTokenClassification.__annotate__•  rV  r.   c                ód  <€ \         SV `  V4       VP                  V n        \        V4      V n        \        V4      V n        \        P                  P                  VP                  4      V n        \        P                  ! VP                  VP                  4      V n        V P                  4        R # rN   ©r4   r5   r£  rS  r  r;  r„  rJ   r   r?   r¤  rA   rc   r8   rC  r_  rB   s   &&€r+   r5   Ú)ModernBertForTokenClassification.__init__•  s{   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör.   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# r“  )rJ   rK   rL   r   r   r™   r   )r)   r*   s   "€r+   r,   r»  £  s•   ø€ ÷ $
ñ $
á×#Ñ# dÕ*ð$
ñ Ÿ™ tÕ+ð$
ñ —l‘l TÕ)ð	$
ñ
 —|‘| dÕ*ð$
ñ —‘˜tÕ#ð$
ñ Ñ+Õ,ð$
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñ$
r.   c                óx  € V P                   ! RRVRVRVRV/VB pV^ ,          pV P                  V4      pV P                  V4      pV P                  V4      p	Rp
Ve=   \	        4       pV! V	P                  RV P                  4      VP                  R4      4      p
\        V
V	VP                  VP                  R7      # )	zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
rG   rÇ   r¹   rH   Nr–  rH  rs   )
r  r„  rA   rC  r   r   r£  r   rO   r  )rC   rG   rÇ   r¹   rH   r”  r¤   rš  rm  r˜  r—  r¶  s   &&&&&&,     r+   rP   Ú(ModernBertForTokenClassification.forward¡  sÖ   € ð —*’*ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð $ A�JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r.   ©rC  rA   r„  r  r£  r�  r¸  r\   s   @@r+   rA  rA  �  s5   ù‡ € ÷
ó 
ð Ø÷$
ñ $
ó ó ÷$
ð $
r.   rA  c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )rB  iÊ  c                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   Ú+ModernBertForQuestionAnswering.__annotate__Ì  s   ø€ ÷ 	ñ 	Ñ/ñ 	r.   c                ód  <€ \         SV `  V4       VP                  V n        \        V4      V n        \        V4      V n        \        P                  P                  VP                  4      V n        \        P                  ! VP                  VP                  4      V n        V P                  4        R # rN   r½  rB   s   &&€r+   r5   Ú'ModernBertForQuestionAnswering.__init__Ì  sy   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰Ör.   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# )	r'   rG   NrÇ   r¹   Ústart_positionsÚend_positionsr¤   rI   )rJ   rL   r   r   r™   r   )r)   r*   s   "€r+   r,   rÅ  Ù  s“   ø€ ÷ #
ñ #
á—<‘< $Õ&ð#
ñ Ÿ™ tÕ+ð#
ñ —l‘l TÕ)ð	#
ñ
 Ÿ™¨Õ,ð#
ñ —|‘| dÕ*ð#
ñ Ñ+Õ,ð#
ñ 
‰u�|‰|Õ	Ñ;Õ	;ñ#
r.   c                óÒ  € V P                   ! V3R VRV/VB pV^ ,          pV P                  V4      pV P                  V4      pV P                  V4      p	V	P	                  ^RR7      w  r«V
P                  R4      P                  4       p
VP                  R4      P                  4       pRpVe   Ve   V P                  ! W«WE3/ VB p\        VV
VVP                  VP                  R7      # )rÇ   r¹   rq   N)r—  Ústart_logitsÚ
end_logitsrO   r  rs   )r  r„  rA   rC  Úsplitr´  rÒ   r™  r   rO   r  )rC   rG   rÇ   r¹   rÉ  rÊ  r¤   rš  rm  r˜  rÌ  rÍ  r—  s   &&&&&&,      r+   rP   Ú&ModernBertForQuestionAnswering.forward×  s  € ð —*’*Øñ
à)ð
ð &ð
ð ñ	
ˆð $ A�JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆà#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆØÒ&¨=Ò+DØ×%Ò% lÀÑiÐbhÑiˆDä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   rÂ  r�  r¸  r\   s   @@r+   rB  rB  Ê  s5   ù‡ € ÷	ó 	ð Ø÷#
ñ #
ó ó ÷#
ð #
r.   rB  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )r@  iÿ  c                ó    <€ V ^8„  d   QhRS[ /# r&   r!   )r)   r*   s   "€r+   r,   Ú(ModernBertForMultipleChoice.__annotate__  rV  r.   c                ó:  <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        \        P                  P                  VP                  4      V n        \        P                  ! VP                  ^4      V n        V P                  4        R# )é   N)r4   r5   r(   rS  r  r;  r„  rJ   r   r?   r¤  rA   rc   r8   rC  r_  rB   s   &&€r+   r5   Ú$ModernBertForMultipleChoice.__init__  sm   ø€ Ü‰Ñ˜Ô ØŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör.   c                ó2  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# r“  )rJ   rK   rL   r   r   r™   r   )r)   r*   s   "€r+   r,   rÒ    sž   ø€ ÷ C
ñ C
á×#Ñ# dÕ*ðC
ñ Ÿ™ tÕ+ðC
ñ —l‘l TÕ)ð	C
ñ
 —|‘| dÕ*ðC
ñ —‘˜tÕ#ðC
ñ Ñ+Õ,ðC
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñC
r.   c                ó8  € Ve   VP                   ^,          MVP                   ^,          pVe"   VP                  RVP                  R4      4      MRpVe"   VP                  RVP                  R4      4      MRpVe"   VP                  RVP                  R4      4      MRpVe2   VP                  RVP                  R4      VP                  R4      4      MRpV P                  ! RRVRVRVRV/VB pV^ ,          p	V P                  P
                  R8X  d¡   \        P                  ! V	P                   ^ ,          V	P                  R7      p
Ve-   VP                  RR7      P                  V	P                  4      pM2\        P                  ! R^ \        P                  V	P                  R	7      pWšV3,          p	M\V P                  P
                  R
8X  dB   VP                  ^RR7      pW’P                  R4      ,          P                  ^R7      V,          p	V P                  V	4      pV P!                  V4      pV P#                  V4      pVP                  RV4      pRpVe   \$        P&                  ! 4       pV! Wõ4      p\)        VVVP*                  VP,                  R7      # )a  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
NrG   rÇ   r¹   rH   r¨  rk  rq   rª   r'  Tr©  r–  rs   éþÿÿÿrH  )r±   r   Úsizer  r(   r®  rJ   r¢   rz   Úargmaxr£   Úfullr³  r±  rÜ   r„  rA   rC  r   r   r   rO   r  )rC   rG   rÇ   r¹   rH   r”  r¤   Únum_choicesrš  rm  Ú	indices_0Úcls_maskÚnum_non_pad_tokensrµ  r˜  Úreshaped_logitsr—  r¶  s   &&&&&&,           r+   rP   Ú#ModernBertForMultipleChoice.forward  s{  € ð  -6Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*’*ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð $ A�JÐð �;‰;×)Ñ)¨UÔ2ÜŸšÐ%6×%<Ñ%<¸QÕ%?ÐHY×H`ÑH`ÔaˆIàÒ)Ø)×0Ñ0°RÐ0Ó8×;Ñ;Ð<M×<TÑ<TÓU‘ô !Ÿ:š: b¨!´5·:±:ÐFW×F^ÑF^Ô_�à 1¸XÐ2EÕ FÑð �[‰[×+Ñ+¨vÔ5Ø!/×!3Ñ!3¸À4Ð!3Ó!HÐØ!2×5MÑ5MÈbÓ5QÕ!Q× VÑ VÐ[\Ð VÓ ]Ð`rÕ rÐàŸ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÒÜ×*Ò*Ó,ˆHÙ˜OÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r.   )rC  r(   rA   r„  r  r�  r¸  r\   s   @@r+   r@  r@  ÿ  s9   ù‡ € ÷
ó 
ð Ø÷C
ñ C
ó ó ÷C
ð C
r.   r@  )rî   )rÔ  )rS  r  r=  r?  rA  rB  r@  )Dr8  Úcollections.abcr   rJ   r   Útorch.nnr   r   r   Ú r   r*  Úactivationsr	   Úintegrationsr
   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr    Úconfiguration_modernbertr"   rÊ   r$   r^   ry   rÕ   rÙ   rã   rå   r  r  rS  r;  r=  r?  rA  rB  r@  Ú__all__rH  r.   r+   Ú<module>ró     sû  ðó, Ý $ã Ý ß AÑ Aå &Ý !ß Lß `Ý 9÷÷ ÷ Lß FÝ &ß 7Ý 0ß YÑ YÝ 5Ý 6ô˜2Ÿ9™9ô ô,:�B—I‘Iô :ô(E< §	¡	ô E<÷P%ò,(ñ Ð-Ó.óBó /ðBñ4 Ð)Ó*ôN)˜"Ÿ)™)ó N)ó +ðN)ôbÐ7ô ð@ ôG^ ó G^ó ðG^ðT ôB@Ð/ó B@ó ðB@ôJ	>˜rŸy™yô 	>ñ ðôô
?
Ð5ó ?
óð
?
ñD ðôô
S
Ð*Có S
óð
S
ñl ðôô
3
Ð'@ó 3
óð
3
ðl ô1
Ð%>ó 1
ó ð1
ñh ðôô
R
Ð";ó R
óð
R
òj‚r.   