+
    œ°³j¦`  ã                   ó*  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIH	t	 ^RI
Ht ^RIHtHt ^RIHtHt ^RIHt ^R	IHtHtHt ^R
IHt ^RIHtHt ^RIHt ^RIH t   ! R R]PB                  4      t" ! R R]PB                  4      t# ! R R]PB                  4      t$R'R R llt%R t&R R lt'R R lt( ! R R]PB                  4      t) ! R R]4      t* ! R R ]PB                  4      t+] ! R! R"]4      4       t,]! R#R$7       ! R% R&],4      4       t-R"R&.t.R# )(é    )ÚCallableN)Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚMLCDVisionConfigc                   óD   a a€ ] tR t^&t oV 3R ltV3R lR ltRtVtV ;t# )ÚMLCDMLPc                ó(  <€ \         SV `  4        Wn        \        VP                  ,          V n        \        P                  ! VP                  VP                  4      V n
        \        P                  ! VP                  VP                  4      V n        R # ©N)ÚsuperÚ__init__Úconfigr   Ú
hidden_actÚactivation_fnÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©Úselfr   Ú	__class__s   &&€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mlcd/modeling_mlcd.pyr   ÚMLCDMLP.__init__'   sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Õ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽó    c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )é   Úhidden_statesÚreturn)ÚtorchÚTensor)ÚformatÚ__classdict__s   "€r'   Ú__annotate__ÚMLCDMLP.__annotate__.   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r)   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r   )r"   r   r#   )r%   r,   s   &&r'   ÚforwardÚMLCDMLP.forward.   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr)   )r   r   r"   r#   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r5   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r&   r1   s   @@r'   r   r   &   s   ù‡ € õK÷÷ ð r)   r   c                   óÎ   a a€ ] tR t^5t oRt]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]	P                  ! 4       R	 4       tR
 tRtVtV ;t# )ÚMLCDRotaryEmbeddingzí
Simple axial 2D rope with same freqs used for H and W grids. The freqs are
pre-computed using `head-dim//4` which is later used to concat H and W positions.
The final angles rotate over the whole head dim, no partial rotation involved.
Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r+   r   r   )r0   r1   s   "€r'   r2   Ú MLCDRotaryEmbedding.__annotate__=   s   ø€ ÷ Oñ OÑ/ñ Or)   c                óÎ  <€ \         SV `  4        Wn        V P                  P                  R ,          V n        V P
                  pV P                  R8w  d/   \        V P                  P                   RV P                   24      hV! V P                  V4      w  q@n	        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚaxialz) supports only axial rope, but requested F©Ú
persistentN)r   r   r   Úrope_parametersrH   Úcompute_axial_rope_parametersÚ
ValueErrorr&   r8   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r%   r   rB   Úrope_init_fnrQ   r&   s   &&&  €r'   r   ÚMLCDRotaryEmbedding.__init__<   s¬   ø€ ä‰ÑÔØŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!CÑ!CˆØ�>‰>˜WÔ$Ü §¡× 7Ñ 7Ð8Ð8aÐbf×bpÑbpÐaqÐrÓsÐsÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr)   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r+   r   r-   )r   Útupler.   r/   Úfloat)r0   r1   s   "€r'   r2   rF   L   s/   ø€ ÷ 5ñ 5Ñ.>ð 5ÑZ_Ñ`e×`lÑ`lÑnsÐ`sÕZtñ 5r)   c           	     óB  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pV^,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©Údtype)rL   Úgetattrr    Únum_attention_headsr.   ÚarangerX   Úto)r   rB   ÚkwargsÚbaseÚdimÚspatial_dimÚattention_factorrQ   s   &&,     r'   rM   Ú1MLCDRotaryEmbedding.compute_axial_rope_parametersJ   s‡   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆØ˜Q•hˆàÐØ˜$¤5§<¢<°°;ÀÌÏÉÔ#UÐXcÕ#cÕdÕeˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r)   c                óZ  € VR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W0P                  P                  4       ,          pVP                  4       V P                  ,          pVP                  4       V P                  ,          pRRR4       V P                  X4      pV P                  X4      pWg3#   + '       g   i     L5; i).NÚmpsÚcpuF)Údevice_typeÚenabled).N)rX   Ú
isinstancerB   ÚtypeÚstrr   rQ   ÚcosrO   ÚsinÚrecomposition_frequencies)r%   ÚxÚposition_idsÚposition_ids_expandedrk   Úfreqsrp   rq   s   &&&     r'   r5   ÚMLCDRotaryEmbedding.forward^   sÓ   € ð !-¨YÕ 7× =Ñ =Ó ?ÐÜ'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ)¯M©M×,?Ñ,?Ó,AÕAˆEØ—)‘)“+ × 6Ñ 6Õ6ˆCØ—)‘)“+ × 6Ñ 6Õ6ˆC÷ Dð
 ×,Ñ,¨SÓ1ˆØ×,Ñ,¨SÓ1ˆØˆxˆ÷ D×Cús   Â
A#DÄD*	c                óˆ   € VR,          VR,          r2\         P                  ! W#.RR7      p\         P                  ! WD.RR7      # )zM
Recompose the frequencies into the final spatial layout used per each grid.
©rd   )ºNNNr   )rz   é   éÿÿÿÿ)r.   Úcat)r%   ÚfreqÚfreq_hÚfreq_wÚfreq_hws   &&   r'   rr   Ú-MLCDRotaryEmbedding.recomposition_frequenciesl   s:   € ð ˜d� T¨$¥Z�Ü—)’)˜VÐ,°"Ô5ˆÜ�yŠy˜'Ð+°Ô4Ð4r)   )rO   r   rQ   rS   rH   r   )r8   r9   r:   r;   Ú__doc__r   r   ÚstaticmethodrM   r.   Úno_gradr5   rr   r<   r=   r>   r?   s   @@r'   rA   rA   5   sy   ù‡ € ññ �X vÔ.÷Oô Oó /ðOð Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_ñó ð÷5ò 5r)   rA   c                   ób   a a€ ] tR t^ut oV3R lV 3R lltV3R lR ltV3R lR ltRtVtV ;t	# )ÚMLCDVisionEmbeddingsc                ó    <€ V ^8„  d   QhRS[ /# rE   r   )r0   r1   s   "€r'   r2   Ú!MLCDVisionEmbeddings.__annotate__v   s   ø€ ÷ jñ jÑ/ñ jr)   c                ó®  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  4      4      V n        \        P                  ! VP                  V P                  V P                  V P                  R R7      V n        V P
                  V P                  ,          ^,          V n        V P                  ^,           V n        \        P"                  ! \        P$                  ! V P                   4      P'                  R4      R R7      V n        R# )F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasrJ   N©r{   r|   )r   r   r   r    Ú	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	Parameterr.   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsrP   r`   Úexpandrt   r$   s   &&€r'   r   ÚMLCDVisionEmbeddings.__init__v   sç   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|š|¬E¯KªK¸¿¹Ó,GÓHˆÔä!ŸyšyØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð !ŸO™O¨t¯©Õ>À1ÕDˆÔØ!×-Ñ-°Õ1ˆÔÜŸIšI¤e§l¢l°4×3EÑ3EÓ&F×&MÑ&MÈgÓ&VÐchÔiˆÖr)   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r+   Ú
embeddingsÚheightÚwidthr-   ©r.   r/   Úint)r0   r1   s   "€r'   r2   r‰   ‹   s;   ø€ ÷ 'Dñ 'D±5·<±<ð 'DÉð 'DÑUXð 'DÑ]b×]iÑ]iñ 'Dr)   c                ó  € VP                   ^,          ^,
          pV P                  P                  P                  ^ 4      pVP                   ^,          ^,
          p\        P
                  P                  4       '       g(   WF8X  d"   W#8X  d   V P                  V P                  4      # VR,          pVR,          pVP                   R,          p	W P                  ,          p
W0P                  ,          p\        VR,          4      pVP                  ^WÌV	4      pVP                  ^ ^^^4      p\        P                  P                  VW«3RRR7      pVP                  ^ ^^^4      P                  ^RV	4      p\        P                   ! Wx3^R7      # )	aÐ  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
ç      à?ÚbicubicF)ÚsizeÚmodeÚalign_cornersry   )rz   :Nr{   N)rz   :r{   NNr|   )ÚshapeÚposition_embeddingÚweightÚ	unsqueezer.   ÚjitÚ
is_tracingrt   r“   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewr}   )r%   rŸ   r    r¡   rš   r«   r›   Úclass_pos_embedÚpatch_pos_embedrd   Ú
new_heightÚ	new_widthÚsqrt_num_positionss   &&&&         r'   Úinterpolate_pos_encodingÚ-MLCDVisionEmbeddings.interpolate_pos_encoding‹   s[  € ð !×&Ñ& qÕ)¨AÕ-ˆØ!×4Ñ4×;Ñ;×EÑEÀaÓHÐØ*×0Ñ0°Õ3°aÕ7ˆô �y‰y×#Ñ#×%Ò%¨+Ô*FÈ6Ì?Ø×*Ñ*¨4×+<Ñ+<Ó=Ð=à,¨UÕ3ˆØ,¨UÕ3ˆà×Ñ˜rÕ"ˆàŸ™Õ.ˆ
ØŸ_™_Õ,ˆ	ä& }°cÕ'9Ó:ÐØ)×1Ñ1°!Ð5GÐ]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�yŠy˜/Ð;ÀÔCÐCr)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r+   Úpixel_valuesr-   )r.   ÚFloatTensorr/   )r0   r1   s   "€r'   r2   r‰   ´   s%   ø€ ÷ 
ñ 
¡E×$5Ñ$5ð 
¹%¿,¹,ñ 
r)   c                ó\  € VP                   ^ ,          pV P                  P                  P                  pV P                  VP	                  VR7      4      pVP                  ^4      P                  ^^4      pV P                  P                  V^R4      p\        P                  ! WT.^R7      pV# )r   r\   ry   r|   )rª   r™   r¬   r]   ra   ÚflattenÚ	transposer–   rœ   r.   r}   )r%   r½   Ú
batch_sizeÚtarget_dtypeÚpatch_embedsÚclass_embedsrŸ   s   &&     r'   r5   ÚMLCDVisionEmbeddings.forward´   s”   € Ø!×'Ñ'¨Õ*ˆ
Ø×+Ñ+×2Ñ2×8Ñ8ˆà×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆà×+Ñ+×2Ñ2°:¸qÀ"ÓEˆÜ—Y’Y Ð;ÀÔCˆ
àÐr)   )	r–   r   r‘   r’   rš   r›   r™   r“   rt   )
r8   r9   r:   r;   r   rº   r5   r<   r=   r>   r?   s   @@r'   r‡   r‡   u   s-   ù‡ € ÷jó j÷*'Dð 'D÷R
÷ 
ð 
r)   r‡   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r+   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrb   )r   ÚModuler.   r/   rX   r   r   )r0   s   "r'   r2   r2   Á   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r)   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r+   )rd   r]   )ÚpÚtrainingr|   )Ú	repeat_kvÚnum_key_value_groupsr.   ÚmatmulrÁ   r   r²   ÚsoftmaxÚfloat32ra   r]   rÎ   rÒ   Ú
contiguous)rÈ   rÉ   rÊ   rË   rÌ   rÍ   rÎ   rb   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r'   Úeager_attention_forwardrÝ   Á   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r)   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nry   r|   )rª   r.   r}   )rs   Úx1Úx2s   &  r'   Úrotate_halfrá   Ú   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r)   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r+   r,   Ún_repr-   r¢   )r0   s   "r'   r2   r2   á   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur)   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rz   rz   Nrz   rz   )rª   rœ   r°   )r,   rã   ÚbatchÚnum_key_value_headsÚslenr[   s   &&    r'   rÓ   rÓ   á   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr)   c                óî   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# )r+   ÚqÚkrp   rq   r-   )r.   r/   rW   )r0   s   "r'   r2   r2   í   sV   € ÷ ñ Ü‡|�|ðÜŸ™ðÜ+0¯<©<ðÜ>C¿l¹lðä
Œ5�<‰<œŸ™Ð%Õ&ñr)   c                 ó°  € V P                   pVP                   pV P                  4       VP                  4       rVP                  R4      P                  4       VP                  R4      P                  4       r2W,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pVP	                  V4      pVP	                  V4      pWg3# )r+   éþÿÿÿ)r]   rX   r­   rá   ra   )ré   rê   rp   rq   Úorig_q_dtypeÚorig_k_dtypeÚq_embedÚk_embeds   &&&&    r'   Úapply_rotary_pos_emb_visionrñ   í   s£   € ð —7‘7€LØ—7‘7€LØ�7‰7‹9�a—g‘g“i€qØ�}‰}˜RÓ ×&Ñ&Ó(¨#¯-©-¸Ó*;×*AÑ*AÓ*CˆØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØ�j‰j˜Ó&€GØ�j‰j˜Ó&€GØÐÐr)   c                   óX   a a€ ] tR t^ût oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚMLCDAttentionzëMulti-headed attention with RoPE. Refer to papers:
- Attention is all you need:
    https://huggingface.co/papers/1706.03762
- RoFormer: Enhanced Transformer with Rotary Position Embedding:
    https://huggingface.co/papers/2104.09864
c                ó    <€ V ^8„  d   QhRS[ /# rE   r   )r0   r1   s   "€r'   r2   ÚMLCDAttention.__annotate__  s   ø€ ÷ @ñ @Ñ/ñ @r)   c                óº  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        V P                  R,          V n        VP                  V n
        RV n        \        P                  ! V P                  V P                  4      V n        \        P                  ! V P                  V P                  4      V n        \        P                  ! V P                  V P                  4      V n        \        P                  ! V P                  V P                  4      V n        VP$                  V n        R# )r¥   FNç      à¿)r   r   r   r    r‘   r_   Ú	num_headsr[   ÚscaleÚattention_dropoutrÎ   Ú	is_causalr   r   Úk_projÚv_projÚq_projÚout_projrÔ   r$   s   &&€r'   r   ÚMLCDAttention.__init__  sÛ   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Õ8ˆŒØ—]‘] DÕ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i’i §¡°·±Ó?ˆŒÜ—i’i §¡°·±Ó?ˆŒÜ—i’i §¡°·±Ó?ˆŒÜŸ	š	 $§.¡.°$·.±.ÓAˆŒØ$*×$?Ñ$?ˆÖ!r)   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# ©r+   r,   Úposition_embeddingsrÌ   Nrb   r-   )r.   r/   rW   r   r   )r0   r1   s   "€r'   r2   rõ     sy   ø€ ÷ -)ñ -)á—|‘|ð-)ñ #¡5§<¡<±·±Ð#=Õ>ð-)ñ Ÿ™ tÕ+ð	-)ñ
 Ñ+Õ,ð-)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ-)r)   c           	     ó¬  € VP                   RR w  rVV P                  V4      P                  WVV P                  V P                  34      pV P                  V4      P                  WVV P                  V P                  34      pV P                  V4      P                  WVV P                  V P                  34      p	V^ ,          P                  ^ 4      P                  4       p
V^,          P                  ^ 4      P                  4       p\        WxW«4      w  rxVP                  ^ ^^^4      P                  4       pVP                  ^ ^^^4      P                  4       pV	P                  ^ ^^^4      P                  4       p	\        P                  ! V P                  P                  \         4      pT! V VVV	V3RV P"                  '       g   RMV P$                  RV P&                  RV P(                  /VB w  rÞVP                  ^^ ^^4      P                  4       pVP+                  WeR4      pV P-                  V4      pVP                  ^^ ^4      P                  4       pWÞ3# )z#Input shape: Batch x Time x ChannelNrÎ   ç        rÍ   rû   r|   )rª   rþ   r°   rø   r[   rü   rý   r­   rX   rñ   r±   rØ   r	   Úget_interfacer   Ú_attn_implementationrÝ   rÒ   rÎ   rù   rû   r´   rÿ   )r%   r,   r  rÌ   rb   rÂ   Ú
seq_lengthÚquery_statesrÙ   rÚ   rp   rq   Úattention_interfacerÜ   rÛ   s   &&&&,          r'   r5   ÚMLCDAttention.forward  s6  € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
ð —{‘{ =Ó1×9Ñ9¸:ÐSW×SaÑSaÐcg×cpÑcpÐ:qÓrˆØ—[‘[ Ó/×7Ñ7¸ÐQU×Q_ÑQ_Ðae×anÑanÐ8oÓpˆ
Ø—{‘{ =Ó1×9Ñ9¸:ÐSW×SaÑSaÐcg×cpÑcpÐ:qÓrˆð " !Õ$×.Ñ.¨qÓ1×7Ñ7Ó9ˆØ! !Õ$×.Ñ.¨qÓ1×7Ñ7Ó9ˆÜ#>¸|ÐY\Ó#bÑ ˆð $×+Ñ+¨A¨q°!°QÓ7×BÑBÓDˆØ×'Ñ'¨¨1¨a°Ó3×>Ñ>Ó@ˆ
Ø#×+Ñ+¨A¨q°!°QÓ7×BÑBÓDˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð
%
ð —J‘Jð
%
ð —n‘nð
%
ð ñ
%
Ñ!ˆð "×)Ñ)¨!¨Q°°1Ó5×@Ñ@ÓBˆØ!×&Ñ& z¸rÓBˆØ—m‘m KÓ0ˆØ!×)Ñ)¨!¨Q°Ó2×=Ñ=Ó?ˆØÐ(Ð(r)   )r   rÎ   r‘   r[   rû   rü   rø   rÔ   rÿ   rþ   rù   rý   r   ©
r8   r9   r:   r;   rƒ   r   r5   r<   r=   r>   r?   s   @@r'   ró   ró   û   s%   ù‡ € ñ÷@ó @÷ -)÷ -)ò -)r)   ró   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMLCDEncoderLayeriC  c                ó    <€ V ^8„  d   QhRS[ /# rE   r   )r0   r1   s   "€r'   r2   ÚMLCDEncoderLayer.__annotate__D  s   ø€ ÷ Sñ SÑ/ñ Sr)   c                óJ  <€ \         SV `  4        VP                  V n        \	        V4      V n        \        P                  ! V P                  VP                  R 7      V n	        \        V4      V n        \        P                  ! V P                  VP                  R 7      V n        R# ©)ÚepsN)r   r   r    r‘   ró   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1r   ÚmlpÚlayer_norm2r$   s   &&€r'   r   ÚMLCDEncoderLayer.__init__D  sl   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ& vÓ.ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÔRˆÔÜ˜6“?ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÔRˆÖr)   c                óà   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[S[,          RS[S[ P
                  ,          /# r  )r.   r/   rW   r   r   r¾   )r0   r1   s   "€r'   r2   r  L  sm   ø€ ÷ "ñ "á—|‘|ð"ñ #¡5§<¡<±·±Ð#=Õ>ð"ñ Ÿ™ tÕ+ð	"ñ
 Ñ+Õ,ð"ñ 
‰u× Ñ Õ	!ñ"r)   c           	     óÈ   € TpV P                  V4      pV P                  ! RRVRVRV/VB w  rWQ,           pTpV P                  V4      pV P                  V4      pWQ,           pV# )ah  
Args:
    hidden_states (`torch.FloatTensor`):
        Input to the layer of shape `(batch, seq_len, embed_dim)`.
        Represents the hidden states from the previous layer or the input embeddings.
    position_embeddings (`tuple[torch.Tensor, torch.Tensor]`):
        A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`.
        Represents absolute positional embeddings for the query and key in the attention mechanism.
    attention_mask (`torch.FloatTensor`):
        Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
r,   r  rÌ   © )r  r  r  r  )r%   r,   r  rÌ   rb   ÚresidualÚ_s   &&&&,  r'   r5   ÚMLCDEncoderLayer.forwardL  s‰   € ð$ !ˆà×(Ñ(¨Ó7ˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð ñ	
Ñˆð !Õ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ Õ0ˆàÐr)   )r‘   r  r  r  r  r   r7   r?   s   @@r'   r  r  C  s    ù‡ € ÷Só S÷"÷ "ò "r)   r  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚMLCDEncoderiq  zŸ
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`MLCDEncoderLayer`].

Args:
    config: MLCDVisionConfig
c                ó    <€ V ^8„  d   QhRS[ /# rE   r   )r0   r1   s   "€r'   r2   ÚMLCDEncoder.__annotate__z  s   ø€ ÷ ,ñ ,Ñ/ñ ,r)   c                óÖ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        RV n	        R# u upi )z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.FN)
r   r   r   r   Ú
ModuleListÚrangeÚnum_hidden_layersr  ÚlayersÚgradient_checkpointing)r%   r   r  r&   s   && €r'   r   ÚMLCDEncoder.__init__z  sS   ø€ ä‰ÑÔØŒÜ—m’mÄuÈV×MeÑMeÔGfÓ$gÑGfÀ!Ô%5°fÖ%=ÑGfÑ$gÓhˆŒØ&+ˆÖ#ùò %hs   ½A&c                óÌ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[S[,          RS[S[,          /# )r+   Úinputs_embedsr  rÌ   Nrb   r-   )r.   r¾   rW   r/   r   r   r   )r0   r1   s   "€r'   r2   r$  �  si   ø€ ÷ !
ñ !
á×(Ñ(ð!
ñ #¡5§<¡<±·±Ð#=Õ>ð!
ñ Ÿ™ tÕ+ð	!
ñ
 Ñ+Õ,ð!
ñ 
‘Õ	 ñ!
r)   c                óZ   € TpV P                    F  pV! VVV3/ VB pK  	  \        VR7      # )aÍ  
Args:
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
        This is useful if you want more control over how to convert `input_ids` indices into associated vectors
        than the model's internal embedding lookup matrix.
    position_embeddings (`tuple[torch.Tensor, torch.Tensor]`):
        A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`.
        Represents absolute positional embeddings for the query and key in the attention mechanism.
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
        [What are attention masks?](../glossary#attention-mask)
)Úlast_hidden_state)r)  r   )r%   r-  r  rÌ   rb   r,   Úencoder_layers   &&&&,  r'   r5   ÚMLCDEncoder.forward�  sF   € ð, &ˆØ!Ÿ[œ[ˆMÙ)ØØ#Øñð ñ	ŠMñ )ô Ø+ô
ð 	
r)   )r   r*  r)  r   r  r?   s   @@r'   r"  r"  q  s#   ù‡ € ñ÷,ó ,÷!
÷ !
ò !
r)   r"  c                   ó”   a a€ ] tR tRt oRtR.tRtRtRtRt	Rt
RtR]R]/t]P                   ! 4       V 3R l4       tV3R	 ltR
tVtV ;t# )ÚMLCDPreTrainedModeli¥  Úvision_modelr  TFr,   Ú
attentionsc                ó  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dú   V P                  P                  p\        P                  ! VP                  RVP                  R,          V,          R7       \        P                  ! VP                  P                  VP                  P                  V,          R7       \        P                  ! VP                  \        P                   ! VP                  P"                  R,          4      P%                  R4      4       R# \	        V\&        4      '       Ed&   V P                  P                  pVP                  R,          ^VP                  P(                  ,          R,          ,          V,          pVP                  R,          V,          p\        P                  ! VP*                  P                  VR7       \        P                  ! VP,                  P                  VR7       \        P                  ! VP.                  P                  VR7       \        P                  ! VP0                  P                  VR7       R# \	        V\2        4      '       dé   V P                  P                  pVP                  P4                  R,          ^VP                  P(                  ,          R,          ,          V,          p^VP                  P4                  ,          R,          V,          p\        P                  ! VP6                  P                  VR7       \        P                  ! VP8                  P                  VR7       R# \	        V\:        4      '       d‚   V P                  P                  pVP                  P4                  VP                  P<                  ,          ^,          R,          V,          p\        P                  ! VP>                  RVR7       R# R# )zInitialize the weightsr  )ÚmeanÚstd)r8  Nr÷   r|   r�   ) r   Ú_init_weightsr   Úinitializer_factorrm   r‡   ÚinitÚnormal_r–   r‘   r™   r¬   Úinitializer_rangeÚcopy_rt   r.   r`   rª   rœ   ró   r(  rþ   rü   rý   rÿ   r   r    r"   r#   ÚMLCDVisionModelr_   Úclass_pos_emb)r%   rÈ   ÚfactorÚin_proj_stdÚout_proj_stdÚfc_stdÚpos_emb_stdr&   s   &&     €r'   r9  Ú!MLCDPreTrainedModel._init_weightsµ  s‘  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fÔ2×3Ò3Ø—[‘[×3Ñ3ˆFÜ�LŠL˜×/Ñ/°c¸v×?OÑ?OÐQUÕ?UÐX^Õ?^Õ_Ü�LŠL˜×/Ñ/×6Ñ6¸F¿M¹M×<[Ñ<[Ð^dÕ<dÕeÜ�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÖhÜ˜¤×.Ó.Ø—[‘[×3Ñ3ˆFØ!×+Ñ+¨TÕ1°q¸6¿=¹=×;ZÑ;ZÕ7ZÐ_cÕ6cÕdÐgmÕmˆKØ"×,Ñ,¨dÕ2°fÕ<ˆLÜ�LŠL˜Ÿ™×-Ñ-°;Õ?Ü�LŠL˜Ÿ™×-Ñ-°;Õ?Ü�LŠL˜Ÿ™×-Ñ-°;Õ?Ü�LŠL˜Ÿ™×/Ñ/°\×BÜ˜¤×(Ò(Ø—[‘[×3Ñ3ˆFØ!Ÿ=™=×4Ñ4°dÕ:ÀÀFÇMÁM×DcÑDcÕ@cÐhlÕ?lÕmÐpvÕvˆKØ˜&Ÿ-™-×3Ñ3Õ3¸Õ<¸vÕEˆFÜ�LŠL˜Ÿ™×*Ñ*°Õ7Ü�LŠL˜Ÿ™×*Ñ*°×<Ü˜¤×0Ò0Ø—[‘[×3Ñ3ˆFØ!Ÿ=™=×4Ñ4¸¿¹×8YÑ8YÕYÐ]^Õ^ÐcgÕgÐjpÕpˆKÜ�LŠL˜×-Ñ-°C¸[×Iñ 1r)   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rE   r   )r0   r1   s   "€r'   r2   Ú MLCDPreTrainedModel.__annotate__¥  s   ø‡ ‚ áÑò r)   r  )r8   r9   r:   r;   Úbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚaccepts_loss_kwargsÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  ró   Ú_can_record_outputsr.   r…   r9  Ú__annotate_func__r<   r=   r>   r?   s   @@r'   r3  r3  ¥  so   ù‡ € ð 'ÐØ+Ð,ÐØ&*Ð#ØÐØÐØ€NØÐØ"&ÐàÐ)Ø�mðÐð
 ‡]‚]ƒ_ôJó ðJ÷# … r)   r3  zN
    The vision model from M_L_C_D without any head or projection on top.
    )Úcustom_introc                   ó˜   a a€ ] tR tRt oRtRtRtV3R lV 3R llt]]	! RR7      ]
RV3R lR	 ll4       4       4       tV3R
 ltRtVtV ;t# )r?  iÓ  r½   r™   c                ó    <€ V ^8„  d   QhRS[ /# rE   r   )r0   r1   s   "€r'   r2   ÚMLCDVisionModel.__annotate__Þ  s   ø€ ÷ 
ñ 
Ñ/ñ 
r)   c                óú  <€ \         SV `  V4       VP                  p\        V4      V n        \
        P                  ! W!P                  R 7      V n        \        V4      V n
        \
        P                  ! W!P                  R 7      V n        \        V4      V n        \
        P                  ! \        P                   ! ^VP                  VP"                  ,          ^,          4      4      V n        V P'                  4        R# r  )r   r   r    r‡   rŸ   r   r  r  Úpre_layrnormr"  ÚencoderÚpost_layernormrA   Úvision_rotary_embeddingr”   r.   r•   r_   r@  Ú	post_init)r%   r   r‘   r&   s   && €r'   r   ÚMLCDVisionModel.__init__Þ  s«   ø€ Ü‰Ñ˜Ô Ø×&Ñ&ˆ	ä.¨vÓ6ˆŒÜŸLšL¨×8MÑ8MÔNˆÔÜ" 6Ó*ˆŒÜ Ÿlšl¨9×:OÑ:OÔPˆÔÜ':¸6Ó'BˆÔ$ÜŸ\š\¬%¯+ª+°a¸×9KÑ9KÈv×OiÑOiÕ9iÐmnÕ9nÓ*oÓpˆÔØ�‰Ör)   F)Útie_last_hidden_statesc                ón   <€ V ^8„  d   QhRS[ P                  R,          RS[S[,          RS[S[,          /# )r+   r½   Nrb   r-   )r.   r¾   r   r   rW   r   )r0   r1   s   "€r'   r2   rV  í  sE   ø€ ÷ @
ñ @
á×'Ñ'¨$Õ.ð@
ñ Ñ+Õ,ð@
ñ 
Ñ+Õ	+ñ	@
r)   c                óN  € Vf   \        R4      hV P                  V4      pV P                  V4      pVP                  R,          V P                  P
                  ,          pVP                  R,          V P                  P
                  ,          p\        P                  ! WAP                  R7      P                  ^4      P                  RV4      p\        P                  ! WQP                  R7      P                  ^ 4      P                  VR4      p\        P                  ! VP                  4       VP                  4       .RR7      pV P                  W84      p	\        P                  ! V P                  V P                  .RR7      p
\        P                  ! V
P!                  4       V	^ ,          .^ R7      \        P                  ! V
P#                  4       V	^,          .^ R7      .p	V P$                  ! R	RVRV	/VB pV^ ,          pVR
,          pV P'                  V4      p\)        VVR7      # )ac  
Example:

```python
>>> import httpx
>>> from io import BytesIO
>>> from PIL import Image
>>> from transformers import AutoProcessor, MLCDVisionModel
>>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448")
>>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448")

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))
>>> inputs = processor(images=image, return_tensors="pt")

>>> with torch.no_grad():
...     outputs = model(**inputs, output_attentions=True)

>>> features = outputs.last_hidden_state
>>> print(f"Extracted features shape: {features.shape}")
>>> print(f"Number of attention layers: {len(outputs.attentions)}")
>>> print(f"Attention shape: {outputs.attentions[0].shape}")
```z You have to specify pixel_values)rB   ry   r-  r  )r/  Úpooler_outputrì   r|   r  )rz   r   rz   )rN   rŸ   rX  rª   r   r“   r.   r`   rB   r­   rœ   ÚstackrÀ   r[  r}   r@  rp   rq   rY  rZ  r   )r%   r½   rb   r,   Únum_patches_heightÚnum_patches_widthÚhpos_idsÚwpos_idsÚpos_idsr  r@  Úencoder_outputsr/  Úpooled_outputs   &&,           r'   r5   ÚMLCDVisionModel.forwardê  sé  € ð@ ÒÜÐ?Ó@Ð@àŸ™¨Ó5ˆØ×)Ñ)¨-Ó8ˆà)×/Ñ/°Õ3°t·{±{×7MÑ7MÕMÐØ(×.Ñ.¨rÕ2°d·k±k×6LÑ6LÕLÐä�LŠLÐ+×4GÑ4GÔH×RÑRÐSTÓU×\Ñ\Ð]_ÐarÓsð 	ô �LŠLÐ*×3FÑ3FÔG×QÑQÐRSÓT×[Ñ[Ð\nÐprÓsð 	ô —+’+˜x×/Ñ/Ó1°8×3CÑ3CÓ3EÐFÈBÔOˆØ"×:Ñ:¸=ÓRÐÜŸ	š	 4×#5Ñ#5°t×7IÑ7IÐ"JÐPRÔSˆä�IŠI�}×(Ñ(Ó*Ð,?ÀÕ,BÐCÈÔKÜ�IŠI�}×(Ñ(Ó*Ð,?ÀÕ,BÐCÈÔKð
Ðð
 Ÿ,š,ñ 
Ø'ð
à 3ð
ð ñ
ˆð ,¨AÕ.ÐØ)¨'Õ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ô
ð 	
r)   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rE   r   )r0   r1   s   "€r'   r2   rV  Ó  s   ø‡ ‚ ñ Ñò r)   )r@  rŸ   rY  rZ  rX  r[  )Úimager   )r8   r9   r:   r;   Úmain_input_nameÚinput_modalitiesÚ_input_embed_layerr   r   r   r   r5   rR  r<   r=   r>   r?   s   @@r'   r?  r?  Ó  sZ   ù‡ € ð %€OØ!ÐØ*Ð÷
ó 
ð  Ù¨EÔ2Ø÷@
ñ @
ó ó 3ó  ð@
÷5 … r)   r?  )r  )/Úcollections.abcr   r.   Útorch.nnr   Ú r   r;  Úactivationsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr	   r
   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_mlcdr   rÏ   r   rA   r‡   rÝ   rá   rÓ   rñ   ró   r  r"  r3  r?  Ú__all__r  r)   r'   Ú<module>r~     s  ðõ( %ã Ý å &Ý !Ý 9ß Kß FÝ &ß BÑ BÝ 0ß GÝ 5Ý 0ôˆb�i‰iô ô=5˜"Ÿ)™)ô =5ô@I˜2Ÿ9™9ô I÷X%ò2(õ	UõôE)�B—I‘Iô E)ôP+Ð1ô +ô\1
�"—)‘)ô 1
ðh ô*J˜/ó *Jó ð*JñZ ðôô
U
Ð)ó U
óð
U
ðp !Ð"3Ð
4‚r)   