+
    œ°³jP ã                   ó¤  € R t ^ RIt^ RIHt ^ RIt^ RIHu Ht ^ RIHt ^RI	H
t ^RIHt ^RIHtHt ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHtHtHt ^RIHtHt ^RI H!t!H"t" ^RI#H$t$ ^RI%H&t&H't'H(t(H)t) ^RI*H+t+ ^RI,H-t-H.t. ^RI/H0t0 ^RI1H2t2H3t3H4t4 ])Pj                  ! ]64      t7R R lt8R R lt9 ! R R]Pt                  4      t; ! R R]Pt                  4      t< ! R R]Pt                  4      t=R R lt>RJR  R! llt? ! R" R#]Pt                  4      t@ ! R$ R%]Pt                  4      tA ! R& R']Pt                  4      tB ! R( R)]Pt                  4      tC ! R* R+]Pt                  4      tDR, tERKR- ltF ! R. R/]Pt                  4      tG ! R0 R1]Pt                  4      tH ! R2 R3]4      tI ! R4 R5]4      tJ ! R6 R7]Pt                  4      tK]' ! R8 R9]"4      4       tL]'! R:R;7       ! R< R=]L4      4       tM]'! R>R;7       ! R? R@]L4      4       tN]'! RAR;7       ! RB RC]L]4      4       tO]'! RDR;7       ! RE RF]L4      4       tP]'! RGR;7       ! RH RI]L]4      4       tQ. RLOtRR# )MzPyTorch Mllama model.N)ÚCallable)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs)ÚMllamaConfigÚMllamaTextConfigÚMllamaVisionConfigc          
      ó¦   € V ^8„  d   QhR\         P                  R\        R\        R\        \         P                  \         P                  3,          /# )é   Úcross_attention_maskÚnum_vision_tokensÚdtypeÚreturn)ÚtorchÚTensorÚintÚstrÚtuple)Úformats   "Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mllama/modeling_mllama.pyÚ__annotate__r,   /   sF   € ÷ ?ñ ?ÜŸ,™,ð?äð?ô ð?ô Œ5�<‰<œŸ™Ð%Õ&ñ	?ó    c                 óþ  € V P                   vr4pV P                  V^R7      p V P                  W4R4      p V P                  ^4      p RV ,
          P	                  V4      pVP                  VP	                  \        P                  4      \        P                  ! V4      P                  4      p \        P                  ! V4      P                  pW8g  P                  RR7      P                  V 4      R,          pW,          p W3# )é   ©Údimç      ð?éÿÿÿÿ).N)ÚshapeÚrepeat_interleaveÚviewÚ	unsqueezeÚtoÚmasked_fillr%   ÚboolÚfinfoÚminÚanyÚtype_as)	r!   r"   r#   Ú
batch_sizeÚtext_total_lengthÚ_Úinverted_cross_attn_maskÚnegative_inf_valueÚfull_text_row_masked_out_masks	   &&&      r+   Ú_prepare_cross_attention_maskrE   /   sô   € ð )=×(BÑ(BÐ%€J AØ/×AÑAÐBSÐYZÐAÓ[ÐØ/×4Ñ4°ZÐTVÓWÐØ/×9Ñ9¸!Ó<Ðð !$Ð&:Õ :×>Ñ>¸uÓEÐØ3×?Ñ?Ø ×#Ñ#¤E§J¡JÓ/´·²¸UÓ1C×1GÑ1GóÐô Ÿš UÓ+×/Ñ/Ðà	Ñ	3×8Ñ8¸RÐ8Ó@×HÑHÐI]Ó^Ð_hÕið "ð Õ9ÐàÐ>Ð>r-   c          
      ó�   € V ^8„  d   QhR\         P                  R\        R\        R\         P                  R\         P                  /# )r    Úaspect_ratio_maskÚnum_patchesÚtarget_lengthr#   r$   )r%   r&   r'   r#   )r*   s   "r+   r,   r,   K   sF   € ÷ ñ Ü—|‘|ðäðô ðô �;‰;ð	ô
 ‡\�\ñr-   c                 ó”  € V P                   w  rEV P                  WE^^4      P                  V4      pVP                  ^^V^4      pW!,
          p^ VRRV) R13&   ^V,
          pVP	                  WEV,          ^4      pWfP                  RR4      ,          \        P                  ! V4      P                  ,          pVP                  ^4      pV# )é   ºNNNNr3   éþÿÿÿ)
r4   r6   r8   ÚrepeatÚreshapeÚ	transposer%   r;   r<   r7   )rG   rH   rI   r#   r?   Úmax_num_tilesÚattention_maskÚpad_patchess   &&&&    r+   Ú$_prepare_aspect_ratio_attention_maskrT   K   sÍ   € ð !2× 7Ñ 7Ñ€JØ&×+Ñ+¨JÀqÈ!ÓL×OÑOÐPUÓV€NØ#×*Ñ*¨1¨a°ÀÓB€Nð  Õ-€KØ*+€N�1�a˜+˜™Ð&Ñ'ð ˜Õ'€Nð $×+Ñ+¨JÈÕ8UÐWXÓY€NØ#×&>Ñ&>¸rÀ2Ó&FÕFÌÏÊÐUZÓI[×I_ÑI_Õ_€NØ#×-Ñ-¨aÓ0€NàÐr-   c                   óT   a a€ ] tR t^ft oRV3R lV 3R llltV3R lR ltRtVtV ;t# )Ú%MllamaPrecomputedAspectRatioEmbeddingc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r    ÚconfigÚis_gated©r   r:   )r*   Ú__classdict__s   "€r+   r,   Ú2MllamaPrecomputedAspectRatioEmbedding.__annotate__g   s   ø€ ÷ 	5ñ 	5Ñ1ð 	5¹Tñ 	5r-   c                ó˜  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        W n        \        P                  ! V P                  ^,           V P                  V P                  ,          4      V n        V'       d2   \        P                  ! \        P                  ! ^4      4      V n        R# R# )rK   N)ÚsuperÚ__init__rQ   Úhidden_sizeÚmax_aspect_ratio_idrZ   r   Ú	EmbeddingÚ	embeddingÚ	Parameterr%   ÚzerosÚgate©ÚselfrY   rZ   Ú	__class__s   &&&€r+   r`   Ú.MllamaPrecomputedAspectRatioEmbedding.__init__g   sŒ   ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ø ŒäŸš d×&>Ñ&>ÀÕ&BÀD×DVÑDVÐY]×YiÑYiÕDiÓjˆŒßÜŸš¤U§[¢[°£^Ó4ˆDŽIñ r-   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©r    Úhidden_stateÚaspect_ratio_idsr$   ©r%   r&   )r*   r\   s   "€r+   r,   r]   r   s1   ø€ ÷ ñ ¡E§L¡Lð ÁEÇLÁLð ÑUZ×UaÑUañ r-   c                óì   € V P                  V4      pVP                  RV P                  ^V P                  4      pV P                  '       d!   W0P
                  P                  4       ,          pW,           pV# )rK   r3   )rd   rO   rQ   ra   rZ   rg   Útanh)ri   rn   ro   Ú
embeddingss   &&& r+   ÚforwardÚ-MllamaPrecomputedAspectRatioEmbedding.forwardr   s]   € Ø—^‘^Ð$4Ó5ˆ
Ø×'Ñ'¨¨D×,>Ñ,>ÀÀ4×CSÑCSÓTˆ
à�=�=ˆ=Ø#§i¡i§n¡nÓ&6Õ6ˆJà#Õ0ˆØÐr-   )rd   rg   ra   rZ   rb   rQ   )T©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r`   rt   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rj   r\   s   @@r+   rV   rV   f   s   ù‡ € ÷	5õ 	5÷÷ ð r-   rV   c                   óP   a a€ ] tR t^}t oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú"MllamaPrecomputedPositionEmbeddingc                ó    <€ V ^8„  d   QhRS[ /# ©r    rY   ©r   )r*   r\   s   "€r+   r,   Ú/MllamaPrecomputedPositionEmbedding.__annotate__~   s   ø€ ÷ 
ñ 
Ñ1ñ 
r-   c                óØ  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  VP
                  ,          ^,          ^,           V n        VP                  V n        VP                  R,          V n        \        P                  ! \        P                  ! ^4      4      V n        \        P                  ! V P                  V P                  4      p\        P                  ! V P                  V,          4      V n        \        P                   ! V P                  ^,           V P                  V P                  ,          V P                  ,          4      V n        R# )r    Nç      à¿)r_   r`   rQ   rb   Ú
image_sizeÚ
patch_sizerH   ra   Úscaler   re   r%   rf   rg   Úrandnrd   rc   Útile_embedding)ri   rY   Úposition_embeddingrj   s   && €r+   r`   Ú+MllamaPrecomputedPositionEmbedding.__init__~   sþ   ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ#)×#=Ñ#=ˆÔ Ø"×-Ñ-°×1BÑ1BÕBÀqÕHÈ1ÕLˆÔØ!×-Ñ-ˆÔØ×'Ñ'¨Õ-ˆŒ
ä—L’L¤§¢¨Q£Ó0ˆŒ	ô #Ÿ[š[¨×)9Ñ)9¸4×;KÑ;KÓLÐÜŸš d§j¡jÐ3EÕ&EÓFˆŒô !ŸlšlØ×$Ñ$ qÕ(¨$×*<Ñ*<¸t×?OÑ?OÕ*OÐRV×RbÑRbÕ*bó
ˆÖr-   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rm   rp   )r*   r\   s   "€r+   r,   r„   ‘   s1   ø€ ÷ ñ ¡E§L¡Lð ÁEÇLÁLð ÑUZ×UaÑUañ r-   c                óÂ  € ^V P                   P                  4       ,
          V P                  ,          pWP                  ^^V P                  V P
                  4      ,           pV P                  V4      pVP                  ^ ,          pVP                  WPP                  V P                  V P
                  4      pV P                   P                  4       V,          pW,           pV# ©rK   )
rg   rr   rd   r6   rH   ra   r‹   r4   rO   rQ   )ri   rn   ro   Úgated_position_embeddingÚtile_position_embeddingr?   Úgated_tile_position_embeddings   &&&    r+   rt   Ú*MllamaPrecomputedPositionEmbedding.forward‘   s¼   € à$%¨¯	©	¯©Ó(8Õ$8¸D¿N¹NÕ#JÐ Ø#×&CÑ&CÀAÀqÈ$×JZÑJZÐ\`×\lÑ\lÓ&mÕmˆð #'×"5Ñ"5Ð6FÓ"GÐØ!×'Ñ'¨Õ*ˆ
Ø"9×"AÑ"AØ×*Ñ*¨D×,<Ñ,<¸d×>NÑ>Nó#
Ðð )-¯	©	¯©Ó(8Ð;RÕ(RÐ%Ø#ÕCˆàÐr-   )rd   rg   ra   rb   rQ   rH   r‰   r‹   rv   r~   s   @@r+   r€   r€   }   s   ù‡ € ÷
ó 
÷&÷ ð r-   r€   c                   óD   a a€ ] tR t^£t oV 3R ltV3R lR ltRtVtV ;t# )ÚMllamaVisionMLPc                ó(  <€ \         SV `  4        Wn        \        VP                  ,          V n        \        P                  ! VP                  VP                  4      V n
        \        P                  ! VP                  VP                  4      V n        R # ©N)r_   r`   rY   r   Ú
hidden_actÚactivation_fnr   ÚLinearra   Úintermediate_sizeÚfc1Úfc2©ri   rY   rj   s   &&€r+   r`   ÚMllamaVisionMLP.__init__¤   sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Õ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr-   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r    Úhidden_statesr$   rp   )r*   r\   s   "€r+   r,   ÚMllamaVisionMLP.__annotate__«   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r-   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r˜   )r�   rš   rž   )ri   r£   s   &&r+   rt   ÚMllamaVisionMLP.forward«   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr-   )rš   rY   r�   rž   rv   r~   s   @@r+   r–   r–   £   s   ù‡ € õK÷÷ ð r-   r–   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r    r£   Ún_repr$   )r%   r&   r'   )r*   s   "r+   r,   r,   ³   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur-   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rL   rL   NrL   rL   )r4   ÚexpandrO   )r£   r¨   ÚbatchÚnum_key_value_headsÚslenÚhead_dims   &&    r+   Ú	repeat_kvr¯   ³   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr-   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r    ÚmoduleÚqueryÚkeyÚvaluerR   NÚscalingÚdropoutÚkwargs)r   ÚModuler%   r&   Úfloatr   r   )r*   s   "r+   r,   r,   À   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r-   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r    )r1   r#   )ÚpÚtrainingr3   )r¯   Únum_key_value_groupsr%   ÚmatmulrP   r   Ú
functionalÚsoftmaxÚfloat32r8   r#   r¶   r¼   Ú
contiguous)r±   r²   r³   r´   rR   rµ   r¶   r·   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r+   Úeager_attention_forwardrÇ   À   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r-   c                   óp   a a€ ] tR t^Ùt oV3R lV 3R llt]! RRRR7      R
V3R lR ll4       tR	tVtV ;t	# )ÚMllamaVisionAttentionc                ó    <€ V ^8„  d   QhRS[ /# r‚   rƒ   )r*   r\   s   "€r+   r,   Ú"MllamaVisionAttention.__annotate__Ú   s   ø€ ÷ \ñ \Ñ1ñ \r-   c                ó  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  VP
                  ,          V n        V P                  R,          V n        ^V n	        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  ,          V P                  RR7      V n        R# )ç      à?F©ÚbiasNr†   )r_   r`   rY   ra   Ú	embed_dimÚattention_headsÚ	num_headsr®   rµ   r½   r   r›   Úq_projÚk_projÚv_projÚo_projrŸ   s   &&€r+   r`   ÚMllamaVisionAttention.__init__Ú   sõ   ø€ Ü‰ÑÔàŒØ×+Ñ+ˆŒØ×/Ñ/ˆŒØ×*Ñ*¨f×.DÑ.DÕDˆŒØ—}‘} dÕ*ˆŒØ$%ˆÔ!ä—i’i §¡°·±ÀÇÁÕ0NÐUZÔ[ˆŒÜ—i’i §¡°·±ÀÇÁÕ0NÐUZÔ[ˆŒÜ—i’i §¡°·±ÀÇÁÕ0NÐUZÔ[ˆŒÜ—i’i §¡°·±Õ >ÀÇÁÐUZÔ[ˆŽr-   rn   r£   úv5.20©Únew_nameÚversionc          	      ó®   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  R,          3,          /# ©r    r£   rR   Nr$   )r%   r&   r)   )r*   r\   s   "€r+   r,   rË   ê   sM   ø€ ÷ #)ñ #)á—|‘|ð#)ñ Ÿ™ tÕ+ð#)ñ
 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ#)r-   c                óî  € V P                  V4      pV P                  V4      pV P                  V4      pVP                  w  rxp	VP                  w  ršp	VP	                  WxV P
                  V P                  4      P                  ^^4      pVP	                  WzV P
                  V P                  4      P                  ^^4      pVP	                  WzV P
                  V P                  4      P                  ^^4      p\        P                  ! V P                  P                  \        4      pV! V VVVV3RRRV P                  /VB w  rÍVP                  WxR4      P                  4       pV P!                  V4      pWÍ3# ©rK   r¶   ç        rµ   r3   )rÓ   rÔ   rÕ   r4   r6   rÒ   r®   rP   r   Úget_interfacerY   Ú_attn_implementationrÇ   rµ   rO   rÂ   rÖ   )ri   r£   rR   r·   r²   r³   r´   r?   Ú	q_seq_lenrA   Ú
kv_seq_lenÚattention_interfacerÆ   rÅ   s   &&&,          r+   rt   ÚMllamaVisionAttention.forwardé   sZ  € ð —‘˜MÓ*ˆØ�k‰k˜-Ó(ˆØ—‘˜MÓ*ˆà#(§;¡;Ñ ˆ
˜qØŸ9™9Ñˆ�qà—
‘
˜:°$·.±.À$Ç-Á-ÓP×ZÑZÐ[\Ð^_Ó`ˆØ�h‰h�z¨t¯~©~¸t¿}¹}ÓM×WÑWÐXYÐ[\Ó]ˆØ—
‘
˜:°4·>±>À4Ç=Á=ÓQ×[Ñ[Ð\]Ð_`Óaˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ñ)¨*ÀÓD×OÑOÓQˆØ—k‘k +Ó.ˆàÐ(Ð(r-   )
rY   rÐ   r®   rÔ   rÒ   r½   rÖ   rÓ   rµ   rÕ   r˜   ©
rw   rx   ry   rz   r`   r   rt   r{   r|   r}   r~   s   @@r+   rÉ   rÉ   Ù   s9   ù‡ € ÷\ó \ñ �^¨oÀwÔO÷#)ñ #)ó P÷#)ð #)r-   rÉ   c                   ót   a a€ ] tR tRt oRV3R lV 3R lllt]! RRRR7      RV3R lR	 ll4       tR
tVtV ;t	# )ÚMllamaVisionEncoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rX   r[   )r*   r\   s   "€r+   r,   Ú%MllamaVisionEncoderLayer.__annotate__  s    ø€ ÷ Fñ FÑ1ð F¹Tñ Fr-   c                óÚ  <€ \         SV `  4        VP                  V n        VP                  V n        W n        VP                  V n        \        V4      V n        \        V4      V n
        \        P                  ! V P                  VP                  R 7      V n        \        P                  ! V P                  VP                  R 7      V n        V'       d™   \        P                   ! \"        P$                  ! ^4      \&        P(                  ,          ^,          4      V n        \        P                   ! \"        P$                  ! ^4      \&        P(                  ,          ^,          4      V n        R# R# )©ÚepsN)r_   r`   ra   rÑ   Únum_attention_headsrZ   rœ   rÉ   Ú	self_attnr–   Úmlpr   Ú	LayerNormÚnorm_epsÚinput_layernormÚpost_attention_layernormre   r%   ÚonesÚmathÚpiÚ	gate_attnÚgate_ffnrh   s   &&&€r+   r`   Ú!MllamaVisionEncoderLayer.__init__  sã   ø€ Ü‰ÑÔà!×-Ñ-ˆÔØ#)×#9Ñ#9ˆÔ Ø ŒØ!'×!9Ñ!9ˆÔä.¨vÓ6ˆŒÜ" 6Ó*ˆŒä!Ÿ|š|¨D×,<Ñ,<À&Ç/Á/ÔRˆÔÜ(*¯ª°T×5EÑ5EÈ6Ï?É?Ô([ˆÔ%çÜŸ\š\¬%¯*ª*°Q«-¼$¿'¹'Õ*AÀAÕ*EÓFˆDŒNÜŸLšL¬¯ª°A«¼¿¹Õ)@À1Õ)DÓEˆDŽMñ r-   rn   r£   rØ   rÙ   c                ó\   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          /# )r    r£   rR   Nrp   )r*   r\   s   "€r+   r,   rë   $  s,   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ñr-   c                óˆ  € TpV P                  V4      pV P                  WR 7      w  rV P                  '       d"   V P                  P	                  4       V,          pW1,           pTpV P                  V4      pV P                  V4      pV P                  '       d"   V P                  P	                  4       V,          pW1,           pV# )©rR   )rô   rð   rZ   rù   rr   rõ   rñ   rú   )ri   r£   rR   ÚresidualrÅ   s   &&&  r+   rt   Ú MllamaVisionEncoderLayer.forward#  s§   € ð !ˆØ×,Ñ,¨]Ó;ˆØ&*§n¡n°] nÓ&bÑ#ˆØ�=�=ˆ=Ø ŸN™N×/Ñ/Ó1°MÕAˆMØ Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ�=�=ˆ=Ø ŸM™M×.Ñ.Ó0°=Õ@ˆMØ Õ0ˆàÐr-   )
rù   rú   ra   rô   rœ   rZ   rñ   rï   rõ   rð   )Fr˜   rç   r~   s   @@r+   ré   ré     s9   ù‡ € ÷Fõ Fñ$ �^¨oÀwÔO÷ñ ó P÷ð r-   ré   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚMllamaVisionEncoderi<  z�
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`MllamaEncoderLayer`].

Args:
    config: MllamaConfig
c                ó    <€ V ^8„  d   QhRS[ /# r‚   rƒ   )r*   r\   s   "€r+   r,   Ú MllamaVisionEncoder.__annotate__E  s   ø€ ÷ ñ Ñ1ñ r-   c           	     óÎ   <€ \         SV `  4        Wn        \        P                  ! \        V4       Uu. uF  p\        W4      NK  	  up4      V n        R V n        Wn        R# u upi )FN)	r_   r`   rY   r   Ú
ModuleListÚrangeré   ÚlayersÚgradient_checkpointing)ri   rY   Ú
num_layersrZ   rA   rj   s   &&&& €r+   r`   ÚMllamaVisionEncoder.__init__E  sU   ø€ Ü‰ÑÔØŒÜ—m’mÔY^Ð_iÔYjÓ$kÑYjÐTUÔ%=¸fÖ%OÑYjÑ$kÓlˆŒØ&+ˆÔ#ØŽùò %ls   ³A"c                ób   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[/# rÝ   ©r%   r&   r   )r*   r\   s   "€r+   r,   r  L  s;   ø€ ÷ ^ñ ^á—|‘|ð^ñ Ÿ™ tÕ+ð^ñ 
ñ	^r-   c                ój   € RpV P                    F  pV! VVR7      pW13,           pK  	  \        WR7      # )aà  
Args:
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
        This is useful if you want more control over how to convert `input_ids` indices into associated vectors
        than the model's internal embedding lookup matrix.
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)

)rn   rR   )Úlast_hidden_stater£   © )r  r   )ri   r£   rR   Úencoder_statesÚencoder_layers   &&&  r+   rt   ÚMllamaVisionEncoder.forwardL  sA   € ð( ˆØ!Ÿ[œ[ˆMÙ)Ø*Ø-ôˆMð ,Ð.>Õ>ŠNñ )ô °Ô]Ð]r-   )rY   r	  r  )é    Fr˜   ©
rw   rx   ry   rz   Ú__doc__r`   rt   r{   r|   r}   r~   s   @@r+   r  r  <  s&   ù‡ € ñ÷õ ÷^÷ ^ò ^r-   r  c                   óZ   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )	ÚMllamaTextRMSNormil  c                ó$   <€ V ^8„  d   QhRS[ RR/# )r    rî   r$   N)r¹   )r*   r\   s   "€r+   r,   ÚMllamaTextRMSNorm.__annotate__m  s   ø€ ÷ $ñ $©ð $¸$ñ $r-   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z0
MllamaTextRMSNorm is equivalent to T5LayerNorm
N)r_   r`   r   re   r%   rö   ÚweightÚvariance_epsilon)ri   ra   rî   rj   s   &&&€r+   r`   ÚMllamaTextRMSNorm.__init__m  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör-   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r¢   rp   )r*   r\   s   "€r+   r,   r  u  s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r-   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r    T)Úkeepdimr3   )	r#   r8   r%   rÁ   ÚpowÚmeanÚrsqrtr  r  )ri   r£   Úinput_dtypeÚvariances   &&  r+   rt   ÚMllamaTextRMSNorm.forwardu  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r-   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)r)   r  r4   r  ©ri   s   &r+   Ú
extra_reprÚMllamaTextRMSNorm.extra_repr|  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr-   )r  r  )g�íµ ÷Æ°>)
rw   rx   ry   rz   r`   rt   r*  r{   r|   r}   r~   s   @@r+   r  r  l  s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr-   r  c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚMllamaTextCrossAttentioni€  z=Multi-headed attention from 'Attention Is All You Need' paperc                óB   <€ V ^8„  d   QhRS[ R,          RS[R,          /# )r    rY   NÚ	layer_idx©r   r'   )r*   r\   s   "€r+   r,   Ú%MllamaTextCrossAttention.__annotate__ƒ  s,   ø€ ÷ Pñ Pá  4Õ'ðPñ ˜•:ñPr-   c                óT  <€ \         SV `  4        Wn        V P                  P                  V n        V P                  P
                  V n        VP                  V n        VP                  V n        VP                  V P                  ,          V n        W n	        V P                  V P
                  ,          V n
        V P                  R,          V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P
                  V P                  ,          RR7      V n        \        P                  ! V P                  V P
                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  ,          V P                  RR7      V n        \%        V P                  VP&                  R7      V n        \%        V P                  VP&                  R7      V n        R# )rÍ   FrÎ   rí   Nr†   )r_   r`   rY   rï   rÒ   r¬   r¶   ra   r®   r/  r½   rµ   r   r›   rÓ   rÔ   rÕ   rÖ   r  Úrms_norm_epsÚq_normÚk_norm©ri   rY   r/  rj   s   &&&€r+   r`   Ú!MllamaTextCrossAttention.__init__ƒ  sg  ø€ ô
 	‰ÑÔØŒØŸ™×8Ñ8ˆŒØ#'§;¡;×#BÑ#BˆÔ Ø—~‘~ˆŒØ!×-Ñ-ˆÔØ×*Ñ*¨d¯n©nÕ<ˆŒØ"ŒØ$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø—}‘} dÕ*ˆŒä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i §¡°·±Õ >À×@PÑ@PÐW\Ô]ˆŒä'¨¯©¸6×;NÑ;NÔOˆŒÜ'¨¯©¸6×;NÑ;NÔOˆŽr-   c                ó4  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r    r£   Úcross_attention_statesNÚpast_key_valuesrR   Ú	use_cacher$   )r%   r&   r   r:   r)   )r*   r\   s   "€r+   r,   r1  ›  sŒ   ø€ ÷ 6)ñ 6)á—|‘|ð6)ñ !&§¡¨tÕ 3ð6)ñ  �ð	6)ñ
 Ÿ™ tÕ+ð6)ñ ˜$•;ð6)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ6)r-   c                óŽ  € VP                  4       w  rxp	V P                  V4      p
V
P                  WxV P                  V P                  4      P                  ^^4      p
V P                  V
4      p
VeÇ   V P                  V4      pV P                  V4      pVP                  VRV P                  V P                  4      P                  ^^4      pVP                  VRV P                  V P                  4      P                  ^^4      pV P                  V4      pVe   VP                  W¼V P                  4      w  r¼MrVed   VP                  4       ^ 8”  dO   VP                  V P                  ,          P                  VP                  V P                  ,          P                   rËM\#        R4      h\$        P&                  ! V P(                  P*                  \,        4      pT! V V
VVV3RV P.                  '       g   RMV P0                  RV P2                  /VB w  rïVP5                  WxR4      P7                  4       pV P9                  V4      pWï3# )z#Input shape: Batch x Time x Channelz^Cross attention layer can't find neither `cross_attn_states` nor cached values for key/values!r¶   rà   rµ   r3   )ÚsizerÓ   r6   rÒ   r®   rP   r4  rÔ   rÕ   r¬   r5  Úupdater/  Úget_seq_lengthr  ÚkeysÚvaluesÚ
ValueErrorr   rá   rY   râ   rÇ   r¼   r¶   rµ   rO   rÂ   rÖ   )ri   r£   r9  r:  rR   r;  r·   ÚbszÚq_lenrA   Úquery_statesrÃ   rÄ   rå   rÆ   rÅ   s   &&&&&&,         r+   rt   Ú MllamaTextCrossAttention.forward›  s  € ð &×*Ñ*Ó,‰ˆ�AØ—{‘{ =Ó1ˆØ#×(Ñ(¨°T·^±^ÀTÇ]Á]ÓS×]Ñ]Ð^_ÐabÓcˆØ—{‘{ <Ó0ˆà!Ò-ØŸ™Ð%;Ó<ˆJØŸ;™;Ð'=Ó>ˆLØ#Ÿ™¨¨b°$×2JÑ2JÈDÏMÉMÓZ×dÑdÐefÐhiÓjˆJØ'×,Ñ,¨S°"°d×6NÑ6NÐPT×P]ÑP]Ó^×hÑhÐijÐlmÓnˆLàŸ™ ZÓ0ˆJØÒ*ð ,;×+AÑ+AÀ*Ð\`×\jÑ\jÓ+kÑ(�
øØÒ(¨_×-KÑ-KÓ-MÐPQÔ-Qà×&Ñ& t§~¡~Õ6×;Ñ;Ø×&Ñ& t§~¡~Õ6×=Ñ=ñ %ô
 Øpóð ô )@×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ñ)¨#°bÓ9×DÑDÓFˆØ—k‘k +Ó.ˆàÐ(Ð(r-   )rY   r¶   r®   ra   r5  rÔ   r/  rÒ   r½   r¬   rÖ   r4  rÓ   rµ   rÕ   )NN)NNNNr  r~   s   @@r+   r-  r-  €  s#   ù‡ € ÙG÷Põ P÷06)÷ 6)ò 6)r-   r-  c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr0   r3   )r4   r%   Úcat)ÚxÚx1Úx2s   &  r+   Úrotate_halfrL  Õ  sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r-   c                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)r7   rL  )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r+   Úapply_rotary_pos_embrU  Ý  sS   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr-   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMllamaTextSelfAttentioniö  c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r    rY   r/  r0  )r*   r\   s   "€r+   r,   Ú$MllamaTextSelfAttention.__annotate__÷  s    ø€ ÷ ^ñ ^Ñ/ð ^¹Cñ ^r-   c                ó¢  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V P                  ,          V n        V P                  V P                  ,          V n	        V P                  R,          V n
        W n        RV n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  ,          V P                  RR7      V n        R# )rÍ   TFrÎ   Nr†   )r_   r`   rY   rï   rÒ   r¶   ra   r¬   r®   r½   rµ   r/  Ú	is_causalr   r›   rÓ   rÔ   rÕ   rÖ   r6  s   &&&€r+   r`   Ú MllamaTextSelfAttention.__init__÷  s4  ø€ Ü‰ÑÔØŒØ×3Ñ3ˆŒØ—~‘~ˆŒØ!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ø×*Ñ*¨d¯n©nÕ<ˆŒØ$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø—}‘} dÕ*ˆŒà"ŒØˆŒä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i §¡°·±Õ >À×@PÑ@PÐW\Ô]ˆŽr-   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r    r£   rR   Úposition_embeddingsrp   )r*   r\   s   "€r+   r,   rZ  
  s5   ø€ ÷ *)ñ *)á—|‘|ð*)ñ Ÿ™ð*)ñ #Ÿ\™\ñ	*)r-   c                ó€  € VP                  4       w  rgpV P                  V4      p	V P                  V4      p
V P                  V4      pV	P	                  WgV P
                  V P                  4      P                  ^^4      p	V
P	                  WgV P                  V P                  4      P                  ^^4      p
VP	                  WgV P                  V P                  4      P                  ^^4      pVw  rÍ\        WšWÍ4      w  ršVe   VP                  W«V P                  4      w  r«\        P                  ! V P                  P                  \         4      pT! V V	V
VV3RV P"                  '       g   RMV P$                  RV P&                  /VB w  ppVP)                  WgR4      P+                  4       pV P-                  V4      pVV3# rß   )r=  rÓ   rÔ   rÕ   r6   rÒ   r®   rP   r¬   rU  r>  r/  r   rá   rY   râ   rÇ   r¼   r¶   rµ   rO   rÂ   rÖ   )ri   r£   rR   r_  r:  r·   rC  rD  rA   rE  rÃ   rÄ   rP  rQ  rå   rÆ   rÅ   s   &&&&&,           r+   rt   ÚMllamaTextSelfAttention.forward
  s£  € ð &×*Ñ*Ó,‰ˆ�Aà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨°T·^±^ÀTÇ]Á]ÓS×]Ñ]Ð^_ÐabÓcˆØ—_‘_ S°×1IÑ1IÈ4Ï=É=ÓY×cÑcÐdeÐghÓiˆ
Ø#×(Ñ(¨°T×5MÑ5MÈtÏ}É}Ó]×gÑgÐhiÐklÓmˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ñ)¨#°bÓ9×DÑDÓFˆØ—k‘k +Ó.ˆà˜LÐ(Ð(r-   )rY   r¶   r®   ra   r\  rÔ   r/  rÒ   r½   r¬   rÖ   rÓ   rµ   rÕ   r˜   rv   r~   s   @@r+   rW  rW  ö  s    ù‡ € ÷^ó ^÷&*)÷ *)ò *)r-   rW  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚMllamaTextMLPi8  c                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©FrÎ   N)r_   r`   rY   ra   rœ   r   r›   Ú	gate_projÚup_projÚ	down_projr   r™   Úact_fnrŸ   s   &&€r+   r`   ÚMllamaTextMLP.__init__9  s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒä˜V×.Ñ.Õ/ˆŽr-   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r˜   )rh  ri  rf  rg  )ri   rI  rh  s   && r+   rt   ÚMllamaTextMLP.forwardD  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr-   )ri  rY   rh  rf  ra   rœ   rg  rv   r~   s   @@r+   rc  rc  8  s   ù‡ € õ	0÷ò r-   rc  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMllamaSelfAttentionDecoderLayeriJ  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rY  r0  )r*   r\   s   "€r+   r,   Ú,MllamaSelfAttentionDecoderLayer.__annotate__K  s   ø€ ÷ 
#ñ 
#Ñ/ð 
#¹Cñ 
#r-   c                ó,  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        W n        R# ))rY   r/  rí   N)r_   r`   ra   rW  rð   rc  rñ   r  r3  rô   rõ   r/  r6  s   &&&€r+   r`   Ú(MllamaSelfAttentionDecoderLayer.__init__K  sp   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä0¸ÔTˆŒä  Ó(ˆŒÜ0°×1CÑ1CÈ×I\ÑI\Ô]ˆÔÜ(9¸&×:LÑ:LÐRX×ReÑReÔ(fˆÔ%à"Žr-   c                ó2  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  S[ P                  3,          R,          RS[S[,          RS[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )r    r£   r9  Nr!   rR   rD   Úposition_idsr:  r;  r_  r·   r$   )	r%   r&   r)   Ú
LongTensorr   r:   r   r
   ÚFloatTensor)r*   r\   s   "€r+   r,   rp  W  s  ø€ ÷ 5ñ 5á—|‘|ð5ñ !&§¡¨tÕ 3ð5ñ $Ÿl™l¨TÕ1ð	5ñ
 Ÿ™ tÕ+ð5ñ (-©U¯\©\¹5¿<¹<Ð-GÕ'HÈ4Õ'Oð5ñ ×&Ñ&¨Õ-ð5ñ  �ð5ñ ˜$•;ð5ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð5ñ Ñ-Õ.ð5ñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñ5r-   c
                óÔ   € TpV P                  V4      pV P                  ! RRVRVRVRVRVRV	/V
B w  rW±,           pTpV P                  V4      pV P                  V4      pW±,           pV# )a  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`, *optional*):
        attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
        query_sequence_length, key_sequence_length)` if default attention is used.

    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    past_key_values (`Cache`, *optional*): cached past key and value projection states
    position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
    kwargs (`dict`, *optional*):
        Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
        into the model
r£   rR   rt  r:  r;  r_  r  )rô   rð   rõ   rñ   )ri   r£   r9  r!   rR   rD   rt  r:  r;  r_  r·   rÿ   Úself_attn_weightss   &&&&&&&&&&,  r+   rt   Ú'MllamaSelfAttentionDecoderLayer.forwardW  s«   € ð> !ˆà×,Ñ,¨]Ó;ˆð ,0¯>ª>ñ ,
Ø'ð,
à)ð,
ð &ð,
ð ,ð	,
ð
  ð,
ð !4ð,
ð ñ,
Ñ(ˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆàÐr-   )ra   rô   r/  rñ   rõ   rð   )NNNNNNFNrv   r~   s   @@r+   rn  rn  J  s   ù‡ € ÷
#ó 
#÷5÷ 5ò 5r-   rn  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	Ú MllamaCrossAttentionDecoderLayeri�  zLCross-attention transformer block with tanh-gated attention and feedforward.c                ó*   <€ V ^8„  d   QhRS[ RS[RR/# )r    rY   r/  r$   Nr0  )r*   r\   s   "€r+   r,   Ú-MllamaCrossAttentionDecoderLayer.__annotate__’  s(   ø€ ÷ 
Fñ 
FÑ/ð 
F¹Cð 
FÀDñ 
Fr-   c                óê  <€ \         SV `  4        W n        \        WR 7      V n        \        VP                  VP                  R7      V n        \        P                  P                  \        P                  ! ^4      4      V n        \        V4      V n        \        VP                  VP                  R7      V n        \        P                  P                  \        P                  ! ^4      4      V n        R# ))r/  rí   N)r_   r`   r/  r-  Ú
cross_attnr  ra   r3  rô   r%   r   re   rf   Úcross_attn_attn_gaterc  rñ   rõ   Úcross_attn_mlp_gater6  s   &&&€r+   r`   Ú)MllamaCrossAttentionDecoderLayer.__init__’  s    ø€ Ü‰ÑÔØ"ŒÜ2°6ÔOˆŒä0°×1CÑ1CÈ×I\ÑI\Ô]ˆÔÜ$)§H¡H×$6Ñ$6´u·{²{À1³~Ó$FˆÔ!ä  Ó(ˆŒÜ(9¸&×:LÑ:LÐRX×ReÑReÔ(fˆÔ%Ü#(§8¡8×#5Ñ#5´e·k²kÀ!³nÓ#EˆÖ r-   c                ó~  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          R	S[R,          R
S[ P                  R,          RS[S[,          RS[S[ P                  ,          /# )r    r£   r9  r!   rR   rD   rt  Nr:  r;  r_  r·   r$   )r%   r&   r)   ru  r   r:   r   r
   )r*   r\   s   "€r+   r,   r}  ž  sÆ   ø€ ÷  ñ  á—|‘|ð ñ !&§¡ð ñ $Ÿl™lð	 ñ
 Ÿ™ð ñ (-©U¯\©\¹5¿<¹<Ð-GÕ'Hð ñ ×&Ñ&¨Õ-ð ñ  �ð ñ ˜$•;ð ñ #Ÿ\™\¨DÕ0ð ñ Ñ-Õ.ð ñ 
‰u�|‰|Õ	ñ r-   c
                óp  € TpV P                  V4      pV P                  ! RR VRVRVRV/V
B w  rW°P                  P                  4       V,          ,           pTpV P	                  V4      pV P                  V4      pVe   VR,          V,          pW°P                  P                  4       V,          ,           pV# )r£   rR   r9  r:  r  )rL   é    )rô   r  r€  rr   rõ   rñ   r�  )ri   r£   r9  r!   rR   rD   rt  r:  r;  r_  r·   rÿ   rÅ   s   &&&&&&&&&&,  r+   rt   Ú(MllamaCrossAttentionDecoderLayer.forwardž  sÐ   € ð !ˆØ×,Ñ,¨]Ó;ˆà&*§o¢oñ '
Ø'ð'
à/ð'
ð $:ð'
ð ,ð	'
ð
 ñ'
Ñ#ˆð !×#<Ñ#<×#AÑ#AÓ#CÀmÕ#SÕSˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ(Ò4Ø9¸$Õ?À-ÕOˆMØ ×#;Ñ#;×#@Ñ#@Ó#BÀ]Õ#RÕRˆàÐr-   )r  r€  r�  rô   r/  rñ   rõ   )NNFNr  r~   s   @@r+   r{  r{  �  s#   ù‡ € ÙV÷
Fó 
F÷ ÷  ò  r-   r{  c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚMllamaRotaryEmbeddingiÂ  Údevicez5.18)rÛ   c                ó    <€ V ^8„  d   QhRS[ /# r‚   ©r   )r*   r\   s   "€r+   r,   Ú"MllamaRotaryEmbedding.__annotate__Ä  s   ø€ ÷ Oñ OÑ/ñ Or-   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r_   r`   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrY   Úrope_parametersrŽ  Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)ri   rY   r‰  Úrope_init_fnr˜  rj   s   &&&  €r+   r`   ÚMllamaRotaryEmbedding.__init__Ã  s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr-   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r    rY   r$   )r   r)   r%   r&   r¹   )r*   r\   s   "€r+   r,   rŒ  Ö  s/   ø€ ÷ 5ñ 5Ñ0@ð 5Ñ\aÑbg×bnÑbnÑpuÐbuÕ\vñ 5r-   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetar®   Nr2   ©r#   )r”  Úgetattrra   rï   r%   Úaranger¹   r8   )rY   r‰  r·   Úbaser1   Úattention_factorr˜  s   &&,    r+   r•  Ú5MllamaRotaryEmbedding.compute_default_rope_parametersÔ  s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r-   c                ób  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      pVR	,          P                  4       p\	        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabledr0   r   )NrL   Nr3   )rL   NrL   )r˜  r¹   rª   r4   Ú
isinstancer‰  Útyper(   r   rP   r%   rH  rP  r–  rQ  r8   r#   )
ri   rI  rt  Úinv_freq_expandedÚposition_ids_expandedr©  ÚfreqsÚembrP  rQ  s
   &&&       r+   rt   ÚMllamaRotaryEmbedding.forwardé  s3  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓeÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   ÃBFÆF.	)r–  rY   r˜  r’  rš  r“  rŽ  r˜   )rw   rx   ry   rz   r   r`   Ústaticmethodr•  r%   Úno_gradr   rt   r{   r|   r}   r~   s   @@r+   rˆ  rˆ  Â  su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð& ‡]‚]ƒ_Øñ<ó ó ÷<ð <r-   rˆ  c                   ó¢   a a€ ] tR tRt oRtR
tRt. ROtRtRt	Rt
RtRtR]].R]]./t]P&                  ! 4       V 3R l4       tV3R ltR	tVtV ;t# )ÚMllamaPreTrainedModeliù  ÚmodelTFr£   Ú
attentionsc                óä  <€ \         SV `  V4       \        V P                  R V P                  P	                  4       P
                  4      p\        V\        4      '       d%   \        P                  ! VP                  VR7       R# \        V\        4      '       dE   \        P                  ! VP                  VR7       \        P                  ! VP                  4       R# \        V\        4      '       dY   VP                   '       dG   \        P                  ! VP"                  VR7       \        P                  ! VP$                  VR7       R# \        V\&        4      '       dC   \        P                  ! VP(                  4       \        P                  ! VP*                  4       R# \        V\,        4      '       d7   VP                   '       d#   \        P                  ! VP                  4       R# R# R# )Úinitializer_range)ÚstdN)r_   Ú_init_weightsr¡  rY   Úget_text_configr¹  r«  ÚMllamaVisionModelÚinitÚnormal_Úclass_embeddingr€   rd   Úzeros_rg   ré   rZ   rù   rú   r{  r€  r�  rV   )ri   r±   rº  rj   s   && €r+   r»  Ú#MllamaPreTrainedModel._init_weights  s*  ø€ ä‰Ñ˜fÔ%Ü�d—k‘kÐ#6¸¿¹×8SÑ8SÓ8U×8gÑ8gÓhˆä�fÔ/×0Ò0Ü�LŠL˜×/Ñ/°S×9Ü˜Ô B×CÒCÜ�LŠL˜×)Ñ)¨sÕ3Ü�KŠK˜Ÿ™Ö$Ü˜Ô 8×9Ò9¸f¿o¿o¸oÜ�LŠL˜×)Ñ)¨sÕ3Ü�LŠL˜Ÿ™¨c×2Ü˜Ô @×AÒAÜ�KŠK˜×3Ñ3Ô4Ü�KŠK˜×2Ñ2Ö3Ü˜Ô E×FÒFØ��ˆÜ—’˜FŸK™KÖ(ñ ñ Gr-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r‚   ©r   )r*   r\   s   "€r+   r,   Ú"MllamaPreTrainedModel.__annotate__ù  s   ø‡ ‚ áÑò r-   r  )ÚimageÚtext)ré   r{  rn  )rw   rx   ry   rz   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_compile_fullgraphÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendrn  r{  rW  r-  Ú_can_record_outputsr%   r³  r»  Ú__annotate_func__r{   r|   r}   r~   s   @@r+   rµ  rµ  ù  sƒ   ù‡ € ð  ÐØ(ÐØ&*Ð#òÐð
 #ÐØ€NØÐØÐØ"&ÐàÐ9Ð;[Ð\ð
 	Ð.Ð0HÐIðÐð ‡]‚]ƒ_ô)ó ð)÷5 … r-   rµ  zH
    The Mllama Vision Model which consists of two vision encoders.
    )Úcustom_introc                   óš   a a€ ] tR tRt oRtRtV3R lV 3R lltR tV3R lR lt]	]
]V3R lR	 l4       4       4       tV3R
 ltRtVtV ;t# )r½  i'  Úvision_modelc                ó    <€ V ^8„  d   QhRS[ /# r‚   rƒ   )r*   r\   s   "€r+   r,   ÚMllamaVisionModel.__annotate__1  s   ø€ ÷ #ñ #Ñ1ñ #r-   c           	     ó4  <€ \         SV `  V4       VP                  V n        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        V P                  V P                  ,          ^,          ^,           V n        VP
                  R,          V n	        \        P                  ! VP                  V P
                  V P                  V P                  RRR7      V n        \        P                  ! V P                  \        P                  ! V P
                  4      ,          4      V n        \#        V4      V n        \'        VRR7      V n        \'        VRR7      V n        \        P,                  ! V P
                  4      V n        \        P,                  ! V P
                  4      V n        \3        WP4                  RR7      V n        \3        WP8                  RR7      V n        V P=                  4        R# )r    ÚvalidF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingrÏ   T)rZ   Nr†   )r_   r`   r‡   rˆ   rQ   ra   Únum_channelsÚintermediate_layers_indicesrH   r‰   r   ÚConv2dÚpatch_embeddingre   r%   rŠ   rÀ  r€   Úgated_positional_embeddingrV   Úpre_tile_positional_embeddingÚpost_tile_positional_embeddingrò   Úlayernorm_preÚlayernorm_postr  Únum_hidden_layersÚtransformerÚnum_global_layersÚglobal_transformerÚ	post_initrŸ   s   &&€r+   r`   ÚMllamaVisionModel.__init__1  sŒ  ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ#×1Ñ1ˆÔØ!×-Ñ-ˆÔØ"×/Ñ/ˆÔØ+1×+MÑ+MˆÔ(à ŸO™O¨t¯©Õ>À1ÕDÀqÕHˆÔØ×'Ñ'¨Õ-ˆŒ
ä!ŸyšyØ×+Ñ+Ø×)Ñ)ØŸ™Ø—?‘?ØØô 
ˆÔô  "Ÿ|š|¨D¯J©J¼¿ºÀT×EUÑEUÓ9VÕ,VÓWˆÔÜ*LÈVÓ*TˆÔ'ä-RÐSYÐdhÔ-iˆÔ*Ü.SÐTZÐeiÔ.jˆÔ+ô  Ÿ\š\¨$×*:Ñ*:Ó;ˆÔÜ Ÿlšl¨4×+;Ñ+;Ó<ˆÔô /¨v×7OÑ7OÐZ_Ô`ˆÔÜ"5°f×>VÑ>VÐaeÔ"fˆÔà�‰Ör-   c                ó   € V P                   # )zW
This function is used to fetch the first embedding layer to activate grads on inputs.
)râ  r)  s   &r+   Úget_input_embeddingsÚ&MllamaVisionModel.get_input_embeddingsV  s   € ð ×#Ñ#Ð#r-   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r    rn   r$   rp   )r*   r\   s   "€r+   r,   r×  \  s#   ø€ ÷ ñ ±%·,±,ð Á5Ç<Á<ñ r-   c                ó�   € VP                   w  r#pV P                  P                  V^V4      p\        P                  ! WQ.^R7      pV# )rK   r0   )r4   rÀ  rª   r%   rH  )ri   rn   r?   rA   ra   rÀ  s   &&    r+   Úapply_class_embeddingÚ'MllamaVisionModel.apply_class_embedding\  sE   € Ø%1×%7Ñ%7Ñ"ˆ
�{Ø×.Ñ.×5Ñ5°jÀ!À[ÓQˆÜ—y’y /Ð!@ÀaÔHˆØÐr-   c                ón   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[/# )r    Úpixel_valuesro   rG   r$   r  )r*   r\   s   "€r+   r,   r×  e  sA   ø€ ÷ H?ñ H?Ù!ŸL™LðH?Ù<A¿L¹LðH?Ù]b×]iÑ]iðH?á	ñH?r-   c                ó¢  € VP                   w  rVrxršVP                  WV,          V,          W‰V
4      pVP                  WV,          R	4      pV P                  P                  P                  pV P                  P                  P
                  pV P                  VP                  WË4      4      pVP                  ^4      P                  ^^4      pVP                   w  pppVP                  WV,          VR	V4      pV P                  Wâ4      pVP                  WV,          V,          VV4      pV P                  V4      pV^,          pVP                  WV,          VVV4      pV P                  Wâ4      pV P                  V4      p^VP                   R
,          ^,          ,
          ^,          p^ ^ ^ V3p\        P                  ! VVR^ R7      pV^ 8”  d   V) MRpVP                  WV,          R	4      p\        VV P                   VP                   ^,          V P                  R7      pVP#                  WV,          R	V4      pV P%                  VVR7      pVP&                  pV P)                  V4      pVP                  WV,          VVV,           V4      pV P+                  Wâ4      pVP                  WV,          VVV,           ,          V4      pV P-                  VVR7      pVP&                  pVP                  WV,          VVV,           V4      pVRRRV13,          pVP                  WVVVV4      pV P.                   Uu. uF  pVP0                  V,          NK  	  pp\2        P4                  ! VR	R7      pVP                  WV,          VVV,           R	4      pVRRRV13,          pVP                  WVVVR	4      p\2        P6                  ! VV.R	R7      p\9        VR7      # u upi )aŽ  
aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
    Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
    These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

    For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
    - An image with aspect ratio [1, 1] would have ID 1
    - An image with aspect ratio [1, 2] would have ID 2
    - An image with aspect ratio [2, 1] would have ID 3

    The id 0 is reserved for padding (i.e., no image).

    If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
    Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

    - 1 for tiles that are **not masked**,
    - 0 for tiles that are **masked**.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, MllamaVisionModel

>>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
>>> model = MllamaVisionModel.from_pretrained(checkpoint)
>>> processor = AutoProcessor.from_pretrained(checkpoint)

>>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))
>>> inputs = processor(images=image, return_tensors="pt")

>>> output = model(**inputs)

>>> print(output.last_hidden_state.shape)
torch.Size([1, 1, 4, 1025, 7680])
```
Úconstant)Úmoder´   N)rG   rH   rI   r#   rþ   rL   r0   )r  r3   rM   )r4   rO   râ  r  r#   r‰  r8   ÚflattenrP   rä  ró  rã  ræ  ÚFÚpadrT   rH   r6   ré  r  rç  rå  rë  rà  r£   r%   ÚstackrH  r   )ri   rö  ro   rG   r·   r?   Únum_concurrent_mediaÚ	num_tilesrß  ÚheightÚwidthÚtarget_dtypeÚtarget_deviceÚpatch_embedsrn   rA   rH   r1   Únum_padding_patchesrÞ  Úslice_indexrR   ÚoutputÚglobal_outputÚiÚall_intermediate_hidden_statesÚintermediate_hidden_statess   &&&&,                      r+   rt   ÚMllamaVisionModel.forwardb  sù  € ð` T`×SeÑSeÑPˆ
¨)À6à#×+Ñ+¨JÕ,MÐPYÕ,YÐ[gÐqvÓwˆØ+×3Ñ3°JÕ4UÐWYÓZÐð ×+Ñ+×2Ñ2×8Ñ8ˆØ×,Ñ,×3Ñ3×:Ñ:ˆØ×+Ñ+¨L¯O©O¸MÓ,XÓYˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆð +×0Ñ0Ñˆˆ;˜Ø#×+Ñ+¨JÕ,MÈyÐZ\Ð^aÓbˆØ×9Ñ9¸,ÓYˆð $×+Ñ+¨JÕ,MÐPYÕ,YÐ[fÐhkÓlˆØ×1Ñ1°,Ó?ˆØ�qÕˆð $×+Ñ+¨JÕ,MÈyÐZeÐgjÓkˆØ×6Ñ6°|ÓVˆà×)Ñ)¨,Ó7ˆð  ! L×$6Ñ$6°rÕ$:¸QÕ$>Õ?À1ÕDÐà�a˜Ð/Ð0ˆä—u’u˜\¨7¸È1ÔMˆØ.AÀAÔ.EÐ*Ñ*È4ˆð +×2Ñ2°:Õ3TÐVXÓYˆÜ=Ø,Ø×(Ñ(Ø&×,Ñ,¨QÕ/Ø—*‘*ô	
ˆð $×(Ñ(¨Õ)JÈBÐPSÓTˆØ×!Ñ!ØØ)ð "ó 
ˆð ×/Ñ/ˆà×*Ñ*¨<Ó8ˆð $×+Ñ+ØÕ-¨y¸+ÐH[Õ:[Ð]`ó
ˆð ×:Ñ:¸<ÓZˆØ#×+Ñ+ØÕ-¨y¸KÐJ]Õ<]Õ/^Ð`có
ˆð ×/Ñ/ØØ)ð 0ó 
ˆð %×6Ñ6ˆð $×+Ñ+ØÕ-¨y¸+ÐH[Õ:[Ð]`ó
ˆð $ A q¨,¨;¨,Ð$6Õ7ˆØ#×+Ñ+¨JÈiÐYdÐfiÓjˆð LP×KkÒKkÓ)lÑKkÀa¨&×*>Ñ*>¸q×*AÐ*AÑKkÐ&Ð)lÜ%*§[¢[Ð1OÐUWÔ%XÐ"ð &@×%GÑ%GØÕ-¨y¸+ÐH[Õ:[Ð]_ó&
Ð"ð &@ÀÀ1ÀlÀ{ÀlÐ@RÕ%SÐ"Ø%?×%GÑ%GØ¨i¸Àbó&
Ð"ô
 —y’y ,Ð0JÐ!KÐQSÔTˆä°Ô>Ð>ùò *ms   Ì-Oc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r‚   rƒ   )r*   r\   s   "€r+   r,   r×  '  s   ø‡ ‚ ñ Ñò r-   )rÀ  rã  rë  ra   r‡   rà  rç  ræ  rQ   rß  rH   râ  rˆ   rå  rä  r‰   ré  )rÆ  )rw   rx   ry   rz   rÈ  rÉ  r`   rï  ró  r   r   r   rt   rÒ  r{   r|   r}   r~   s   @@r+   r½  r½  '  s[   ù‡ € ð 'ÐØ!Ð÷#ó #òJ$÷ð ð  ØØ÷H?ó ó ó  ðH?÷} … r-   r½  zc
    The Mllama Text Model which consists of transformer with self and cross attention layers.
    c                   ó†   a a€ ] tR tRt oRtR	tV3R lV 3R llt]]]	R
V3R lR ll4       4       4       t
V3R ltRtVtV ;t# )ÚMllamaTextModelið  zlanguage_model.modelc                ó    <€ V ^8„  d   QhRS[ /# r‚   r‹  )r*   r\   s   "€r+   r,   ÚMllamaTextModel.__annotate__ú  s   ø€ ÷ ñ Ñ/ñ r-   c                óª  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  ^,           VP                  V P                  4      V n        VP                  V n	        . p\        VP                  4       FI  pW0P                  9   d   VP                  \        W4      4       K/  VP                  \        W4      4       KK  	  \
        P                  ! V4      V n        \#        VP                  VP$                  R7      V n        \)        VR7      V n        RV n        V P/                  4        R# )é   rí   ©rY   FN)r_   r`   Úpad_token_idÚpadding_idxÚ
vocab_sizer   rc   ra   Úembed_tokensÚcross_attention_layersr  rè  Úappendr{  rn  r  r  r  r3  Únormrˆ  Ú
rotary_embr	  rì  )ri   rY   r  r/  rj   s   &&  €r+   r`   ÚMllamaTextModel.__init__ú  sú   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒÜŸLšL¨×):Ñ):¸QÕ)>À×@RÑ@RÐTX×TdÑTdÓeˆÔØ&,×&CÑ&CˆÔ#àˆÜ˜v×7Ñ7Ö8ˆIØ×7Ñ7Ô7Ø—‘Ô>¸vÓQÖRà—‘Ô=¸fÓPÖQñ	 9ô —m’m FÓ+ˆŒÜ% f×&8Ñ&8¸f×>QÑ>QÔRˆŒ	Ü/°vÔ>ˆŒà&+ˆÔ#Ø�‰Ör-   c                ó   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[R,          R	S[ P                  R,          R
S[R,          RS[S[,          RS[	/# )r    Ú	input_idsNrR   rt  r9  r!   rD   r:  Úinputs_embedsr;  r·   r$   )
r%   ru  r&   rv  r)   r   r:   r   r
   r   )r*   r\   s   "€r+   r,   r    sé   ø€ ÷ n
ñ n
á×#Ñ# dÕ*ðn
ñ Ÿ™ tÕ+ðn
ñ ×&Ñ&¨Õ-ð	n
ñ
 !&× 1Ñ 1°DÕ 8ðn
ñ $Ÿl™l¨TÕ1ðn
ñ (-©U¯\©\¹5¿<¹<Ð-GÕ'HÈ4Õ'Oðn
ñ  �ðn
ñ ×(Ñ(¨4Õ/ðn
ñ ˜$•;ðn
ñ Ñ-Õ.ðn
ñ 
!ñn
r-   c
                óV  € V	e   T	MV P                   P                  p	VRJ VRJ,          '       d   \        R4      hVf   V P                  V4      pTpV	'       d   Vf   \	        V P                   R7      pVfb   Ve   VP                  4       M^ p\        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                   VVVVR7      pV P                  W³R7      p\        V P                  4       Fo  w  ppWðP                  9   pVRJ ;'       g!    VRJ;'       d    VP                  V4      ^ 8H  pV'       d   Vf   V'       d   KV  V! V3RVRVR	VR
VRVRVRV	RV/V
B pKq  	  V P!                  V4      p\#        VVR7      # )aY  
cross_attention_states (`torch.FloatTensor`, *optional*):
    Output of the vision model, used for cross-attention. This tensor contains the processed image features that
    the language model will attend to.
cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
    Cross-attention mask to control the interaction between text tokens and image tiles.
    This 4D tensor defines which image tiles each text token should attend to.

    For each text token (in seq_length):
    - 1 indicates the token **should attend** to the corresponding image tile
    - 0 indicates the token **should not attend** to the corresponding image tile
full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
    A tuple containing two tensors that mask out rows in the cross-attention mechanism:
    - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
      A value of 0 indicates that the corresponding text token's entire row in the cross-attention
      matrix should be masked out (all image tokens ignored).
    - The second tensor has the same shape and is used internally to apply the masking during
      the forward pass of cross-attention layers.
    This mask is derived from the cross_attention_mask and is used to handle cases where a text token
    should not attend to any image token.

Example:

```python
>>> from transformers import AutoProcessor, MllamaTextModel

>>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
>>> model = MllamaTextModel.from_pretrained(checkpoint)
>>> processor = AutoProcessor.from_pretrained(checkpoint)

>>> text = "<|image|>If I had to write a haiku for this one"
>>> inputs = processor(text=text, return_tensors="pt")

>>> output = model(**inputs)

>>> print(output.last_hidden_state.shape)
torch.Size([1, 13, 4096])
```
Nú:You must specify exactly one of input_ids or inputs_embedsr  ©r‰  )rY   r   rR   r:  rt  )rt  r9  r!   rR   rD   rt  r:  r;  r_  )r  r:  )rY   r;  rB  r  r   r?  r%   r¢  r4   r‰  r7   r	   r  Ú	enumerater  r  r  r   )ri   r  rR   rt  r9  r!   rD   r:  r   r;  r·   r£   Úpast_seen_tokensÚcausal_maskr_  ÚidxÚdecoder_layerÚis_cross_attention_layerÚis_cross_attention_cache_emptys   &&&&&&&&&&,        r+   rt   ÚMllamaTextModel.forward  sî  € ðn "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	à˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMà%ˆç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð #Ÿo™o¨m˜oÓWÐô #,¨D¯K©KÖ"8ÑˆC�ð (+×.IÑ.IÑ'IÐ$Ø-<ÀÐ-D÷ .ð .Ø tÐ+×XÐX°×0NÑ0NÈsÓ0SÐWXÑ0Xð +÷ (Ð,BÒ,J×OmÙá)Øñà'=ðð &:ðð  +ð	ð
 /Lðð *ðð !0ðð $ðð %8ðð ñŠMñ #9ð2 Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r‚   r‹  )r*   r\   s   "€r+   r,   r  ð  ó   ø‡ ‚ ñ Ñò r-   )r  r  r	  r  r  r  r  r  )rÇ  )	NNNNNNNNN)rw   rx   ry   rz   rÈ  rÉ  r`   r   r   r   rt   rÒ  r{   r|   r}   r~   s   @@r+   r  r  ð  sQ   ù‡ € ð /ÐØ Ð÷ó ð*  ØØ÷n
ñ n
ó ó ó  ðn
÷E … r-   r  zE
    The Mllama Text Model with a language modeling head on top.
    c                   óp   a a€ ] tR tRt oRtRtV 3R lt]]R	V3R lR ll4       4       t	V3R lt
RtVtV ;t# )
ÚMllamaForCausalLMiƒ  TÚlanguage_modelc                ó‚  <€ \         SV `  VP                  4       4       VP                  4       V n        V P                  P                  V n        \
        P                  V P                  4      V n        \        P                  ! V P                  P                  V P                  R R7      V n        V P                  4        R# re  )r_   r`   r¼  Útext_configr  r  Ú_from_configr¶  r   r›   ra   Úlm_headrì  rŸ   s   &&€r+   r`   ÚMllamaForCausalLM.__init__�  s„   ø€ Ü‰Ñ˜×/Ñ/Ó1Ô2Ø!×1Ñ1Ó3ˆÔØ×*Ñ*×5Ñ5ˆŒÜ$×1Ñ1°$×2BÑ2BÓCˆŒ
Ü—y’y ×!1Ñ!1×!=Ñ!=¸t¿¹ÐUZÔ[ˆŒà�‰Ör-   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[R,          R	S[ P
                  R,          R
S[ P                  R,          RS[R,          RS[S[ P                  ,          RS[S[	,          RS[S[
,          /# )r    r  NrR   rt  r9  r!   rD   r:  r   Úlabelsr;  Úlogits_to_keepr·   r$   )r%   ru  r&   r)   r   rv  r:   r'   r   r   r   )r*   r\   s   "€r+   r,   ÚMllamaForCausalLM.__annotate__˜  s  ø€ ÷ W
ñ W
á×#Ñ# dÕ*ðW
ñ Ÿ™ tÕ+ðW
ñ ×&Ñ&¨Õ-ð	W
ñ
 !&× 0Ñ 0°4Õ 7ðW
ñ $×.Ñ.°Õ5ðW
ñ (-©U¯\©\¹5¿<¹<Ð-GÕ'HÈ4Õ'OðW
ñ  �ðW
ñ ×(Ñ(¨4Õ/ðW
ñ × Ñ  4Õ'ðW
ñ ˜$•;ðW
ñ ™eŸl™lÕ*ðW
ñ Ñ+Õ,ðW
ñ 
Ñ'Õ	'ñW
r-   c                ó¢  € V P                   ! RRVRVRVRVRVRVRVRVR	V
/	VB pVP                  p\        V\        4      '       d   \	        V) R
4      MTpV P                  VRVR3,          4      P                  4       pR
pV	e   V P                  ! VW�P                  3/ VB p\        VVVP                  VP                  VP                  R7      # )aã	  
cross_attention_states (`torch.FloatTensor`, *optional*):
    Output of the vision model, used for cross-attention. This tensor contains the processed image features that
    the language model will attend to.
cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
    Cross-attention mask to control the interaction between text tokens and image tiles.
    This 4D tensor defines which image tiles each text token should attend to.

    For each text token (in seq_length):
    - 1 indicates the token **should attend** to the corresponding image tile
    - 0 indicates the token **should not attend** to the corresponding image tile
full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
    A tuple containing two tensors that mask out rows in the cross-attention mechanism:
    - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
      A value of 0 indicates that the corresponding text token's entire row in the cross-attention
      matrix should be masked out (all image tokens ignored).
    - The second tensor has the same shape and is used internally to apply the masking during
      the forward pass of cross-attention layers.
    This mask is derived from the cross_attention_mask and is used to handle cases where a text token
    should not attend to any image token.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MllamaForCausalLM

>>> model = MllamaForCausalLM.from_pretrained("Llama-3.2-11B-Vision")
>>> tokenizer = AutoTokenizer.from_pretrained("Llama-3.2-11B-Vision")

>>> prompt = "If I had to write a haiku, it would be:"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6)
>>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
>>> print(result)
If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful.
I love the idea of snowflakes gently falling, each one
```
r  r9  rR   rt  r!   rD   r:  r   r;  NrL   ©ÚlossÚlogitsr:  r£   r·  r  )r¶  r  r«  r'   Úslicer4  r¹   Úloss_functionr  r   r:  r£   r·  )ri   r  rR   rt  r9  r!   rD   r:  r   r7  r;  r8  r·   Úoutputsr£   Úslice_indicesr=  r<  s   &&&&&&&&&&&&,     r+   rt   ÚMllamaForCausalLM.forward–  s  € ð| —*’*ñ 
Øð
à#9ð
ð *ð
ð &ð	
ð
 "6ð
ð +Hð
ð ,ð
ð (ð
ð  ð
ð ñ
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓA×GÑGÓIˆàˆØÒØ×%Ò% f¨f·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r‚   r‹  )r*   r\   s   "€r+   r,   r9  ƒ  r-  r-   )r4  r¶  r2  r  )NNNNNNNNNNr…  )rw   rx   ry   rz   rÌ  rÈ  r`   r   r   rt   rÒ  r{   r|   r}   r~   s   @@r+   r/  r/  ƒ  sC   ù‡ € ð "ÐØ(Ðõð Ø÷W
ñ W
ó ó ðW
÷+ … r-   r/  zr
    The Mllama model which consists of a vision encoder and a language model without language modeling head.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚMllamaModeliò  c                ó    <€ V ^8„  d   QhRS[ /# r‚   rÄ  )r*   r\   s   "€r+   r,   ÚMllamaModel.__annotate__ø  s   ø€ ÷ ñ ™|ñ r-   c                ó:  <€ \         SV `  V4       VP                  P                  V n        VP                  P                  V n        VP
                  P                  V n        VP
                  P                  V n        \        P                  VP
                  4      V n
        \        P                  VP                  4      V n        \        P                  ! VP
                  P                  VP                  P                  R R7      V n        V P!                  4        R# )TrÎ   N)r_   r`   r2  r  ra   Úvision_configrQ   Úvision_output_dimr½  r3  rÕ  r  r0  r   r›   Úmulti_modal_projectorrì  rŸ   s   &&€r+   r`   ÚMllamaModel.__init__ø  sÏ   ø€ Ü‰Ñ˜Ô Ø ×,Ñ,×7Ñ7ˆŒØ!×-Ñ-×9Ñ9ˆÔØ#×1Ñ1×?Ñ?ˆÔØ!'×!5Ñ!5×!GÑ!GˆÔä-×:Ñ:¸6×;OÑ;OÓPˆÔÜ-×:Ñ:¸6×;MÑ;MÓNˆÔÜ%'§Y¢YØ× Ñ ×2Ñ2Ø×Ñ×*Ñ*Øô&
ˆÔ"ð
 	�‰Ör-   c                óÆ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[ P                  R,          RS[R,          RS[S[,          RS[/# )r    r  Nrö  rG   ro   rR   r!   r9  rt  r:  r   r;  r·   r$   )	r%   ru  rv  r&   r   r:   r   r
   r   )r*   r\   s   "€r+   r,   rG  
  s  ø€ ÷ d
ñ d
á×#Ñ# dÕ*ðd
ñ ×'Ñ'¨$Õ.ðd
ñ !Ÿ<™<¨$Õ.ð	d
ñ
  Ÿ,™,¨Õ-ðd
ñ Ÿ™ tÕ+ðd
ñ $Ÿl™l¨TÕ1ðd
ñ !&§¡¨tÕ 3ðd
ñ ×&Ñ&¨Õ-ðd
ñ  �ðd
ñ ×(Ñ(¨4Õ/ðd
ñ ˜$•;ðd
ñ Ñ-Õ.ðd
ñ 
!ñd
r-   c                ó’  € VRJ V
RJ,          '       d   \        R4      hVe   Ve   \        R4      hVem   Vf   \        R4      hV P                  VVVR7      pVP                  pV P                  V4      P	                  RVP
                  R,          V P                  4      pVe0   \        VV P                  P                  V P                  R7      w  rnMRpVe”   V	e   V	P                  4       M^ pVe   VP
                  ^,          MV
P
                  ^,          pVe   VP                  MV
P                  p\        P                  ! VVR7      V,           pVRRV3,          pVRRV3,          pV P                  ! RR	VR
VRVRVRVRVRV	RVRV
/	VB p\        VP                  VP                   VP"                  VP$                  R7      # )aº  
aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
    Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

    - 1 for tiles that are **not masked**,
    - 0 for tiles that are **masked**.
aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
    Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
    These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

    For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
    - An image with aspect ratio [1, 1] would have ID 1
    - An image with aspect ratio [1, 2] would have ID 2
    - An image with aspect ratio [2, 1] would have ID 3

    The id 0 is reserved for padding (i.e., no image).

    If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
    Cross-attention mask to control the interaction between text tokens and image tiles.
    This 4D tensor defines which image tiles each text token should attend to.

    For each text token (in seq_length):
    - 1 indicates the token **should attend** to the corresponding image tile
    - 0 indicates the token **should not attend** to the corresponding image tile
cross_attention_states (`torch.FloatTensor`, *optional*):
    Output of the vision model, used for cross-attention. This tensor contains the processed image features that
    the language model will attend to.
Nr"  zM`pixel_values` and `cross_attention_states` cannot be provided simultaneouslyzA`aspect_ratio_ids` must be provided if `pixel_values` is provided)rö  ro   rG   )r"   r#   r#  rL   r  rR   rt  r9  r!   rD   r:  r;  r   )r  r:  r£   r·  r3   rM   r  )rB  rÕ  r  rK  rO   r4   ra   rE   rH   r#   r?  r‰  r%   r¢  r0  r   r:  r£   r·  )ri   r  rö  rG   ro   rR   r!   r9  rt  r:  r   r;  r·   Úvision_outputsrD   r%  Úseq_lenr‰  Úcurrent_posr@  s   &&&&&&&&&&&&,       r+   rt   ÚMllamaModel.forward  s(  € ð\ ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ#Ð(>Ò(JÜÐlÓmÐmàÒ#ØÒ'Ü Ð!dÓeÐeà!×.Ñ.Ø)Ø!1Ø"3ð /ó ˆNð
 &4×%EÑ%EÐ"Ø%)×%?Ñ%?Ð@VÓ%W×%_Ñ%_ØÐ*×0Ñ0°Õ4°d×6FÑ6Fó&Ð"ð  Ò+ÜB_Ø$Ø"&×"3Ñ"3×"?Ñ"?Ø—j‘jôCÑ?Ð Ð"?ð -1Ð)àÒ+ØCRÒC^˜×=Ñ=Ô?ÐdeÐØ,5Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆGØ)2Ò)>�Y×%Ò%ÀM×DXÑDXˆFÜŸ,š, w°vÔ>ÐAQÕQˆKà#7¸¸1¸kÐ8IÕ#JÐ Ø,IÈ!ÈQÐP[ÐJ[Õ,\Ð)à×%Ò%ñ 
Øð
à)ð
ð &ð
ð $:ð	
ð
 "6ð
ð +Hð
ð ,ð
ð  ð
ð (ð
ð ñ
ˆô 'Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r-   )ra   r0  rQ   rK  rÕ  rJ  r  )NNNNNNNNNNN)rw   rx   ry   rz   r`   r   r   rt   r{   r|   r}   r~   s   @@r+   rE  rE  ò  s9   ù‡ € ÷ó ð  Ø÷d
ñ d
ó ó ÷d
ð d
r-   rE  zS
    The Mllama model which consists of a vision encoder and a language model.
    c                   ó„   a a€ ] tR tRt oV3R lV 3R llt]]R	V3R lR ll4       4       tR
V 3R lltV 3R lt	Rt
VtV ;t# )ÚMllamaForConditionalGenerationiq  c                ó    <€ V ^8„  d   QhRS[ /# r‚   rÄ  )r*   r\   s   "€r+   r,   Ú+MllamaForConditionalGeneration.__annotate__y  s   ø€ ÷ ñ ™|ñ r-   c                óò   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  P                  VP                  P                  R R7      V n	        V P                  4        R# re  )r_   r`   rE  r¶  r   r›   r2  ra   r  r4  rì  rŸ   s   &&€r+   r`   Ú'MllamaForConditionalGeneration.__init__y  sS   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ü—y’y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰Ör-   c                ó(  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  ,          RS[S[,          RS[	S[
,          /# )r    r  Nrö  rG   ro   rR   r!   r9  rt  r:  r   r7  r;  r8  r·   r$   )r%   ru  rv  r&   r   r:   r'   r   r   r)   r   )r*   r\   s   "€r+   r,   rV  �  s3  ø€ ÷ m
ñ m
á×#Ñ# dÕ*ðm
ñ ×'Ñ'¨$Õ.ðm
ñ !Ÿ<™<¨$Õ.ð	m
ñ
  Ÿ,™,¨Õ-ðm
ñ Ÿ™ tÕ+ðm
ñ $Ÿl™l¨TÕ1ðm
ñ !&§¡¨tÕ 3ðm
ñ ×&Ñ&¨Õ-ðm
ñ  �ðm
ñ ×(Ñ(¨4Õ/ðm
ñ × Ñ  4Õ'ðm
ñ ˜$•;ðm
ñ ™eŸl™lÕ*ðm
ñ Ñ+Õ,ðm
ñ  
Ñ'Õ	'ñ!m
r-   c                ó¶  € V P                   ! RRVRVRVRVRVRVRVRVR	V	R
V
RV/VB pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe3   V P                  ! VW°P                  P                  P                  3/ VB p\        VVVP                  VP                  VP                  R7      # )a/  
aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
    Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

    - 1 for tiles that are **not masked**,
    - 0 for tiles that are **masked**.
aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
    Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
    These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

    For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
    - An image with aspect ratio [1, 1] would have ID 1
    - An image with aspect ratio [1, 2] would have ID 2
    - An image with aspect ratio [2, 1] would have ID 3

    The id 0 is reserved for padding (i.e., no image).

    If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
    Cross-attention mask to control the interaction between text tokens and image tiles.
    This 4D tensor defines which image tiles each text token should attend to.

    For each text token (in seq_length):
    - 1 indicates the token **should attend** to the corresponding image tile
    - 0 indicates the token **should not attend** to the corresponding image tile
cross_attention_states (`torch.FloatTensor`, *optional*):
    Output of the vision model, used for cross-attention. This tensor contains the processed image features that
    the language model will attend to.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, MllamaForConditionalGeneration

>>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
>>> model = MllamaForConditionalGeneration.from_pretrained(checkpoint)
>>> processor = AutoProcessor.from_pretrained(checkpoint)

>>> prompt = "<|image|>If I had to write a haiku for this one"
>>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> inputs = processor(text=prompt, images=image, return_tensors="pt")

>>> # Generate
>>> output = model.generate(**inputs, max_new_tokens=15)

>>> prompt_len = inputs.input_ids.shape[-1]
>>> generated_ids = output[:, prompt_len:]
>>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
>>> print(generated_text)
[', it would be:.\\nA stop sign in Chinatown.\\n']
```
r  rö  rG   ro   r!   r9  rR   rt  r:  r   r;  NrL   r;  r  )r¶  r  r«  r'   r>  r4  r?  rY   r2  r  r   r:  r£   r·  )ri   r  rö  rG   ro   rR   r!   r9  rt  r:  r   r7  r;  r8  r·   r@  r£   rA  r=  r<  s   &&&&&&&&&&&&&&,     r+   rt   Ú&MllamaForConditionalGeneration.forward  s-  € ðb —*’*ñ 
Øð
à%ð
ð 0ð
ð .ð	
ð
 "6ð
ð $:ð
ð *ð
ð &ð
ð ,ð
ð (ð
ð  ð
ð ñ
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f¨f·k±k×6MÑ6M×6XÑ6XÑcÐ\bÑcˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r-   c                ó”   <€ \         SV `  ! V3R V	RV
RVRVRVRVRVRVRVR	VR
V/VB pV'       g   V
'       d   RVR&   RVR&   RVR&   V# )r:  r;  r   rt  rR   rö  ro   rG   r!   r8  Úis_first_iterationN)r_   Úprepare_inputs_for_generation)ri   r  r   rR   rt  rö  ro   rG   r!   r:  r;  r8  r]  r·   Úmodel_inputsrj   s   &&&&&&&&&&&&&, €r+   r^  Ú<MllamaForConditionalGeneration.prepare_inputs_for_generationð  sº   ø€ ô$ ‘wÒ<Øñ
à+ð
ð  ð
ð (ð	
ð
 &ð
ð *ð
ð &ð
ð .ð
ð 0ð
ð "6ð
ð *ð
ð  2ð
ð ñ
ˆ÷$ "§iØ+/ˆL˜Ñ(Ø/3ˆLÐ+Ñ,Ø04ˆLÐ,Ñ-àÐr-   c           	     ó°   <€ VP                  R R4      p\        SV `  ! RRVRVRV/VB pVe)   \        P                  ! WURR	R1R3,          .^R7      VR &   V# )
r!   Nr@  Úmodel_kwargsÚis_encoder_decoderrL   .r0   r  r3   )Úgetr_   Ú#_update_model_kwargs_for_generationr%   rH  )ri   r@  rb  rc  r·   Úcross_attention_mask_prevrj   s   &&&&, €r+   re  ÚBMllamaForConditionalGeneration._update_model_kwargs_for_generation  s„   ø€ Ø$0×$4Ñ$4Ð5KÈTÓ$RÐ!Ü‘wÒBñ 
Øð
à%ð
ð  2ð
ð ñ	
ˆð %Ò0Ü38·9²9Ø*ÀaÈÉÈcÀkÕ,RÐSÐYZô4ˆLÐ/Ñ0ð Ðr-   )r4  r¶  )NNNNNNNNNNNNr…  )NNNNNNNNNFr…  F)rw   rx   ry   rz   r`   r   r   rt   r^  re  r{   r|   r}   r~   s   @@r+   rT  rT  q  sD   ù‡ € ÷ó ð Ø÷m
ñ m
ó ó ðm
÷^)÷Võ r-   rT  )rà   r�   )rT  r/  r  r½  rµ  rE  )Sr  r÷   Úcollections.abcr   r%   Útorch.nn.functionalr   r¿   rû  Ú r   r¾  Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr	   Úmodeling_flash_attention_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_mllamar   r   r   Ú
get_loggerrw   ÚloggerrE   rT   r¸   rV   r€   r–   r¯   rÇ   rÉ   ré   r  r  r-  rL  rU  rW  rc  rn  r{  rˆ  rµ  r½  r  r/  rE  rT  Ú__all__r  r-   r+   Ú<module>r}     s_  ðñ ã Ý $ã ß Ð Ý å &Ý !ß .Ý )Ý /Ý BÝ 9ß `Ñ `÷÷ GÝ &ß RÓ RÝ 0ß GÝ 5ß TÑ Tð 
×	Ò	˜HÓ	%€õ?õ8ô6¨B¯I©Iô ô."¨¯©ô "ôL�b—i‘iô õ 	U÷%ô24)˜BŸI™Iô 4)ôn)˜rŸy™yô )ôX,^˜"Ÿ)™)ô ,^ô`J˜Ÿ	™	ô Jô(Q)˜rŸy™yô Q)òj(ôô2>)˜bŸi™iô >)ôD�B—I‘Iô ô$BÐ&@ô BôJ/Ð'Aô /ôf4<˜BŸI™Iô 4<ðn ô*)˜Oó *)ó ð*)ñZ ðôô
A?Ð-ó A?óð
A?ñH ðôô
K
Ð+ó K
óð
K
ñ\ ðôô
g
Ð-¨ó g
óð
g
ñT ðôô
w
Ð'ó w
óð
w
ñt ðôô
sÐ%:¸Oó sóð
sòl‚r-   