+
    œ°³jÎœ  ã                   óÈ  € ^ RI Ht ^ RIHt ^ RIt^ RIHt ^ RIHt ^RI	H
t ^RIHtHt ^RIHt ^RIHtHtHtHtHtHtHt ^R	IHtHt ^R
IHtHtHt ^RI H!t!H"t" ^RI#H$t$H%t% ^RI&H't' ^RI(H)t)H*t*H+t+H,t, ^RI-H.t. ^RI/H0t0 ^RI1H2t2H3t3H4t4H5t5H6t6H7t7H8t8H9t9H:t: ^RI;H<t<H=t=H>t>H?t? ^RI@HAtA ],P„                  ! ]C4      tD]*! RR7      ] ! R R]0]4      4       4       tE]*! RR7      ] ! R R]4      4       4       tF ! R R]?4      tG ! R R]<4      tH ! R R]P’                  4      tJ ! R  R!]44      tK ! R" R#]74      tL ! R$ R%]84      tM ! R& R']24      tN ! R( R)]4      tORtP ! R* R+]64      tQR, R- ltR ! R. R/]54      tS ! R0 R1]34      tT ! R2 R3]Pª                  4      tVRBR4 R5 lltWR6 R7 ltX ! R8 R9]>4      tY ! R: R;]=4      tZ]*! R<R=7       ! R> R?]]Q4      4       t[ ! R@ RA]]Q4      t\. RCOt]R# )Dé    )ÚCallable)ÚAnyN)Ústrict)Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocast)ÚGemma2Config)	ÚGemma2AttentionÚGemma2ForCausalLMÚ	Gemma2MLPÚGemma2ModelÚGemma2PreTrainedModelÚGemma2RMSNormÚGemma2RotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚPaliGemmaCausalLMOutputWithPastÚ!PaliGemmaForConditionalGenerationÚPaliGemmaModelÚPaligemmaModelOutputWithPast)ÚSiglipVisionConfigzgoogle/gemma-3-4b-it)Ú
checkpointc                   ó®   a a€ ] tR t^Dt oRtRtRRRRRRRRR	RR
RRRRRRR/	tRRRR/tRtRt	Rt
RtRtRtRtR tV3R lV 3R lltR tV3R ltRtVtV ;t# )ÚGemma3TextConfiga�  
query_pre_attn_scalar (`float`, *optional*, defaults to 256):
    scaling factor used on the attention scores
final_logit_softcapping (`float`, *optional*):
    Scaling factor when applying tanh softcapping on the logits.
attn_logit_softcapping (`float`, *optional*):
    Scaling factor when applying tanh softcapping on the attention scores.
use_bidirectional_attention (`bool`, *optional*, defaults to `False`):
    If True, the model will attend to all text tokens instead of using a causal mask. This does not change
    behavior for vision tokens.

```python
>>> from transformers import Gemma3TextModel, Gemma3TextConfig
>>> # Initializing a Gemma3Text gemma3_text-7b style configuration
>>> configuration = Gemma3TextConfig()
>>> # Initializing a model from the gemma3_text-7b style configuration
>>> model = Gemma3TextModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```
Úgemma3_textzlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.self_attn.o_projÚrowwisezlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚglobalg    €„.AÚlocalg     ˆÃ@i@  i   NFc                ó’  € V P                   '       d    V P                  ^,          ^,           V n        VP                  R^4      V n        V P                  fV   \        V P                  4       Uu. uF0  p\        V^,           V P                  ,          4      '       d   RMRNK2  	  upV n        \        P                  ! R/ VB  R# u upi )é   Úsliding_window_patternNÚsliding_attentionÚfull_attention© )
Úuse_bidirectional_attentionÚsliding_windowÚgetÚ_sliding_window_patternÚlayer_typesÚrangeÚnum_hidden_layersÚboolr	   Ú__post_init__)ÚselfÚkwargsÚis   &, Új/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gemma3/modular_gemma3.pyrF   ÚGemma3TextConfig.__post_init__s   s­   € Ø×+×+Ð+Ø#'×#6Ñ#6¸!Õ#;¸qÕ"@ˆDÔð (.§z¡zÐ2JÈAÓ'NˆÔ$à×ÑÒ#ô ˜t×5Ñ5Ô6ó á6�Aô (,¨Q°­U°d×6RÑ6RÕ,R×'SÒ'SÑ#ÐYiÒiÙ6ñ ˆDÔô
 	×&Ò&Ñ0¨Ô0ùò s   Á.6Cc                ó6   <€ V ^8„  d   QhRS[ S[S[3,          /# )r9   Úreturn)ÚdictÚstrr   )ÚformatÚ__classdict__s   "€rJ   Ú__annotate__ÚGemma3TextConfig.__annotate__‚   s   ø€ ÷ ñ ™™c¡3˜h�ñ ó    c                ó„   <€ \         SV `  4       p\        V R R4      '       d   V P                  ^,
          ^,          VR&   V# )r>   Fr?   )ÚsuperÚto_dictÚgetattrr?   )rG   ÚoutputÚ	__class__s   & €rJ   rW   ÚGemma3TextConfig.to_dict‚   sA   ø€ Ü‘‘Ó"ˆô �4Ð6¸×>Ò>Ø(,×(;Ñ(;¸aÕ(?À1Õ'DˆFÐ#Ñ$ØˆrT   c                ó´  € VP                  R R4      pRRR/RRR//pV P                  e   V P                  MTV n        Ve#   V P                  R,          P                  V4       V P                  P                  R4      f   RR/V P                  R&   V P                  R,          P	                  RVP                  RV P
                  R,          4      4       V P                  P                  R4      f   RR/V P                  R&   V P                  R,          P	                  RVP                  RV P
                  R	,          4      4       V P                  4        V# )
Úrope_scalingNr;   Ú	rope_typeÚdefaultr<   Ú
rope_thetar6   Úrope_local_base_freqr7   )ÚpopÚrope_parametersÚupdater@   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rG   rH   r]   Údefault_rope_paramss   &,  rJ   Úconvert_rope_params_to_dictÚ,Gemma3TextConfig.convert_rope_params_to_dict‹   sM  € Ø—z‘z .°$Ó7ˆð
   +¨yÐ!9Ø˜{¨IÐ6ð
Ðð 8<×7KÑ7KÒ7W˜t×3Ò3Ð]pˆÔØÒ#Ø× Ñ Ð!1Õ2×9Ñ9¸,ÔGð ×Ñ×#Ñ#Ð$4Ó5Ò=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Õ.×9Ñ9Ø˜&Ÿ*™* \°4×3EÑ3EÀhÕ3OÓPô	
ð ×Ñ×#Ñ#Ð$7Ó8Ò@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Õ1×<Ñ<Ø˜&Ÿ*™*Ð%;¸T×=OÑ=OÐPWÕ=XÓYô	
ð
 	×$Ñ$Ô&ØˆrT   c                óÄ   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[S[,          R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   # )	r9   Ú
vocab_sizeÚmax_position_embeddingsNrB   Úfinal_logit_softcappingÚattn_logit_softcappingrc   r>   )ÚintÚlistrO   ÚfloatrN   rE   )rP   rQ   s   "€rJ   rR   rS   D   s‡   ø‡ ‚ ñN ÑñO ñP !Ñ*ñQ ñR ‘c•˜TÕ!Ñ(ñS ñT # T�\Ñ0ñU ñV " D�LÑ/ñW ñX ˜D•[Ñ'ñY ñZ "&¨¥Ñ4ò[ rT   )rA   rB   rc   r?   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚbase_model_tp_planrf   rl   rm   rB   rn   ro   rc   r>   rF   rW   ri   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rZ   rQ   s   @@rJ   r1   r1   D   s›   ù‡ € ñð, €Jà# YØ# YØ# YØ#Ð%EØ#Ð%EØ# YØ  )Ø 	Ø  )ð
Ðð ˜{¨G°XÐ>€Mà€JØ#*ÐØ$(€KØ,0ÐØ+/ÐØ#'€OØ/4Ðò1÷ó ò÷O … rT   r1   c                   ó‚   a a€ ] tR t^©t oRtRtRRRRRR/tR	]R
]/t	Rt
RtRtRtRtRtRtRtV 3R ltV3R ltRtVtV ;t# )ÚGemma3ConfigaÃ  
mm_tokens_per_image (`int`, *optional*, defaults to 256):
    The number of tokens per image embedding.
boi_token_index (`int`, *optional*, defaults to 255999):
    The begin-of-image token index to wrap the image prompt.
eoi_token_index (`int`, *optional*, defaults to 256000):
    The end-of-image token index to wrap the image prompt.

Example:

```python
>>> from transformers import Gemma3ForConditionalGeneration, Gemma3Config, SiglipVisionConfig, Gemma3TextConfig

>>> # Initializing a Siglip-like vision config
>>> vision_config = SiglipVisionConfig()

>>> # Initializing a Gemma3 Text config
>>> text_config = Gemma3TextConfig()

>>> # Initializing a Gemma3 gemma-3-4b style configuration
>>> configuration = Gemma3Config(vision_config, text_config)

>>> # Initializing a model from the gemma-3-4b style configuration
>>> model = Gemma3ForConditionalGeneration(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```Úgemma3Úimage_token_idÚimage_token_indexÚboi_token_idÚboi_token_indexÚeoi_token_idÚeoi_token_indexÚtext_configÚvision_configNé   iÿç i è i   g{®Gáz”?Tc                óÜ  <€ V P                   f&   \        4       V n         \        P                  R4       M:\	        V P                   \
        4      '       d   \        R/ V P                   B V n         \	        V P                  \
        4      '       d   \        R/ V P                  B V n        M2V P                  f%   \        4       V n        \        P                  R4       \        SV `$  ! R/ VB  R # )Nz@text_config is None, using default Gemma3TextConfig text config.zFvision_config is None, using default SiglipVisionConfig vision config.r=   )
rˆ   r1   ÚloggerÚinfoÚ
isinstancerN   r‰   r.   rV   rF   )rG   rH   rZ   s   &,€rJ   rF   ÚGemma3Config.__post_init__Ý   s­   ø€ Ø×ÑÒ#Ü/Ó1ˆDÔÜ�K‰KÐZÕ[Ü˜×(Ñ(¬$×/Ò/Ü/ÑC°$×2BÑ2BÑCˆDÔä�d×(Ñ(¬$×/Ò/Ü!3Ñ!I°d×6HÑ6HÑ!IˆDÕØ×ÑÒ'Ü!3Ó!5ˆDÔÜ�K‰KÐ`Ôaä‰ÒÑ' Ô'rT   c                ó6  <€ V ^8„  d   Qh/ S[ S[S[S[3,          ,          R,          ;R&   S[S[S[S[3,          ,          R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R	&   # )
r9   Nrˆ   r‰   Úmm_tokens_per_imager…   r‡   rƒ   Úinitializer_rangeÚtie_word_embeddings)r1   rN   rO   r   r.   rp   rr   rE   )rP   rQ   s   "€rJ   rR   ÚGemma3Config.__annotate__©   s¹   ø‡ ‚ ñV "¡D©©c¨¥NÕ2°TÕ9Ñ@ñW ñX &©©S±#¨X­Õ6¸Õ=ÑDñY ñZ ˜t�Ñ)ñ[ ñ\ ˜4•ZÑ)ñ] ñ^ ˜4•ZÑ)ñ_ ñ` ˜T•zÑ+ña ñb ˜t•|Ñ*ñc ñd  �Ñ+òe rT   )rˆ   r‰   )rs   rt   ru   rv   rw   rx   Úattribute_mapr1   r.   Úsub_configsrˆ   r‰   r‘   r…   r‡   rƒ   r’   r“   rF   rz   r{   r|   r}   r~   s   @@rJ   r€   r€   ©   s{   ù‡ € ñð: €JàÐ-ØÐ)ØÐ)ð€Mð 	Ð'ØÐ+ð€Kð
 =A€KØ@D€MØ&)ÐØ")€OØ")€OØ$+ÐØ&*ÐØ'+Ðõ(÷i … rT   r€   c                   ó   € ] tR t^ítRtR# )ÚGemma3ModelOutputWithPastr=   N©rs   rt   ru   rv   r{   r=   rT   rJ   r˜   r˜   í   ó   † ÛrT   r˜   c                   ó   € ] tR t^ñtRtR# )ÚGemma3CausalLMOutputWithPastr=   Nr™   r=   rT   rJ   rœ   rœ   ñ   rš   rT   rœ   c                   ó^   a a€ ] tR t^õt oRtRV3R lV 3R llltV3R lV 3R lltRtVtV ;t	# )ÚGemma3TextScaledWordEmbeddingzT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r9   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scale©rp   rr   )rP   rQ   s   "€rJ   rR   Ú*Gemma3TextScaledWordEmbedding.__annotate__ú   s0   ø€ ÷ Rñ R¡sð R¹3ð RÉSð RÑ_dñ RrT   c                ó–   <€ \         SV `  WV4       W@n        \        P                  ! \
        P                  ! V4      R R7      V n        R# )F©Ú
persistentN)rV   Ú__init__Úscalar_embed_scaleÚnnÚBufferÚtorchÚtensorr£   )rG   r    r¡   r¢   r£   rZ   s   &&&&&€rJ   r©   Ú&Gemma3TextScaledWordEmbedding.__init__ú   s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÔQˆÖrT   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r9   Ú	input_ids©r­   ÚTensor)rP   rQ   s   "€rJ   rR   r¥   ÿ   s   ø€ ÷ Sñ S¡§¡ñ SrT   c                óŠ   <€ \         SV `  V4      V P                  P                  V P                  P
                  4      ,          # ©N)rV   Úforwardr£   ÚtoÚweightÚdtype)rG   r±   rZ   s   &&€rJ   r¶   Ú%Gemma3TextScaledWordEmbedding.forwardÿ   s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÕRÐRrT   )r£   rª   )ç      ð?)
rs   rt   ru   rv   rw   r©   r¶   r{   r|   r}   r~   s   @@rJ   rž   rž   õ   s(   ù‡ € ñ÷Rõ R÷
S÷ Só SrT   rž   c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )Ú	Gemma3MLPi  c                ó    <€ V ^8„  d   QhRS[ /# ©r9   Úconfig©r1   )rP   rQ   s   "€rJ   rR   ÚGemma3MLP.__annotate__  s   ø€ ÷ !ñ !Ñ/ñ !rT   c                ó&   <€ \         SV `  V4       R # rµ   ©rV   r©   ©rG   rÀ   rZ   s   &&€rJ   r©   ÚGemma3MLP.__init__  s   ø€ Ü‰Ñ˜Ö rT   r=   ©rs   rt   ru   rv   r©   r{   r|   r}   r~   s   @@rJ   r½   r½     s   ù‡ € ÷!÷ !ó !rT   r½   c                   óB   a a€ ] tR tRt oRV3R lV 3R llltRtVtV ;t# )ÚGemma3RMSNormi  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r9   ÚdimÚepsr¤   )rP   rQ   s   "€rJ   rR   ÚGemma3RMSNorm.__annotate__	  s   ø€ ÷ +ñ +™Cð +¡eñ +rT   c                ó(   <€ \         SV `  WR 7       R# )©rË   rÌ   NrÄ   )rG   rË   rÌ   rZ   s   &&&€rJ   r©   ÚGemma3RMSNorm.__init__	  s   ø€ Ü‰Ñ˜SÐÖ*rT   r=   )g�íµ ÷Æ°>rÇ   r~   s   @@rJ   rÉ   rÉ     s   ù‡ € ÷+÷ +õ +rT   rÉ   c                   ó„   a € ] tR tRt o R	V 3R lR lltR
V 3R lR llt]P                  ! 4       ]R 4       4       t	Rt
V tR# )ÚGemma3RotaryEmbeddingi  Nc                ó    <€ V ^8„  d   QhRS[ /# r¿   rÁ   )rP   rQ   s   "€rJ   rR   Ú"Gemma3RotaryEmbedding.__annotate__  s   ø€ ÷ Uñ UÑ/ñ UrT   c           
     óþ  € \         P                  P                  V 4       VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n	        / V n
        V P                   Fû  pV P                  P                  V,          pVf   K&  VR,          V P                  V&   V P                  pV P                  V,          R8w  d   \        V P                  V,          ,          pV! V P                  W#R7      w  rg\        W R2\         P                  ! VRR7      4       \        W R2\         P                  ! VP!                  4       RR7      4       \        W R2V4       Ký  	  R # )	Nr^   r_   ©Ú
layer_typeÚ	_inv_freqFr§   Ú_original_inv_freqÚ_attention_scaling)r«   ÚModuler©   rm   Úmax_seq_len_cachedÚoriginal_max_seq_lenrÀ   ÚsortedÚsetrB   r^   rc   Úcompute_default_rope_parametersr   Úsetattrr¬   Úclone)rG   rÀ   Údevicer×   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalings   &&&     rJ   r©   ÚGemma3RotaryEmbedding.__init__  s-  € Ü
�	‰	×Ñ˜4Ô Ø"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà)4°[Õ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÕ)¨YÔ6Ü2°4·>±>À*Õ3MÕN�Ù4@ÀÇÁÈfÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó +rT   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r9   rÀ   r×   NrM   )r1   rO   Útupler­   r³   rr   )rP   rQ   s   "€rJ   rR   rÔ   #  s7   ø€ ÷ 5ñ 5Ù ð5Ù;>À½:ð5á	‰u�|‰|™UÐ"Õ	#ñ5rT   c           	     ó>  € V P                   V,          R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
r`   Úhead_dimNr»   ©r¹   )rc   rX   Úhidden_sizeÚnum_attention_headsr­   Úarangerr   r·   )rÀ   rã   r×   rH   ÚbaserË   Úattention_factorÚinv_freqs   &&&,    rJ   rà   Ú5Gemma3RotaryEmbedding.compute_default_rope_parameters#  sƒ   € ð" ×%Ñ% jÕ1°,Õ?ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4rT   c                óZ  € \        W R 24      p\        W R24      pVR
,          P                  VP                  ^ ,          R^4      P                  \        P
                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	\        P                  ! W™3RR7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP                  VP                  R	7      XP                  VP                  R	7      3#   + '       g   i     LH; i)rØ   rÚ   N)r¹   rã   ÚmpsÚcpuF)Údevice_typeÚenabled©rË   rí   )NºNNNNéÿÿÿÿ)rû   Nrû   )rX   ÚexpandÚshaper·   r­   rr   rã   rŽ   ÚtyperO   r   Ú	transposeÚcatÚcosÚsinr¹   )rG   ÚxÚposition_idsr×   ró   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrø   ÚfreqsÚembr  r  s   &&&&         rJ   r¶   ÚGemma3RotaryEmbedding.forward<  sJ  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã:A FÆF*	)rÀ   rB   rÜ   rÝ   r^   rµ   )NN)rs   rt   ru   rv   r©   rà   r­   Úno_gradr   r¶   r{   r|   ©rQ   s   @rJ   rÒ   rÒ     s=   ø‡ € ÷Uò U÷*5ò 5ð2 ‡]‚]ƒ_Øñ<ó ó ö<rT   rÒ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3AttentioniS  c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r9   rÀ   Ú	layer_idx©r1   rp   )rP   rQ   s   "€rJ   rR   ÚGemma3Attention.__annotate__T  s    ø€ ÷ Rñ RÑ/ð R¹Cñ RrT   c                ón  <€ \         SV `  W4       V P                  R 8X  d   VP                  MRV n        V P                  R 8H  V n        V P
                  P                  '       * V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )r;   NrÏ   )rV   r©   r×   r?   Ú
is_slidingrÀ   r>   Ú	is_causalrÉ   rì   Úrms_norm_epsÚq_normÚk_norm©rG   rÀ   r  rZ   s   &&&€rJ   r©   ÚGemma3Attention.__init__T  s‚   ø€ Ü‰Ñ˜Ô+Ø7;·±ÐJ]Ô7]˜f×3Ò3ÐcgˆÔØŸ/™/Ð-@Ñ@ˆŒØ!Ÿ[™[×DÑDÔDˆŒä#¨¯©¸V×=PÑ=PÔQˆŒÜ#¨¯©¸V×=PÑ=PÔQˆŽrT   c                ó(  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r9   Úhidden_statesÚposition_embeddingsÚattention_maskNÚpast_key_valuesrH   rM   )r­   r³   r   r   r   rê   )rP   rQ   s   "€rJ   rR   r  ]  s‰   ø€ ÷ *)ñ *)á—|‘|ð*)ñ #Ÿ\™\ð*)ñ Ÿ™ tÕ+ð	*)ñ
  �ð*)ñ Ñ+Õ,ð*)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ*)rT   c           	     ól  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      pV P                  V	4      p	Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \         4      pT! V VV	V
V3RV P"                  '       d   V P$                  MRRV P&                  RV P(                  /VB w  rïVP*                  ! . VORN5!  P-                  4       pV P/                  V4      pWï3# )NÚdropoutg        Úscalingr?   rü   )rþ   rì   Úq_projÚviewr   Úk_projÚv_projr  r  r(   rd   r  r   Úget_interfacerÀ   Ú_attn_implementationr)   ÚtrainingÚattention_dropoutr$  r?   ÚreshapeÚ
contiguousÚo_proj)rG   r  r  r   r!  rH   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesr  r  Úattention_interfaceÚattn_outputÚattn_weightss   &&&&&,          rJ   r¶   ÚGemma3Attention.forward]  s³  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð /3¯m¯m¨m�D×*Ò*Àð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(rT   )r  r  r  r  r?   )NNN©	rs   rt   ru   rv   r©   r¶   r{   r|   r}   r~   s   @@rJ   r  r  S  s    ù‡ € ÷Ró R÷*)÷ *)ò *)rT   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3DecoderLayeriŠ  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r  r  )rP   rQ   s   "€rJ   rR   ÚGemma3DecoderLayer.__annotate__‹  s    ø€ ÷ 
cñ 
cÑ/ð 
c¹Cñ 
crT   c                óÐ  <€ \         SV `  4        Wn        VP                  V n        W n        \        WR 7      V n        \        V4      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        R# ))rÀ   r  ©rÌ   N)rV   r©   rÀ   rî   r  r  Ú	self_attnr½   ÚmlprÉ   r  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormr  s   &&&€rJ   r©   ÚGemma3DecoderLayer.__init__‹  sª   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ"ŒÜ(°ÔLˆŒÜ˜VÓ$ˆŒÜ,¨T×-=Ñ-=À6×CVÑCVÔWˆÔÜ(5°d×6FÑ6FÈF×L_ÑL_Ô(`ˆÔ%Ü)6°t×7GÑ7GÈV×M`ÑM`Ô)aˆÔ&Ü*7¸×8HÑ8HÈf×NaÑNaÔ*bˆÖ'rT   c                óD  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )	r9   r  r  r   Nr  r!  rH   rM   )r­   r³   Ú
LongTensorr   r   r   rê   ÚFloatTensor)rP   rQ   s   "€rJ   rR   r=  —  s£   ø€ ÷ ñ á—|‘|ðñ #Ÿ\™\ðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ Ñ+Õ,ðñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	UñrT   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRV/VB w  rV P                  V4      pWq,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pWq,           pV# )r  r  r   r  r!  r=   )rB  r@  rC  rD  rA  rE  )	rG   r  r  r   r  r!  rH   ÚresidualÚ_s	   &&&&&&,  rJ   r¶   ÚGemma3DecoderLayer.forward—  s½   € ð !ˆà×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ,ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Õ0ˆàÐrT   )	rÀ   rî   rB  r  rA  rC  rE  rD  r@  )NNNNr9  r~   s   @@rJ   r;  r;  Š  s    ù‡ € ÷
có 
c÷÷ ò rT   r;  c                   ó\   a € ] tR tRt o RtRtR.t]P                  ! 4       R 4       t	Rt
V tR# )ÚGemma3PreTrainedModeli»  Úmodelr;  c                ó4  € \         P                  ! W4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# R VP                  P                  9   d#   \        P
                  ! VP                  4       R# \        V\        4      '       d.   \        P                  ! VP                  VP                  4       R# \        V\        4      '       d´   VP                   F¡  pVP                   pVP"                  V,          R8w  d   \$        VP"                  V,          ,          pV! VP&                  VR7      w  rE\        P(                  ! \+        W R24      V4       \        P(                  ! \+        W R24      V4       K£  	  R# R# )ÚRMSNormr_   rÖ   rØ   rÙ   N)r   Ú_init_weightsrŽ   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightrZ   rs   r¸   rž   Ú	constant_r£   rª   rÒ   rB   rà   r^   r   rÀ   Úcopy_rX   )rG   Úmoduler×   rå   ræ   rL  s   &&    rJ   rS  Ú#Gemma3PreTrainedModel._init_weightsÀ  s  € ä×%Ò% dÔ3Ü�fÔ7×8Ò8Ü�KŠK˜×9Ñ9Ö:à˜&×*Ñ*×3Ñ3Ô3Ü�KŠK˜Ÿ™Ö&Ü˜Ô =×>Ò>Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÖIÜ˜Ô 5×6Ò6Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó 1ñ 7rT   r=   N)ÚimageÚtext)rs   rt   ru   rv   Úbase_model_prefixÚinput_modalitiesÚ_no_split_modulesr­   r  rS  r{   r|   r  s   @rJ   rO  rO  »  s4   ø‡ € ØÐØ(ÐØ-Ð.Ðà
‡]‚]ƒ_ñ^ó ö^rT   rO  c                ór   € V ^8„  d   QhR\         R\        \         \         \         \         .\        3,          /# )r9   r?   rM   )rp   r   rE   )rP   s   "rJ   rR   rR   Ô  s/   € ÷ 
ñ 
´#ð 
¼(ÄCÌÌcÔSVÐCWÔY]ÐC]Õ:^ñ 
rT   c                ó   a € R V 3R llpV# )z9
Enables a bidirectional mask within the sliding window.
c          
      óT   € V ^8„  d   QhR\         R\         R\         R\         R\        /# )r9   Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrM   )rp   rE   )rP   s   "rJ   rR   Ú3_bidirectional_window_overlay.<locals>.__annotate__Ù  s/   € ÷ 4ñ 4œcð 4¬Sð 4¼ð 4Äcð 4Ìdñ 4rT   c                ó,   <€ \        W#,
          4      S8  # )z‹A token can attend to any other token if their absolute distance is within
the (exclusive) sliding window size (distance < sliding_window).)Úabs)rd  re  rf  rg  r?   s   &&&&€rJ   Ú
inner_maskÚ1_bidirectional_window_overlay.<locals>.inner_maskÙ  s   ø€ ô �5•>Ó" ^Ñ3Ð3rT   r=   )r?   rk  s   f rJ   Ú_bidirectional_window_overlayrm  Ô  s   ø€ ÷
4ð 4ð
 ÐrT   c                   ód   a a€ ] tR tRt oRtV3R lV 3R lltR	V3R lR lltV3R ltRtVt	V ;t
# )
ÚGemma3TextModeliá  c                ó    <€ V ^8„  d   QhRS[ /# r¿   rÁ   )rP   rQ   s   "€rJ   rR   ÚGemma3TextModel.__annotate__å  s   ø€ ÷ 
ñ 
Ñ/ñ 
rT   c                óÀ   <€ \         SV `  V4       \        VP                  VP                  V P
                  V P                  P                  R ,          R7      V n        R# )ç      à?)r£   N)rV   r©   rž   rl   rî   r¢   rÀ   Úembed_tokensrÅ   s   &&€rJ   r©   ÚGemma3TextModel.__init__å  sM   ø€ Ü‰Ñ˜Ô ô :Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÕQmô
ˆÖrT   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r9   r±   Nr   r  r!  Úinputs_embedsÚ	use_cacherH   rM   )	r­   rH  r³   r   rI  rE   r   r   r   )rP   rQ   s   "€rJ   rR   rq  í  s�   ø€ ÷ C
ñ C
á×#Ñ# dÕ*ðC
ñ Ÿ™ tÕ+ðC
ñ ×&Ñ&¨Õ-ð	C
ñ
  �ðC
ñ ×(Ñ(¨4Õ/ðC
ñ ˜$•;ðC
ñ Ñ+Õ,ðC
ñ 
!ñC
rT   c                ót  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       gƒ   RV P                  RVRVRVRV/p
V
P                  4       pV P                  P                  '       d)   R	 V
R
&   \        V P                  P                  4      VR
&   R\!        R/ V
B R\#        R/ VB /p	Tp/ p\%        V P                  P&                  4       F  pV P)                  WÃV4      WÞ&   K  	  \+        V P,                  R V P                  P.                   4       FY  w  ppV! V3RW�P                  P&                  V,          ,          RWÐP                  P&                  V,          ,          RVRV/VB pK[  	  V P1                  V4      p\3        VVR7      # )Nú:You must specify exactly one of input_ids or inputs_embeds)rÀ   ©rã   rÀ   rw  r   r!  r  c                  óN   € \         P                  ! R \         P                  R7      # )Trí   )r­   r®   rE   )Úargss   *rJ   Ú<lambda>Ú)Gemma3TextModel.forward.<locals>.<lambda>  s   € ÄÇÂÈTÔY^×YcÑYcÕ@drT   Úor_mask_functionr<   r;   r  )Úlast_hidden_stater!  r=   )Ú
ValueErrorrt  r   rÀ   Úget_seq_lengthr­   rð   rþ   rã   Ú	unsqueezerŽ   rN   Úcopyr>   rm  r?   r   r   rß   rB   Ú
rotary_embÚ	enumerateÚlayersrD   Únormr   )rG   r±   r   r  r!  rw  rx  rH   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsr  r  r×   rI   Údecoder_layers   &&&&&&&,         rJ   r¶   ÚGemma3TextModel.forwardí  s>  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð #.×"2Ñ"2Ó"4Ðà�{‰{×6×6Ð6Ù2d�Ð.Ñ/Ü:WÐX\×XcÑXc×XrÑXrÓ:sÐ#Ð$6Ñ7ð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%]ÐI\Ñ%]ð#Ðð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAˆ}Ù)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8¿¹×8OÑ8OÐPQÕ8RÕ$Sðð *ð	ð
 !0ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
rT   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r¿   rÁ   )rP   rQ   s   "€rJ   rR   rq  á  ó   ø‡ ‚ ÙÑò rT   )rt  ©r]  )NNNNNN)rs   rt   ru   rv   r_  r©   r¶   rz   r{   r|   r}   r~   s   @@rJ   ro  ro  á  s(   ù‡ € à Ð÷
ó 
÷C
ò C
÷ … rT   ro  c                   óJ   a a€ ] tR tRt oV3R lV 3R lltV3R ltRtVtV ;t# )ÚGemma3ForCausalLMi3  c                ó    <€ V ^8„  d   QhRS[ /# r¿   rÁ   )rP   rQ   s   "€rJ   rR   ÚGemma3ForCausalLM.__annotate__6  s   ø€ ÷ -ñ -Ñ/ñ -rT   c                óF   <€ \         SV `  V4       \        V4      V n        R # rµ   )rV   r©   ro  rP  rÅ   s   &&€rJ   r©   ÚGemma3ForCausalLM.__init__6  s   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆŽ
rT   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r¿   rÁ   )rP   rQ   s   "€rJ   rR   r–  3  r‘  rT   )rP  )	rs   rt   ru   rv   r©   rz   r{   r|   r}   r~   s   @@rJ   r”  r”  3  s   ù‡ € ÷-ó -÷ … rT   r”  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )rT  i;  c                ó    <€ V ^8„  d   QhRS[ /# r¿   ©r€   )rP   rQ   s   "€rJ   rR   Ú&Gemma3MultiModalProjector.__annotate__<  s   ø€ ÷ \ñ \™|ñ \rT   c                ó®  <€ \         SV `  4        \        P                  ! \        P
                  ! VP                  P                  VP                  P                  4      4      V n	        \        VP                  P                  VP                  P                  R 7      V n        \        VP                  P                  VP                  P                  ,          4      V n        \        VP"                  R,          4      V n        V P                   V P$                  ,          V n        \        P(                  ! V P&                  V P&                  R7      V n        R# )r?  rs  )Úkernel_sizeÚstrideN)rV   r©   r«   Ú	Parameterr­   Úzerosr‰   rî   rˆ   rW  rÉ   Úlayer_norm_epsÚmm_soft_emb_normrp   Ú
image_sizeÚ
patch_sizeÚpatches_per_imager‘   Útokens_per_siderŸ  Ú	AvgPool2dÚavg_poolrÅ   s   &&€rJ   r©   Ú"Gemma3MultiModalProjector.__init__<  sì   ø€ Ü‰ÑÔä*,¯,ª,Ü�KŠK˜×,Ñ,×8Ñ8¸&×:LÑ:L×:XÑ:XÓYó+
ˆÔ'ô !.Ø× Ñ ×,Ñ,°&×2FÑ2F×2UÑ2Uô!
ˆÔô "% V×%9Ñ%9×%DÑ%DÈ×H\ÑH\×HgÑHgÕ%gÓ!hˆÔÜ" 6×#=Ñ#=¸sÕ#BÓCˆÔØ×1Ñ1°T×5IÑ5IÕIˆÔÜŸš°×1AÑ1AÈ$×JZÑJZÔ[ˆŽrT   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r9   Úvision_outputsr²   )rP   rQ   s   "€rJ   rR   r�  L  s   ø€ ÷ @ñ @¡e§l¡lñ @rT   c                óž  € VP                   w  r#pVP                  ^^4      pVP                  W$V P                  V P                  4      pVP	                  4       pV P                  V4      pVP                  ^4      pVP                  ^^4      pV P                  V4      p\        P                  ! WpP                  4      pVP                  V4      # )é   )rþ   r   r-  r§  r.  rª  Úflattenr¤  r­   ÚmatmulrW  Útype_as)	rG   r­  Ú
batch_sizerL  rî   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	   &&       rJ   r¶   Ú!Gemma3MultiModalProjector.forwardL  sÇ   € Ø%3×%9Ñ%9Ñ"ˆ
�{à"0×":Ñ":¸1¸aÓ"@ÐØ"9×"AÑ"AØ T×%;Ñ%;¸T×=SÑ=Só#
Ðð #:×"DÑ"DÓ"FÐà $§¡Ð.EÓ FÐØ 5× =Ñ =¸aÓ @ÐØ 5× ?Ñ ?ÀÀ1Ó EÐà $× 5Ñ 5Ð6KÓ LÐä#(§<¢<Ð0E×GfÑGfÓ#gÐ Ø'×/Ñ/°Ó?Ð?rT   )rª  rŸ  rW  r¤  r§  r¨  r9  r~   s   @@rJ   rT  rT  ;  s#   ù‡ € ÷\ó \÷ @÷ @ð @rT   rT  c                ó†   € V ^8„  d   QhR\         P                  R\         P                  R,          R\         P                  /# )r9   Útoken_type_idsrã   NrM   )r­   r³   rã   )rP   s   "rJ   rR   rR   _  s6   € ÷ ñ ´E·L±Lð Ì%Ï,É,ÐY]ÕJ]ð Ôin×iuÑiuñ rT   c                 ó$  € V ^8H  P                  VR7      p\        P                  P                  VR^ R7      RRR13,          pW#( ,          p\        P
                  ! VP                  4       ^R7      ^,
          p\        P                  ! W%R4      pV# )r¯  r{  )Úvaluerû   Nrú   )r¯  r   rü   )r·   r«   Ú
functionalÚpadr­   Úcumsumrp   Úwhere)rº  rã   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss   &&     rJ   Úget_block_sequence_ids_for_maskrÆ  _  s…   € ð  !Ñ#×'Ñ'¨vÐ'Ó6€HÜŸ™×)Ñ)¨(°FÀ!Ð)ÓDÀQÈÈÈÀVÕLÐØÐ!3Õ3€OÜ—’˜_×0Ñ0Ó2¸Ô:¸QÕ>€IÜŸš X¸"Ó=ÐØÐrT   c                óæ   € V ^8„  d   QhR\         R\        P                  R\        P                  R,          R\        R,          R\        P                  R,          R\        P                  R\        /# )	r9   rÀ   rw  r   Nr!  r  rÅ  rM   )r	   r­   r³   r   rN   )rP   s   "rJ   rR   rR   j  sl   € ÷ 9ñ 9Üð9ä—<‘<ð9ô —L‘L 4Õ'ð9ô ˜T•\ð	9ô
 —,‘, Õ%ð9ô Ÿ™ð9ô 
ñ9rT   c           
     ó~  € RV RVRVRVRV/p\        R/ VBRV/B p\        VRR4      '       d.   R	VP                  9   d   VP                  P                  R	4      pM^ p\	        R/ VBR
V/B w  p	  r«r¬V	'       d   TpM\        WRW¼4      p\        R/ VBR\        V4      R\        V P                  4      R
V/B pRVRV/# )z×Create full_attention and sliding_attention masks with correct composition.

For global (full attention) layers:  OR(causal, blockwise)
For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
rÀ   rw  r   r!  r  rÅ  r  NTr  r€  Úand_mask_functionr<   r;   r=   )	r   rX   r  Úindexr
   r   r   r   r?   )rÀ   rw  r   r!  r  rÅ  rŒ  Ú	full_maskÚsliding_layer_idxÚ
early_exitrL  Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks   &&&&&&         rJ   Úcreate_masks_for_vision_modelrÒ  j  s  € ð 	�&Ø˜Ø˜.Ø˜?Ø˜ð€Kô #ÑX [ÑXÐEWÒX€Iô ˆ ¨d×3Ò3¸À×@ZÑ@ZÔ8ZØ+×6Ñ6×<Ñ<¸TÓBÑàÐô 4Nñ 4Ø
ñ4à#ò4Ñ0€J��1�a A÷ Ø$6Ñ!ä$@Ø°	ó%
Ð!ô &ñ Ø
ñä*Ð+DÓEðô 1°×1FÑ1FÓGðð $ò	€Lð 	˜)Ø˜\ðð rT   c                   ó    a a€ ] tR tRt oRtV3R lV 3R llt]]! RR7      V3R lR l4       4       t]]RV3R	 lR
 ll4       4       t	Rt
VtV ;t# )ÚGemma3Modeli¦  Fc                ó    <€ V ^8„  d   QhRS[ /# r¿   rœ  )rP   rQ   s   "€rJ   rR   ÚGemma3Model.__annotate__ª  s   ø€ ÷ #ñ #™|ñ #rT   c                ó*   <€ \         SV `  V4       V =R # rµ   )rV   r©   Útext_config_dtyperÅ   s   &&€rJ   r©   ÚGemma3Model.__init__ª  s   ø€ Ü‰Ñ˜Ô ØÒ"rT   zOProjects the last hidden state from the vision model into language model space.©Úcustom_introc                ó`   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[,          /# )r9   Úpixel_valuesrH   rM   )r­   rI  r   r   rê   r   )rP   rQ   s   "€rJ   rR   rÖ  °  s7   ø€ ÷ ñ Ù!×-Ñ-ðÙ9?Ñ@RÕ9Sðá	Ñ+Õ	+ñrT   c                óv   € V P                   ! RR VRR/VB pVP                  pV P                  V4      Vn        V# )rÝ  Úreturn_dictTr=   )Úvision_towerr�  Úmulti_modal_projectorÚpooler_output)rG   rÝ  rH   r­  r�  s   &&,  rJ   Úget_image_featuresÚGemma3Model.get_image_features®  sI   € ð
 ×*Ò*Ña¸ÐaÐRVÐaÐZ`ÑaˆØ*×<Ñ<ÐØ'+×'AÑ'AÐBSÓ'TˆÔ$àÐrT   c                ó†  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[S[,          RS[S[	,          /# )r9   r±   NrÝ  r   r  r!  rº  rw  Úlabelsrx  Ú	lm_kwargsrM   )
r­   rH  rI  r³   r   rE   r   r   rê   r˜   )rP   rQ   s   "€rJ   rR   rÖ  »  så   ø€ ÷ G
ñ G
á×#Ñ# dÕ*ðG
ñ ×'Ñ'¨$Õ.ðG
ñ Ÿ™ tÕ+ð	G
ñ
 ×&Ñ&¨Õ-ðG
ñ  �ðG
ñ ×(Ñ(¨4Õ/ðG
ñ ×(Ñ(¨4Õ/ðG
ñ × Ñ  4Õ'ðG
ñ ˜$•;ðG
ñ Ñ.Õ/ðG
ñ 
Ñ*Õ	*ñG
rT   c
                ó˜  € VR J VR J,          '       d   \        R4      hVeS   V P                  P                  V P                  8¼  d.   WP                  P                  8H  pVP	                  4       p^ WË&   MTpVf   V P                  4       ! V4      pVeh   V P                  VRR7      P                  pVP                  VP                  VP                  4      pV P                  WVR7      pVP                  W½4      p\        T;p\        4      '       gX   RV P                  P                  4       RVRVRVR	V/pVe%   \!        WgP                  R
7      p\#        RRV/VB pM\%        R/ VB pV P&                  ! RRVR	VRVRVRV	RR/V
B p\)        VP*                  VP,                  VP.                  VP0                  Ve   XR7      # R R7      # )Nrz  T)rß  )rw  Úimage_featuresrÀ   rw  r   r!  r  r{  rÅ  rx  rß  )r�  r!  r  Ú
attentionsÚimage_hidden_statesr=   )r‚  rÀ   r‚   rl   râ   Úget_input_embeddingsrã  râ  r·   rã   r¹   Úget_placeholder_maskÚmasked_scatterrŽ   rN   Úget_text_configrÆ  rÒ  r   Úlanguage_modelr˜   r�  r!  r  rê  )rG   r±   rÝ  r   r  r!  rº  rw  ræ  rx  rç  Úspecial_image_maskÚllm_input_idsré  r‹  rŒ  rÅ  Úoutputss   &&&&&&&&&&,       rJ   r¶   ÚGemma3Model.forward¹  s  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZð Ò  T§[¡[×%?Ñ%?À4Ç?Á?Ô%RØ!*¯k©k×.HÑ.HÑ!HÐØ%ŸO™OÓ-ˆMØ01ˆMÒ-à%ˆMàÒ Ø ×5Ñ5Ô7¸ÓFˆMð Ò#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:ØÀ~ð ";ó "Ðð *×8Ñ8Ð9KÓ\ˆMô °Ð?Ð-Ä×FÒFà˜$Ÿ+™+×5Ñ5Ó7Ø Ø  .Ø! ?Ø ðˆKð Ò)Ü%DÀ^×\pÑ\pÔ%qÐ"Ü&Cñ 'Ø'9ð'à!ñ'Ñ#ô
 '@Ñ&NÀ+Ñ&NÐ#à×%Ò%ñ 
Ø.ð
à%ð
ð ,ð
ð (ð	
ð
  ð
ð ð
ð ñ
ˆô )Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ò2J ô
ð 	
ð
 QUô
ð 	
rT   r=   )	NNNNNNNNN)rs   rt   ru   rv   Úaccepts_loss_kwargsr©   r   r   rã  r¶   r{   r|   r}   r~   s   @@rJ   rÔ  rÔ  ¦  sb   ù‡ € àÐ÷#ó #ð ÙÐ!rÔs÷ó tó ðð Ø÷G
ñ G
ó ó ÷G
ð G
rT   rÔ  c                   óv   a a€ ] tR tRt o]]RV3R lR ll4       4       tR	V 3R lltR
V3R lR lltRt	Vt
V ;t# )ÚGemma3ForConditionalGenerationi  c                ó°  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[S[ P                  ,          RS[S[,          RS[	S[
,          /# )r9   r±   NrÝ  r   r  r!  rº  rw  ræ  rx  Úlogits_to_keeprç  rM   )r­   rH  rI  r³   r   rE   rp   r   r   rê   rœ   )rP   rQ   s   "€rJ   rR   Ú+Gemma3ForConditionalGeneration.__annotate__  sù   ø€ ÷ [
ñ [
á×#Ñ# dÕ*ð[
ñ ×'Ñ'¨$Õ.ð[
ñ Ÿ™ tÕ+ð	[
ñ
 ×&Ñ&¨Õ-ð[
ñ  �ð[
ñ ×(Ñ(¨4Õ/ð[
ñ ×(Ñ(¨4Õ/ð[
ñ × Ñ  4Õ'ð[
ñ ˜$•;ð[
ñ ™eŸl™lÕ*ð[
ñ Ñ.Õ/ð[
ñ 
Ñ-Õ	-ñ[
rT   c                óÊ  € V P                   ! RRVRVRVRVRVRVRVRV	R	VR
R/
VB pV^ ,          p\        V
\        4      '       d   \        V
) R4      MT
pV P	                  VRVR3,          4      pRpVe7   V P
                  ! RRVR	VRV P                  P                  P                  /VB p\        VVVP                  VP                  VP                  VP                  R7      # )aÉ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

>>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
>>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

>>> messages = [
...     {
...         "role": "system",
...         "content": [
...             {"type": "text", "text": "You are a helpful assistant."}
...         ]
...     },
...     {
...         "role": "user", "content": [
...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
...             {"type": "text", "text": "Where is the cat standing?"},
...         ]
...     },
... ]

>>> inputs = processor.apply_chat_template(
...     messages,
...     tokenize=True,
...     return_dict=True,
...     return_tensors="pt",
...     add_generation_prompt=True
... )
>>> # Generate
>>> generate_ids = model.generate(**inputs)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
```
r±   rÝ  rº  r   r  r!  rw  rx  ræ  rß  TNrû   Úlogitsrl   )Úlossrü  r!  r  rê  rë  r=   )rP  rŽ   rp   ÚsliceÚlm_headÚloss_functionrÀ   rˆ   rl   rœ   r!  r  rê  rë  )rG   r±   rÝ  r   r  r!  rº  rw  ræ  rx  rù  rç  ró  r  Úslice_indicesrü  rý  s   &&&&&&&&&&&,     rJ   r¶   Ú&Gemma3ForConditionalGeneration.forward  sD  € ðz —*’*ñ 
Øð
à%ð
ð *ð
ð *ð	
ð
 &ð
ð ,ð
ð (ð
ð  ð
ð ð
ð ð
ð ñ
ˆð   �
ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%ñ ØðØ&,ðØ9=¿¹×9PÑ9P×9[Ñ9[ðØ_hñˆDô ,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
rT   c                ó\   <€ \         SV `  ! V3R VRV/VB pV'       g   V'       d   RVR&   V# )rx  Úis_first_iterationNrº  )rV   Úprepare_inputs_for_generation)rG   r±   rx  r  rH   Úmodel_inputsrZ   s   &&&&, €rJ   r  Ú<Gemma3ForConditionalGeneration.prepare_inputs_for_generatione  sE   ø€ Ü‘wÒ<Øñ
Ø!*ð
Ø?Qð
ØU[ñ
ˆ÷ #¯)à-1ˆLÐ)Ñ*àÐrT   c                óà   <€ V ^8„  d   QhRS[ RS[P                  RS[P                  R,          RS[R,          RS[P                  R,          RS[P                  R,          RS[R,          R	S[/# )
r9   rÀ   rw  r   Nr!  r  rº  r  rM   )r	   r­   r³   r   rE   rN   )rP   rQ   s   "€rJ   rR   rú  o  s€   ø€ ÷ 8ñ 8Ù ð8á—|‘|ð8ñ Ÿ™ tÕ+ð8ñ  �ð	8ñ
 —l‘l TÕ)ð8ñ Ÿ™ tÕ+ð8ñ ! 4�Kð8ñ 
ñ8rT   c           
     óš   € R V P                  4       RVRVRVRV/pVe$   \        WQP                  R7      p	\        RRV	/VB # \	        R/ VB # )rÀ   rw  r   r!  r  r{  rÅ  r=   )rï  rÆ  rã   rÒ  r   )
rÀ   rw  r   r!  r  rº  r  rH   rŒ  rÅ  s
   &&&&&&&,  rJ   r   Ú8Gemma3ForConditionalGeneration.create_masks_for_generateo  ss   € ð �f×,Ñ,Ó.Ø˜]Ø˜nØ˜Ø˜Lð
ˆð Ò%Ü!@À×XlÑXlÔ!mÐÜ0ñ Ø#5ðàñð ô
 )Ñ7¨;Ñ7Ð7rT   r=   )
NNNNNNNNNr   )TF)NF)rs   rt   ru   rv   r   r   r¶   r  r   r{   r|   r}   r~   s   @@rJ   r÷  r÷    s<   ù‡ € ØØ÷[
ñ [
ó ó ð[
÷z÷8÷ 8ò 8rT   r÷  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.rÚ  c                   ó0   a € ] tR tRt o RtV 3R ltRtV tR# )Ú#Gemma3TextForSequenceClassificationi‹  c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r¿   rÁ   )rP   rQ   s   "€rJ   rR   Ú0Gemma3TextForSequenceClassification.__annotate__‹  s   ø‡ ‚ ñ Ñò rT   r=   Nr’  )rs   rt   ru   rv   r_  rz   r{   r|   r  s   @rJ   r  r  ‹  s   ø‡ € ð !Ð÷ ƒ rT   r  c                   óB   a a€ ] tR tRt oRV3R lV 3R llltRtVtV ;t# )ÚGemma3ForSequenceClassificationi•  c                ób  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[,          RS[/
# )r9   r±   NrÝ  r   r  r!  rº  rw  ræ  rH   rM   )r­   rH  rI  r³   r   r   r   r   )rP   rQ   s   "€rJ   rR   Ú,Gemma3ForSequenceClassification.__annotate__–  sÄ   ø€ ÷ 
ñ 
á×#Ñ# dÕ*ð
ñ ×'Ñ'¨$Õ.ð
ñ Ÿ™ tÕ+ð	
ñ
 ×&Ñ&¨Õ-ð
ñ  �ð
ñ ×(Ñ(¨4Õ/ð
ñ ×(Ñ(¨4Õ/ð
ñ × Ñ  4Õ'ð
ñ Ñ+Õ,ð
ñ 
*ñ
rT   c	                óD   <€ \         S
V `  ! RR VRVRVRVRVRVRVRV/V	B # )	r±   r   r  r!  rw  rÝ  rº  ræ  r=   )rV   r¶   )rG   r±   rÝ  r   r  r!  rº  rw  ræ  rH   rZ   s   &&&&&&&&&,€rJ   r¶   Ú'Gemma3ForSequenceClassification.forward–  sl   ø€ ô ‰wŠñ 

Øð

à)ð

ð &ð

ð ,ð	

ð
 (ð

ð &ð

ð *ð

ð ð

ð ñ

ð 
	
rT   r=   )NNNNNNNN)rs   rt   ru   rv   r¶   r{   r|   r}   r~   s   @@rJ   r  r  •  s   ù‡ € ÷
÷ 
õ 
rT   r  rµ   )	r€   r1   rO  ro  r”  r÷  rÔ  r  r  )^Úcollections.abcr   Útypingr   r­   Útorch.nnr«   Úhuggingface_hub.dataclassesr   Ú r   rU  Úcache_utilsr   r   Úconfiguration_utilsr	   Úmasking_utilsr
   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úgemma2.configuration_gemma2r    Úgemma2.modeling_gemma2r!   r"   r#   r$   r%   r&   r'   r(   r)   Úpaligemma.modeling_paligemmar*   r+   r,   r-   Úsiglipr.   Ú
get_loggerrs   rŒ   r1   r€   r˜   rœ   Ú	Embeddingrž   r½   rÉ   rÒ   r  r;  ÚGEMMA3_START_DOCSTRINGrO  rm  ro  r”  rÛ   rT  rÆ  rÒ  rÔ  r÷  r  r  Ú__all__r=   rT   rJ   Ú<module>r,     s  ðõ %Ý ã Ý Ý .å &ß .Ý 3÷÷ ñ ÷ \ß uÑ u÷÷ GÝ &ß RÓ RÝ +Ý 6÷
÷ 
õ 
÷ó õ (ð 
×	Ò	˜HÓ	%€ñ Ð1Ô2Øô`�|Ð%5ó `ó ó 3ð`ñF Ð1Ô2Øô?(Ð#ó ?(ó ó 3ð?(ôD	Ð <ô 	ô	Ð#Bô 	ôS B§L¡Lô Sô!�	ô !ô
+�Mô +ô
B<Ð1ô B<ôL4)�oô 4)ôn+Ð3ô +ð\ Ð ô^Ð1ô ^õ2
ôO
�kô O
ôd-Ð)ô -ô!@ §	¡	ô !@÷Hõ9ôx\
�.ô \
ô~C8Ð%Fô C8ñL ð^ôô
!Ð*JÐLaó !óð
!ô

Ð&FÐH]ô 
ò4
‚rT   