+
    œ°³jÊ ã                   ó¸  € ^ RI t ^ RIHt ^ RIHtHt ^ RIHt ^ RIH	t	 ^ RI
t
^ RIHt ^ RIHu Ht ^ RIHt ^RIHt ^RIHt ^R	IHtHt ^R
IHt ^RIHtHt ^RIH t H!t! ^RI"H#t# ^RI$H%t%H&t& ^RI'H(t( ^RI)H*t*H+t+H,t,H-t-H.t.H/t/ ^RI0H1t1 ^RI2H3t3 ^RI4H5t5 ^RI6H7t7H8t8H9t9H:t: ^RI;H<t< ^RI=H>t>H?t?H@t@HAtAHBtB ^RICHDtDHEtEHFtFHGtG ^RIHHItI ]-! 4       '       d   ^ RIJHKtK ].P˜                  ! ]M4      tN]+! RR7      ] ! R R]<4      4       4       tO]+! RR7      ] ! R R]4      4       4       tP]+! RR7      ] ! R  R!]I4      4       4       tQ]+! RR7      ] ! R" R#]4      4       4       tR]+] ! R$ R%]!4      4       4       tS ! R& R']G4      tT ! R( R)]D4      tU ! R* R+]P¬                  4      tW ! R, R-]P¬                  4      tX ! R. R/]P¬                  4      tY ! R0 R1]P¬                  4      tZ ! R2 R3]P¬                  4      t[ ! R4 R5]P¬                  4      t\ ! R6 R7]P¬                  4      t] ! R8 R9]P¬                  4      t^ ! R: R;]P¬                  4      t_ ! R< R=]P¬                  4      t` ! R> R?]B4      ta ! R@ RA]P¬                  4      tb ! RB RC]74      tc ! RD RE]P¬                  4      tdRaRF RG llte ! RH RI]P¬                  4      tf ! RJ RK]>4      tg ! RL RM]84      th ! RN RO]h4      ti ! RP RQ]@4      tj]+! RRRS7       ! RT RU]A4      4       tk]+! RVRS7       ! RW RX]?4      4       tl ! RY RZ]P¬                  4      tm]+! R[RS7       ! R\ R]]F4      4       tn]+! R^RS7       ! R_ R`]E4      4       to. RbOtpR# )cé    N)ÚUserDict)ÚCallableÚSequence)Ú	dataclass)ÚAny)Ústrict)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_accelerate_availableÚloggingÚtorch_compilable_check)Úmerge_with_config_defaults)Úcapture_outputs)Ú	AutoModel)Ú	Gemma2MLPÚGemma2PreTrainedModelÚeager_attention_forwardÚrotate_half)ÚGemma3TextConfig)ÚGemma3DecoderLayerÚGemma3ForCausalLMÚGemma3RotaryEmbeddingÚGemma3TextModelÚGemma3TextScaledWordEmbedding)ÚPaliGemmaCausalLMOutputWithPastÚ!PaliGemmaForConditionalGenerationÚPaliGemmaModelÚPaligemmaModelOutputWithPast)ÚTimmWrapperConfig)Úadd_hook_to_modulezgoogle/gemma-3n-E4B)Ú
checkpointc                   óê   a € ] tR t^Kt o RtRtRRRRRRRRR	RR
RRRRRRRRR/
tRRRR/tRtRt	Rt
RtRt^#t^tRtRtRtRt^ tRtRt^t^t^@tRt]! 4       t]! 4       t]! 4       tR tR  tV 3R! lt R"t!V t"R# )#ÚGemma3nTextConfigaÈ  
vocab_size_per_layer_input (`int`, *optional*, defaults to 262144):
    Vocabulary size of the per-layer text embeddings that augment the standard embeddings.
hidden_size_per_layer_input (`int`, *optional*, defaults to 256):
    Dimension of the hidden representations for per-layer embeddings.
altup_active_idx (`int`, *optional*, defaults to 0):
    The index of the prediction from which AltUp will compute additional predictions or correct the active prediction.
altup_coef_clip (`float`, *optional*, defaults to 120.0):
    The maximum amplitude of an AltUp prediction or correction coefficient weight.
altup_correct_scale (`bool`, *optional*, defaults to `True`):
    If True, apply the `AltUp.correct_output_scale` to the corrected prediction at `altup_active_idx`.
altup_num_inputs (`int`, *optional*, defaults to 4):
    The number of predictions that AltUp should make given the input sequence.
num_kv_shared_layers (`int`, *optional*, defaults to 15):
    The number of layers that share KV cache values. During the forward pass, the last `num_kv_shared_layers`
    layers in the model "share" the KV values in that each local and global layer in this range uses the KV
    cache values computed for the last local or global layer, respectively, before entering this range. The
    value should be a multiple of the attention pattern size (see `layer_types` parameter).
laurel_rank (`int`, *optional*, defaults to 64):
    The intermediate size for the linear projections in the Learned Augmented Residual Layer.
activation_sparsity_pattern (`Sequence[float]`, *optional*):
    The sparsity factor used to extract the top-k activations for a given layer. The provided Sequence must
    explicitly provide a sparsity value for each layer in the model. By default, the first 10 layers are
    sparse with a sparsity factor of 0.95 and the rest are dense.

```python
>>> from transformers import Gemma3nTextModel, Gemma3nTextConfig

>>> # Initializing a Gemma3nText gemma3n_text-E4B style configuration
>>> configuration = Gemma3nTextConfig()

>>> # Initializing a model from the gemma3n_text-E4B style configuration
>>> model = Gemma3nTextModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```
Úgemma3n_textzlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.self_attn.v_normzlayers.*.self_attn.o_projÚrowwisezlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚglobalg    €„.AÚlocalç     ˆÃ@i  é   é   é   i @  i €  i   Ng      >@g      ^@Tc                óF  € \        V P                  \        4      '       dB   \        V P                  4      ;q P                  8w  d   \        R V P                   RV R24      h\        V P                  \        4      '       g$   V P                  .V P                  ,          V n        V P                  fA   \        V P                  4       Uu. uF  q3^,           ^,          ^ 8X  d   RMRNK  	  upV n        V P                  fE   V P                  ^
8”  d   ^
M^ pR.V,          R.V P                  V,
          ,          ,           V n        \        V P                  4      ;qPP                  8w  d   \        RV P                   RV R24      h\        P                  ! R	/ VB  R# u upi )
zjintermediate_size must have an explicit intermediate size for every layer or one for all layers. Expected z values but got Ú.NÚfull_attentionÚsliding_attentiongffffffî?ç        zeactivation_sparsity_pattern must have an explicit activation sparsity value for every layer.Expected © )Ú
isinstanceÚintermediate_sizer   ÚlenÚnum_hidden_layersÚ
ValueErrorÚlayer_typesÚrangeÚactivation_sparsity_patternr   Ú__post_init__)ÚselfÚkwargsÚintsize_lenÚiÚnum_sparse_layersÚlen_asps   &,    Úl/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gemma3n/modular_gemma3n.pyrJ   ÚGemma3nTextConfig.__post_init__š   sŽ  € ä�t×-Ñ-¬x×8Ò8Ü # D×$:Ñ$:Ó ;Ð;�×@VÑ@VÔVäðØ ×2Ñ2Ð3Ð3CÀKÀ=ÐPQðSóð ô ˜D×2Ñ2´H×=Ò=Ø&*×&<Ñ&<Ð%=À×@VÑ@VÕ%VˆDÔ"à×ÑÒ#äW\Ð]a×]sÑ]sÔWtó ÙWtÐRS¨¥U¨a¥K°1Ô$4Ñ Ð:MÒMÑWtñ ˆDÔð ×+Ñ+Ò3Ø&*×&<Ñ&<¸rÔ&A¡ÀqÐØ04¨vÐ8IÕ/IÈSÈEØ×&Ñ&Ð):Õ:õMõ 0ˆDÔ,ô ˜4×;Ñ;Ó<Ð<ˆG×AWÑAWÔWÜðØ ×2Ñ2Ð3Ð3CÀGÀ9ÈAðOóð ô
 	×&Ò&Ñ0¨Ô0ùò! s   Ã
!Fc                ó´  € VP                  R R4      pRRR/RRR//pV P                  e   V P                  MTV n        Ve#   V P                  R,          P                  V4       V P                  P                  R4      f   RR/V P                  R&   V P                  R,          P	                  RVP                  RV P
                  R,          4      4       V P                  P                  R4      f   RR/V P                  R&   V P                  R,          P	                  RVP                  RV P
                  R	,          4      4       V P                  4        V# )
Úrope_scalingNr?   Ú	rope_typeÚdefaultr>   Ú
rope_thetar6   Úrope_local_base_freqr7   )ÚpopÚrope_parametersÚupdateÚgetÚ
setdefaultÚdefault_thetaÚstandardize_rope_params)rK   rL   rT   Údefault_rope_paramss   &,  rQ   Úconvert_rope_params_to_dictÚ-Gemma3nTextConfig.convert_rope_params_to_dict¹   sM  € Ø—z‘z .°$Ó7ˆð
   +¨yÐ!9Ø˜{¨IÐ6ð
Ðð 8<×7KÑ7KÒ7W˜t×3Ò3Ð]pˆÔØÒ#Ø× Ñ Ð!1Õ2×9Ñ9¸,ÔGð ×Ñ×#Ñ#Ð$4Ó5Ò=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Õ.×9Ñ9Ø˜&Ÿ*™* \°4×3EÑ3EÀhÕ3OÓPô	
ð ×Ñ×#Ñ#Ð$7Ó8Ò@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Õ1×<Ñ<Ø˜&Ÿ*™*Ð%;¸T×=OÑ=OÐPWÕ=XÓYô	
ð
 	×$Ñ$Ô&Øˆó    c                ó^  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ S[S[ ,          ,          ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R	&   S[S[,          R
,          ;R&   S[;R&   S[ ;R&   S[;R&   S[;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[S[S[,          ,          R
,          ;R&   # )é   Ú
vocab_sizeÚvocab_size_per_layer_inputÚhidden_sizeÚhidden_size_per_layer_inputrC   rE   Únum_key_value_headsÚmax_position_embeddingsÚsliding_windowNrG   Úfinal_logit_softcappingÚaltup_active_idxÚaltup_coef_clipÚaltup_correct_scaleÚaltup_num_inputsÚnum_kv_shared_layersÚlaurel_rankrI   )ÚintÚlistÚstrÚfloatÚbool)ÚformatÚ__classdict__s   "€rQ   Ú__annotate__ÚGemma3nTextConfig.__annotate__K   s2  ø‡ ‚ ñr Ññs ñt !$Ñ-ñu ñv Ññw ñx "%Ñ*ñy ñz ™T¡#�Y•Ñ/ñ{ ñ| Ññ} ñ~ Ñ ñ ñ@ !Ñ)ñA ñB ÑñC ñD ‘c•˜TÕ!Ñ(ñE ñF #Ñ)ñG ñH ÑñI ñJ Ñ"ñK ñL Ñ$ñM ñN ÑñO ñP Ñ"ñQ ñR ÑñS ñT "'©©e­Õ!4°tÕ!;ÑBòU rc   )rI   rC   rG   rZ   )#Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚbase_model_tp_planr^   rf   rg   rh   ri   rC   rE   rj   rk   rl   rG   rm   rn   ro   rp   rq   rr   rs   rI   ÚAttributeErrorÚattn_logit_softcappingÚuse_bidirectional_attentionÚquery_pre_attn_scalarrJ   ra   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©rz   s   @rQ   r1   r1   K   sò   ø‡ € ñ%ðN  €Jà# YØ# YØ# YØ#Ð%EØ#Ð%EØ#Ð%EØ# YØ  )Ø 	Ø  )ðÐð ˜{¨G°XÐ>€Mà€JØ&-ÐØ€KØ'*ÐØ)/ÐØÐØ ÐØ#)ÐØ€NØ$(€KØ%)ÐØÐØ"€OØ $ÐØÐØ "ÐØ€KØ>BÐÙ+Ó-ÐÙ"0Ó"2ÐÙ*Ó,Ðò1ò>÷] ƒ rc   r1   c                   ó€   a € ] tR t^×t o RtRt^€tRt^€tRt	Rt
Rt^t^t^ tRt^t^t^t^tRtRtRtRtRtV 3R	 ltR
tV tR# )ÚGemma3nAudioConfiga“  
vocab_offset (`int`, *optional*, defaults to 262272):
    Offset between the tokenizer vocab index for the token ids embedded by `Gemma3nMultimodalEmbedder` and the
    0-indexed `Gemma3nMultimodalEmbedder.embedding` table.
input_feat_size (`int`, *optional*, defaults to 128):
    The number of channels in each mel-spectrogram frame.
gradient_clipping (`float`, *optional*, defaults to 10000000000.0):
    Clipping value used to stabilize extremely large gradient values.
conf_attention_chunk_size (`int`, *optional*, defaults to 12):
    The sub-sequence size for local attention processing inside the Conformer ("conf") section of the
    Universal Speech Model.
conf_attention_context_left (`int`, *optional*, defaults to 13):
    The left context size of the local attention inside the Conformer ("conf") section of the
    Universal Speech Model.
conf_attention_context_right (`int`, *optional*, defaults to 0):
    The right context size of the local attention inside the Conformer ("conf") section of the
    Universal Speech Model.
conf_attention_logit_cap (`float`, *optional*, defaults to 50.0):
    Logit cap applied during local attention inside the Conformer ("conf") section of the
    Universal Speech Model.
conf_num_attention_heads (`int`, *optional*, defaults to 8):
    The number of attention heads in local attention inside the Conformer ("conf") section of the
    Universal Speech Model.
conf_num_hidden_layers (`int`, *optional*, defaults to 12):
    The number of layers that use local attention inside the Conformer ("conf") section of the
    Universal Speech Model.
conf_conv_kernel_size (`int`, *optional*, defaults to 5):
    Convolution kernel size for the conformer block inside the Conformer ("conf") section of the
    Universal Speech Model.
conf_reduction_factor (`int`, *optional*, defaults to 4):
    Reduction factor used in the conformer block inside the Conformer ("conf") section of the
    Universal Speech Model.
conf_residual_weight (`float`, *optional*, defaults to 0.5):
    Residual connection weight inside the Conformer ("conf") section of the
    Universal Speech Model.
sscp_conv_channel_size (`tuple(int, int)`, *optional*, defaults to `(128, 32)`):
    The channel sizes for the first and second convolutional layers in the Sub-sample Convolution Projection
    ("sscp") section of the Universal Speech Model.
sscp_conv_group_norm_eps (`float`, *optional*, defaults to 0.001):
    Epsilon used in group normalization in the subsample convolution projection in the Sub-sample Convolution
    Projection ("sscp") section of the Universal Speech Model.
sscp_conv_kernel_size (`tuple(tuple(int, int), tuple(int, int))`, *optional*, defaults to `((3, 3), (3, 3))`):
    Kernel sizes of the two convolutional layers in the subsample convolution projection  in the Sub-sample
    Convolution Projection ("sscp") section of the Universal Speech Model. The kernel sizes are specified as a
    tuple of height and width for each layer, where the height corresponds to the time dimension and the width
    corresponds to the frequency dimension.
sscp_conv_stride_size (`tuple(tuple(int, int), tuple(int, int))`, *optional*, defaults to `((2, 2), (2, 2))`):
    Stride sizes of the two convolutional layers in the subsample convolution projection in the Sub-sample
    Convolution Projection ("sscp") section of the Universal Speech Model. The stride sizes are specified as a
    tuple of height and width for each layer, where the height corresponds to the time dimension and the width
    corresponds to the frequency dimension.

Example:

```python
>>> from transformers import Gemma3nAudioConfig, Gemma3nAudioEncoder

>>> # Initializing a Gemma3nAudioEncoder gemma3n_audio-E4B-style configuration
>>> configuration = Gemma3nAudioConfig()

>>> # Initializing a model from the gemma3n_audio-E4B style configuration
>>> model = Gemma3nAudioEncoder(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```
Úgemma3n_audioi   ç�íµ ÷Æ°>g    _ Bg      I@ç      à?çü©ñÒMbP?c                óØ  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R&   S[ ;R&   S[ ;R&   S[ ;R	&   S[;R
&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[S[ ,          S[S[ S[ 3,          ,          ;R&   S[;R&   S[S[S[S[ S[ 3,          S[S[ S[ 3,          3,          ,          ;R&   S[S[S[S[ S[ 3,          S[S[ S[ 3,          3,          ,          ;R&   # )re   rf   Úvocab_offsetÚinput_feat_sizerh   Úrms_norm_epsÚgradient_clippingÚconf_attention_chunk_sizeÚconf_attention_context_leftÚconf_attention_context_rightÚconf_attention_logit_capÚconf_num_attention_headsÚconf_num_hidden_layersÚconf_conv_kernel_sizeÚconf_reduction_factorÚconf_residual_weightÚsscp_conv_channel_sizeÚsscp_conv_group_norm_epsÚsscp_conv_kernel_sizeÚsscp_conv_stride_size)rt   rw   ru   Útuple)ry   rz   s   "€rQ   r{   ÚGemma3nAudioConfig.__annotate__×   su  ø‡ ‚ ñR ÑñS ñT Ñ%ñU ñV ÑñW ñX ÑñY ñZ Ññ[ ñ\ Ñ/ñ] ñ^  #Ñ'ñ_ ñ` "%Ñ)ña ñb #&Ñ)ñc ñd $Ñ*ñe ñf "Ñ%ñg ñh  Ñ$ñi ñj Ñ"ñk ñl Ñ"ñm ñn  Ñ%ño ñp !¡�I©©c±3¨h­Õ7ÑCñq ñr $Ñ*ñs ñt  ¡%©©c±3¨h­¹¹sÁC¸x½Ð(HÕ"IÕIñ ñu ñ|  ¡%©©c±3¨h­¹¹sÁC¸x½Ð(HÕ"IÕIñ ò} rc   rA   Né€  )é€   é    )©é   rª   r©   )©re   re   r«   )r}   r~   r   r€   r�   r‚   rf   r“   r”   rh   r•   r–   r—   r˜   r™   rš   r›   rœ   r�   rž   rŸ   r    r¡   r¢   r£   rˆ   r‰   rŠ   r‹   s   @rQ   r�   r�   ×   sŽ   ø‡ € ñBðH !€Jà€JØ%€LØ€OØ€KØ€LØ/ÐØ%'ÐØ')ÐØ()Ð Ø&*ÐØ$%ÐØ"$ÐØ!"ÐØ!"ÐØ"%ÐØ:CÐØ&*ÐðMÐðMÐ÷} ƒ rc   r�   c                   óT   a € ] tR tRt o RtRtRtRtRtRt	^€t
RtR	tR
tV 3R ltRtV tR
# )ÚGemma3nVisionConfigi;  a_  
architecture (`str`, *optional*, defaults to `"resnet50"`):
    The timm architecture to load.
do_pooling (`bool`, *optional*, defaults to `True`):
    Whether to do pooling for the last_hidden_state in `TimmWrapperModel` or not.
model_args (`dict[str, Any]`, *optional*):
    Additional keyword arguments to pass to the `timm.create_model` function. e.g. `model_args={"depth": 3}`
    for `timm/vit_base_patch32_clip_448.laion2b_ft_in12k_in1k` to create a model with 3 blocks. Defaults to `None`.
vocab_offset (`int`, *optional*, defaults to 262144):
    Offset between the tokenizer vocab index for the token ids embedded by `Gemma3nMultimodalEmbedder` and the
    0-indexed `Gemma3nMultimodalEmbedder.embedding` table.

Example:
```python
>>> from transformers import Gemma3nVisionConfig, TimmWrapper

>>> # Initializing a TimmWrapper gemma3n_vision-E4B-style configuration
>>> configuration = Gemma3nVisionConfig()

>>> # Initializing a gemma3n_vision-E4B-style TimmWrapper from the configuration
>>> model = TimmWrapper(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```
Úgemma3n_visionç{®Gáz”?FÚmobilenetv5_300m_encr:   r9   r�   Nc                óˆ   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[R,          ;R	&   # )
re   Úinitializer_rangeÚ
do_poolingÚarchitecturerh   rf   r“   r•   NÚ
model_args)rw   rx   rv   rt   Údict)ry   rz   s   "€rQ   r{   Ú Gemma3nVisionConfig.__annotate__;  s�   ø‡ ‚ ñ@ Ñ#ñA ñB ÑñC ñD Ñ.ñE ñF ÑñG ñH ÑñI ñJ ÑñK ñL ÑñM ñN �t•Ñ"òO rc   rA   )r}   r~   r   r€   r�   r‚   r²   r³   r´   rh   rf   r“   r•   rµ   rˆ   r‰   rŠ   r‹   s   @rQ   r­   r­   ;  sB   ø‡ € ñð6 "€Jà#ÐØ€JØ.€LØ€KØ€JØ€LØ€LØ"€J÷O ƒ rc   r­   c                   óŽ   a a€ ] tR tRt oRtRtR]R]R]/t	Rt
RtRt^¼tRtR	tR
tRtRtRtRtRtRtRtV 3R ltV3R ltRtVtV ;t# )ÚGemma3nConfigie  av  
audio_soft_tokens_per_image (`int`, *optional*, defaults to 188):
    The number of soft tokens per audio clip.
vision_soft_tokens_per_image (`int`, *optional*, defaults to 256):
    The number of soft tokens per image.
boi_token_id (`int`, *optional*, defaults to 255999):
    The begin-of-image token index to wrap the image prompt.
eoi_token_id (`int`, *optional*, defaults to 262144):
    The end-of-image token index to wrap the image prompt.
boa_token_id (`int`, *optional*, defaults to 256000):
    The begin-of-audio token index to wrap the audio prompt.
eoa_token_id (`int`, *optional*, defaults to 262272):
    The end-of-audio token index to wrap the audio prompt.

Example:

```python
>>> from transformers import Gemma3nForConditionalGeneration, Gemma3nConfig, Gemma3nTextConfig

>>> # Initializing a MobileNet vision config, which is loaded from TIMM
>>> vision_config = Gemma3nVisionConfig()

>>> # Initializing a Gemma3n Audio config
>>> audio_config = Gemma3nAudioConfig()

>>> # Initializing a Gemma3n Text config
>>> text_config = Gemma3nTextConfig()

>>> # Initializing a Gemma3n gemma-3-4b style configuration
>>> configuration = Gemma3nConfig(text_config, vision_config, audio_config)

>>> # Initializing a model from the gemma-3-4b style configuration
>>> model = Gemma3nTextConfig(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```Úgemma3nÚtext_configÚvision_configÚaudio_configNr;   iÿç r9   i  i è r¦   i�  r¯   Tc                ó¶  <€ V P                   f&   \        4       V n         \        P                  R4       M:\	        V P                   \
        4      '       d   \        R/ V P                   B V n         \	        V P                  \
        4      '       d   \        R/ V P                  B V n        M2V P                  f%   \        4       V n        \        P                  R4       \	        V P                  \
        4      '       d   \        R/ V P                  B V n        M2V P                  f%   \        4       V n        \        P                  R4       \        SV `,  ! R/ VB  R # )NzAtext_config is None, using default Gemma3nTextConfig text config.zGvision_config is None, using default Gemma3nVisionConfig vision config.z7audio_config is None. Using default Gemma3nAudioConfig.rA   )r»   r1   ÚloggerÚinforB   r¶   r¼   r­   r½   r�   ÚsuperrJ   )rK   rL   Ú	__class__s   &,€rQ   rJ   ÚGemma3nConfig.__post_init__¤  sú   ø€ Ø×ÑÒ#Ü0Ó2ˆDÔÜ�K‰KÐ[Õ\Ü˜×(Ñ(¬$×/Ò/Ü0ÑD°4×3CÑ3CÑDˆDÔä�d×(Ñ(¬$×/Ò/Ü!4Ñ!J°t×7IÑ7IÑ!JˆDÕØ×ÑÒ'Ü!4Ó!6ˆDÔÜ�K‰KÐaÔbä�d×'Ñ'¬×.Ò.Ü 2Ñ G°T×5FÑ5FÑ GˆDÕØ×ÑÒ&Ü 2Ó 4ˆDÔÜ�K‰KÐQÔRä‰ÒÑ' Ô'rc   c                óê  <€ V ^8„  d   Qh/ S[ S[S[S[3,          ,          R,          ;R&   S[S[S[S[3,          ,          R,          ;R&   S[S[S[S[3,          ,          R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R	&   S[R,          ;R
&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[;R&   # )re   Nr»   r¼   r½   Úaudio_soft_tokens_per_imageÚvision_soft_tokens_per_imageÚboi_token_idÚeoi_token_idÚimage_token_idÚboa_token_idÚeoa_token_idÚaudio_token_idr²   Útie_word_embeddingsÚ	use_cache)	r1   r¶   rv   r   r­   r�   rt   rw   rx   )ry   rz   s   "€rQ   r{   ÚGemma3nConfig.__annotate__e  s5  ø‡ ‚ ñ` #¡T©#©s¨(¥^Õ3°dÕ:ÑAña ñb '©©c±3¨h­Õ7¸$Õ>ÑEñc ñd %¡t©C±¨H¥~Õ5¸Õ<ÑCñe ñf "% t¥Ñ1ñg ñh #&¨¥*Ñ2ñi ñj ˜•*Ñ&ñk ñl ˜•*Ñ&ñm ñn ˜$•JÑ(ño ñp ˜•*Ñ&ñq ñr ˜•*Ñ&ñs ñt ˜$•JÑ(ñu ñv ˜t•|Ñ*ñw ñx  �Ñ+ñy ñz Ñò{ rc   )r½   r»   r¼   )r}   r~   r   r€   r�   r‚   r1   r­   r�   Úsub_configsr»   r¼   r½   rÅ   rÆ   rÇ   rÈ   rÉ   rÊ   rË   rÌ   r²   rÍ   rÎ   rJ   rˆ   r‰   rŠ   Ú__classcell__©rÂ   rz   s   @@rQ   r¹   r¹   e  s…   ù‡ € ñ$ðL €JàÐ(ØÐ,ØÐ*ð€Kð >B€KØAE€MØ?C€LØ.1ÐØ/2Ð Ø&€LØ&€LØ!(€NØ&€LØ&€LØ!(€NØ&*ÐØ'+ÐØ€Iõ(÷ … rc   r¹   c                   ó4   a € ] tR tRt o RtRtV 3R ltRtV tR# )ÚGemma3nAudioEncoderModelOutputiº  zm
audio_mel_mask (`torch.BoolTensor`, *optional*):
    A torch.BoolTensor of shape `(batch_size, num_frames)`
Nc                óH   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   # )re   NÚaudio_mel_mask)ÚtorchÚ
BoolTensor)ry   rz   s   "€rQ   r{   Ú+Gemma3nAudioEncoderModelOutput.__annotate__º  s   ø‡ ‚ ñ ×$Ñ$ tÕ+Ñ2ò rc   rA   )	r}   r~   r   r€   r�   rÖ   rˆ   r‰   rŠ   r‹   s   @rQ   rÔ   rÔ   º  s   ø‡ € ñð
 /3€N÷ ƒ rc   rÔ   c                   ó4   a € ] tR tRt o RtRtV 3R ltRtV tR# )ÚGemma3nModelOutputWithPastiÅ  a±  
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
audio_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
Nc                óH   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   # ©re   NÚaudio_hidden_states©r×   ÚFloatTensor)ry   rz   s   "€rQ   r{   Ú'Gemma3nModelOutputWithPast.__annotate__Å  s   ø‡ ‚ ñ ×*Ñ*¨TÕ1Ñ8ò rc   rA   ©	r}   r~   r   r€   r�   rÞ   rˆ   r‰   rŠ   r‹   s   @rQ   rÛ   rÛ   Å  s   ø‡ € ñð 59Ð÷ ƒ rc   rÛ   c                   ó4   a € ] tR tRt o RtRtV 3R ltRtV tR# )ÚGemma3nCausalLMOutputWithPasti×  a
  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
audio_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
Nc                óH   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   # rÝ   rß   )ry   rz   s   "€rQ   r{   Ú*Gemma3nCausalLMOutputWithPast.__annotate__×  s   ø‡ ‚ ñ& ×*Ñ*¨TÕ1Ñ8ò' rc   rA   râ   r‹   s   @rQ   rä   rä   ×  s   ø‡ € ñð$ 59Ð÷' ƒ rc   rä   c                   óf   a a€ ] tR tRt oR	V3R lV 3R llltV3R lR ltV3R lR ltRtVtV ;t	# )
ÚGemma3nRMSNormií  c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )re   ÚdimÚepsÚ
with_scale)rt   rw   rx   )ry   rz   s   "€rQ   r{   ÚGemma3nRMSNorm.__annotate__î  s'   ø€ ÷ Lñ L™Cð L¡eð LÁñ Lrc   c                óÆ   <€ \         SV `  4        W n        W0n        V P                  '       d4   \        P
                  ! \        P                  ! V4      R R7      V n        R# R# )T)Úrequires_gradN)	rÁ   Ú__init__rë   rì   ÚnnÚ	Parameterr×   ÚonesÚweight)rK   rê   rë   rì   rÂ   s   &&&&€rQ   rð   ÚGemma3nRMSNorm.__init__î  s@   ø€ Ü‰ÑÔØŒØ$Œà�?�?ˆ?ÜŸ,š,¤u§z¢z°#£ÀdÔKˆDŽKñ rc   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )re   Úhidden_states©r×   ÚTensor)ry   rz   s   "€rQ   r{   rí   ö  s   ø€ ÷ =ñ =¡5§<¡<ñ =rc   c                ó¤   € VP                  ^4      P                  RRR7      V P                  ,           pV\        P                   ! VR4      ,          # )re   T)Úkeepdiméÿÿÿÿç      à¿)ÚpowÚmeanrë   r×   )rK   r÷   Úmean_squareds   && rQ   Ú_normÚGemma3nRMSNorm._normö  sA   € Ø$×(Ñ(¨Ó+×0Ñ0°¸TÐ0ÓBÀTÇXÁXÕMˆàœuŸyšy¨°tÓ<Õ<Ð<rc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©re   r÷   Úreturnrø   )ry   rz   s   "€rQ   r{   rí   û  s#   ø€ ÷ 4ñ 4¡U§\¡\ð 4±e·l±lñ 4rc   c                óÆ   € V P                  VP                  4       4      pV P                  '       d!   W P                  P                  4       ,          pVP	                  V4      # ©N)r  rw   rì   rô   Útype_as)rK   r÷   Únormed_outputs   && rQ   ÚforwardÚGemma3nRMSNorm.forwardû  sH   € ØŸ
™
 =×#6Ñ#6Ó#8Ó9ˆØ�?�?ˆ?Ø)¯K©K×,=Ñ,=Ó,?Õ?ˆMØ×$Ñ$ ]Ó3Ð3rc   )rë   rô   rì   )r�   T)
r}   r~   r   r€   rð   r  r
  r‰   rŠ   rÑ   rÒ   s   @@rQ   rè   rè   í  s*   ù‡ € ÷Lõ L÷=ð =÷
4÷ 4ð 4rc   rè   c                   ót   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR ltV3R lR	 ltR
tVt	V ;t
# )Ú%Gemma3nAudioRelativePositionEmbeddingi  c                ó    <€ V ^8„  d   QhRS[ /# ©re   Úconfig©r�   )ry   rz   s   "€rQ   r{   Ú2Gemma3nAudioRelativePositionEmbedding.__annotate__  s   ø€ ÷ lñ lÑ1ñ lrc   c                ó´  <€ \         SV `  4        Wn        V P                  P                  V n        V P                  P
                  V n        V P                  V P                  ,          V n        \        ^ V P                  P                  ^,
          4      V n
        V P                  P                  V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        RpRpV P                  ^,          p\         P"                  ! \%        V4      \%        V4      ,          4      \        V^,
          ^4      ,          pV\&        P(                  ! \&        P*                  ! V4      V) ,          4      ,          p\        P,                  ! VP%                  4       P/                  ^ 4      P/                  ^ 4      RR7      V n        R# )r   F©Úbiasç      ð?r8   ©Ú
persistentN)rÁ   rð   r  r›   Ú	num_headsrh   ÚchannelsÚhead_dimÚmaxr˜   Úmax_backwardr™   Úmax_forwardrñ   ÚLinearÚpos_projÚmathÚlogrw   r×   ÚexpÚarangeÚBufferÚ	unsqueezeÚinv_timescales)rK   r  Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementr'  rÂ   s   &&     €rQ   rð   Ú.Gemma3nAudioRelativePositionEmbedding.__init__  sC  ø€ Ü‰ÑÔØŒàŸ™×=Ñ=ˆŒØŸ™×/Ñ/ˆŒØŸ™¨¯©Õ7ˆŒÜ  4§;¡;×#JÑ#JÈQÕ#NÓOˆÔØŸ;™;×CÑCˆÔäŸ	š	 $§-¡-°·±À$Ç-Á-Õ1OÐV[Ô\ˆŒàˆØˆØŸ™¨!Õ+ˆÜ"&§(¢(¬5°Ó+?Ä%ÈÓBVÕ+VÓ"WÔZ]Ð^lÐopÕ^pÐrsÓZtÕ"tÐØ&¬¯ª´5·<²<ÀÓ3OÐSjÐRjÕ3jÓ)kÕkˆÜ Ÿiši¨×(<Ñ(<Ó(>×(HÑ(HÈÓ(K×(UÑ(UÐVWÓ(XÐejÔkˆÖrc   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )re   ÚpositionÚdtyper  )r×   rù   r/  )ry   rz   s   "€rQ   r{   r    s1   ø€ ÷ )ñ )±%·,±,ð )ÁuÇ{Á{ð )ÑW\×WcÑWcñ )rc   c                ó\  € VP                  4       P                  R4      pWP                  P                  VP                  \
        P                  R7      ,          p\
        P                  ! \
        P                  ! V4      \
        P                  ! V4      .RR7      pVP                  V4      # )é   )Údevicer/  ©rê   rü   )rw   r&  r'  Útor2  r×   Úfloat32ÚcatÚsinÚcosÚtype)rK   r.  r/  Úscaled_timeÚtiming_signals   &&&  rQ   Ú_get_timing_signal_1d_posÚ?Gemma3nAudioRelativePositionEmbedding._get_timing_signal_1d_pos  s{   € Ø—>‘>Ó#×-Ñ-¨bÓ1ˆØ×!4Ñ!4×!7Ñ!7¸x¿¹ÔV[×VcÑVcÐ!7Ó!dÕdˆÜŸ	š	¤5§9¢9¨[Ó#9¼5¿9º9À[Ó;QÐ"RÐXZÔ[ˆØ×!Ñ! %Ó(Ð(rc   c                ór   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[RS[RS[RS[RS[ P                  /# )	re   Úterm_bd_before_shiftÚ
batch_sizer  Únum_query_blocksÚquery_block_sizeÚkey_context_sizeÚmax_span_plus_1r  ©r×   rù   rt   )ry   rz   s   "€rQ   r{   r    sc   ø€ ÷ ;ñ ;á#Ÿl™lð;ñ ð;ñ ð	;ñ
 ð;ñ ð;ñ ð;ñ ð;ñ 
�‰ñ;rc   c                ó  € V^,           V,
          p^ V3p	\         P                  P                  W4      p
V
P                  VVVWV^,           ,          34      pVRRRRWV,          13,          pVP                  VVVVV34      pV# )a"  Performs the relative shift.

Args:
  term_bd_before_shift: Tensor of shape [B, N, U, W, F_span]. batch_size
    (B), num_heads (N), num_query_blocks (U), query_block_size (W),
    key_context_size (C = W+L+R), max_span_plus_1 (F_span = L+R+1).

Returns:
  Tensor of shape [B, N, U, W, C].
ºNNNN)rñ   Ú
functionalÚpadÚreshape)rK   r?  r@  r  rA  rB  rC  rD  Úpad_amount_last_dimÚpadding_tupleÚterm_bd_paddedÚterm_bd_reshapedÚterm_bd_slicedÚterm_bd_shifteds   &&&&&&&&      rQ   Ú_relative_shiftÚ5Gemma3nAudioRelativePositionEmbedding._relative_shift  s¯   € ð4  0°!Õ3°ÕFÐð Ð/Ð0ˆäŸ™×*Ñ*Ð+?ÓOˆð
 *×1Ñ1àØØ Ø °qÕ$8Õ9ð	ó
Ðð *¨!¨Q°Ð3XÐ5EÕ5XÐ3XÐ*XÕYˆð )×0Ñ0àØØ Ø Ø ðó
ˆð Ðrc   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )re   ÚqueriesÚkeysr  rø   )ry   rz   s   "€rQ   r{   r  \  s3   ø€ ÷ L)ñ L)™uŸ|™|ð L)±5·<±<ð L)ÁEÇLÁLñ L)rc   c           	     ój  € VP                   w  r4rVpVP                   w   r‰ p\        P                  ! V P                  V P                  ) ^,
          RVP
                  R7      P                  ^ 4      p
V
P                   ^,          pV P                  W¡P                  R7      pV P                  V4      pVP                  ^W°P                  V P                  4      P                  ^ 4      pVP                  ^ ^^^^4      pVP                  ^ ^^^^4      p\        P                  ! VV4      pVP                  ^ ^^^^4      pVP                  ^^^ 4      pVP                  W6WE,          V4      p\        P                  ! VV4      pVP                  VVVVV4      pV P!                  VVVVVV	V4      pVV,           # )r1  ©r2  ©r/  rü   )Úshaper×   r$  r  r  r2  r&  r<  r/  r   rJ  r  r  ÚsqueezeÚpermuteÚmatmulrQ  )rK   rT  rU  r@  rA  rB  r  r  Ú_rC  Úpos_indicesrD  Úsin_emb_timing_signalÚprojected_sin_embÚsin_embÚ	queries_pÚkeys_p_tÚterm_acÚ
q_permutedÚ
s_permutedÚ
q_reshapedÚterm_bd_unshifed_matmulÚterm_bd_unshifedrP  s   &&&                     rQ   r
  Ú-Gemma3nAudioRelativePositionEmbedding.forward\  sÄ  € ð OVÏmÉmÑKˆ
Ð&6À8Ø'+§z¡zÑ$ˆˆ1  1ô —l’l 4×#4Ñ#4°t×7GÑ7GÐ6GÈ!Õ6KÈRÐX_×XfÑXfÔg×qÑqØó
ˆð &×+Ñ+¨AÕ.ˆà $× >Ñ >ØŸ}™}ð !?ó !
Ðð
 !ŸM™MÐ*?Ó@Ðà#×+Ñ+¨A¨ÇÁÐPT×P]ÑP]Ó^×fÑfØó
ˆð —O‘O A q¨!¨Q°Ó2ˆ	Ø—<‘<  1 a¨¨AÓ.ˆÜ—,’,˜y¨(Ó3ˆð —_‘_ Q¨¨1¨a°Ó3ˆ
ð —_‘_ Q¨¨1Ó-ˆ
ð  ×'Ñ'¨
Ð?OÕ?bÐdlÓmˆ
ô
 #(§,¢,¨z¸:Ó"FÐð 3×:Ñ:ØØØØØó
Ðð ×.Ñ.ØØØØØØØó
ˆð ˜Õ(Ð(rc   )r  r  r  r'  r  r  r  r   )r}   r~   r   r€   rð   r<  rQ  r
  r‰   rŠ   rÑ   rÒ   s   @@rQ   r  r    s8   ù‡ € ÷ló l÷&)ð )÷;ð ;÷zL)÷ L)ð L)rc   r  c                   óŒ   a a€ ] tR tRt oV3R lV 3R lltR tV3R lR ltV3R lR ltV3R	 lR
 ltV3R lR lt	Rt
VtV ;t# )ÚGemma3nAudioAttentioni«  c                ó    <€ V ^8„  d   QhRS[ /# r  r  )ry   rz   s   "€rQ   r{   Ú"Gemma3nAudioAttention.__annotate__¬  s   ø€ ÷ iñ iÑ1ñ irc   c                ó~  <€ \         SV `  4        Wn        V P                  P                  V n        V P                  P
                  V n        V P
                  V P                  ,          V n        V P                  P                  V n        V P                  P                  V n
        \        ^ V P                  P                  ^,
          4      V n        V P                  P                  V n        V P                  V P                  ,           V P                  ,           V n        \#        V4      V n        \&        P(                  ! \*        P,                  ! V P                  34      4      V n        \&        P0                  ! V P
                  V P                  V P                  ,          RR7      V n        \&        P0                  ! V P
                  V P                  V P                  ,          RR7      V n        \&        P0                  ! V P
                  V P                  V P                  ,          RR7      V n        V P                  R,          pR\*        P&                  P8                  P;                  \*        P<                  ! R4      4      ,          p\&        P>                  ! W#,          PA                  4       PC                  4       RR7      V n"        V PG                  4       p\&        P>                  ! VRR7      V n$        \&        P>                  ! \*        P<                  ! V P                  4      PK                  4       RR7      V n&        R# )r   Fr  r  r@   r  Nrý   )'rÁ   rð   r  r›   r  rh   r  r—   Ú
chunk_sizer™   Úmax_future_horizonr  r˜   Úmax_past_horizonrš   Úattention_logits_soft_capÚcontext_sizer  Úrelative_position_embeddingrñ   rò   r×   ÚzerosÚper_dim_scaler  Úq_projÚk_projÚv_projrH  ÚsoftplusÚtensorr%  ÚcloneÚdetachÚq_scaleÚcreate_local_causal_valid_maskÚlocal_causal_valid_maskrw   Úsoftcap)rK   r  r  Úr_softplus_0r�  rÂ   s   &&   €rQ   rð   ÚGemma3nAudioAttention.__init__¬  s  ø€ Ü‰ÑÔØŒàŸ™×=Ñ=ˆŒØŸ;™;×2Ñ2ˆÔØ×(Ñ(¨D¯N©NÕ:ˆŒàŸ+™+×?Ñ?ˆŒØ"&§+¡+×"JÑ"JˆÔÜ # A t§{¡{×'NÑ'NÐQRÕ'RÓ SˆÔØ)-¯©×)MÑ)MˆÔ&Ø ŸO™O¨d×.CÑ.CÕCÀd×F]ÑF]Õ]ˆÔä+PÐQWÓ+XˆÔ(ÜŸ\š\¬%¯+ª+°t·}±}Ð6FÓ*GÓHˆÔä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒà—-‘- Õ%ˆØœUŸX™X×0Ñ0×9Ñ9¼%¿,º,ÀsÓ:KÓLÕLˆÜ—y’y 'Õ"8×!?Ñ!?Ó!A×!HÑ!HÓ!JÐW\Ô]ˆŒà"&×"EÑ"EÓ"GÐÜ')§y¢yÐ1HÐUZÔ'[ˆÔ$ä—y’y¤§¢¨d×.LÑ.LÓ!M×!SÑ!SÓ!UÐbgÔhˆŽrc   c                ó.  € \         P                  ! \         P                  ! V P                  V P                  3\         P
                  R 7      ^ R7      P                  p\         P                  ! \         P                  ! V P                  V P                  3\         P
                  R 7      V P                  V P                  ,           R7      p\         P                  ! V P                  V P                  3\         P
                  R 7      pW1,          V,          pV# )rX  )Údiagonal)	r×   Útrilró   rt  rp  rx   ÚTrr  rq  )rK   Úlower_causal_maskÚupper_causal_maskr�  s   &   rQ   r€  Ú4Gemma3nAudioAttention.create_local_causal_valid_maskÊ  sÂ   € Ü!ŸJšJÜ�JŠJ˜×)Ñ)¨4¯?©?Ð;Ä5Ç:Á:ÔNØô
÷ ‰!ð 	ô "ŸJšJÜ�JŠJ˜Ÿ™¨×):Ñ):Ð;Ä5Ç:Á:ÔNØ×*Ñ*¨T×-DÑ-DÕDô
Ðô #(§*¢*¨d¯o©o¸t×?PÑ?PÐ-QÔY^×YcÑYcÔ"dÐØ"9Õ"MÐPaÕ"aÐØ&Ð&rc   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )re   ÚxÚpad_leftÚ	pad_rightr  rE  )ry   rz   s   "€rQ   r{   rn  ×  s1   ø€ ÷ ñ ™5Ÿ<™<ð ±3ð Á3ð É5Ï<É<ñ rc   c                óª   € VP                   vrEpVP                  WB.VO54      pVP                  WC.VO54      p\        P                  ! WqV.^R7      pV# )r1  r3  )rY  Ú	new_zerosr×   r6  )	rK   r�  rŽ  r�  Úbatchr]  Ú
tail_shapeÚleftÚrights	   &&&&     rQ   Ú	_pad_dim1ÚGemma3nAudioAttention._pad_dim1×  sV   € Ø !§¡Ðˆ�:Ø�{‰{˜EÐ9¨jÑ9Ó:ˆØ—‘˜UÐ;°
Ñ;Ó<ˆÜ�IŠI�t Ð&¨AÔ.ˆØˆrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   rn  Þ  s#   ø€ ÷ ñ ©u¯|©|ð ÁÇÁñ rc   c                ód  € VP                   pVR,          w  r4W@P                  ,           ^,
          V P                  ,          pWPP                  ,          V,
          ;p^ 8”  d   V P                  V^ V4      pW5V P                  3VR,          ,           pVP                  V4      P	                  4       pV# )a  Turns a sequence to non overlapping blocks.

Args:
    hidden_states: a tensor of [batch, time, ...].

Returns:
    A tensor of [batch, num_blocks, block_size, ...], with necessary
    paddings,
    where output[:, i, ...] are x[:, i*block_size:(i+1)*block_size, ...].
ºNre   Nºre   NN)rY  rp  r–  rJ  Ú
contiguous)rK   r÷   rY  ÚbÚtÚ
num_blocksÚpadding_lenÚpermute_dimss   &&      rQ   Ú_convert_to_blockÚ'Gemma3nAudioAttention._convert_to_blockÞ  s•   € ð ×#Ñ#ˆØ�R�y‰ˆØŸ/™/Õ)¨AÕ-°$·/±/ÕAˆ
à%¯©Õ7¸!Õ;Ð;ˆK¸qÔ@Ø ŸN™N¨=¸!¸[ÓIˆMà t§¡Ð7¸%À½)ÕCˆØ%×-Ñ-¨lÓ;×FÑFÓHˆØÐrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   rn  ô  s#   ø€ ÷ .'ñ .'±E·L±Lð .'ÁUÇ\Á\ñ .'rc   c                ór  € V P                   pV P                  V P                  ,           ^,
          pV P                  WV4      pV P                  pV P                  pVP                  ^WER7      pVP                  ^8”  d+   VP                  ^8”  d   \        P                  ! VR^R7      pVP                  4       # )a“  Extracts temporal context for every block.

Args:
    hidden_states: a tensor of [batch, time, ...].

Returns:
    A tensor of [batch, num_blocks, context_size, ...], with necessary
    paddings,
    where context_size = block_size + left_context + right_context,
    and output[:, i, ...] are x[:, start-left_context:end+right_context,
    ...],
    start = i * block_size, end = (i + 1) * block_size.
)Ú	dimensionÚsizeÚstep)ÚsourceÚdestinationrü   )
rr  rq  rp  r–  rt  ÚunfoldÚndimr×   Úmovedimrœ  )rK   r÷   rŽ  r�  Ú	frame_lenÚ
frame_stepÚ
x_unfoldeds   &&     rQ   Ú_extract_block_contextÚ,Gemma3nAudioAttention._extract_block_contextô  s£   € ð ×(Ñ(ˆð ×+Ñ+¨d¯o©oÕ=ÀÕAˆ	ØŸ™ }À	ÓJˆà×%Ñ%ˆ	Ø—_‘_ˆ
ð #×)Ñ)°A¸IÐ)ÓWˆ
ð ×Ñ Ô! j§o¡o¸Ô&9ô Ÿš z¸"È!ÔLˆJà×$Ñ$Ó&Ð&rc   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )re   r÷   Úmaskr  ©r×   rù   rØ   )ry   rz   s   "€rQ   r{   rn  $  s5   ø€ ÷ dñ d¡U§\¡\ð d¹×9IÑ9Ið dÉeÏlÉlñ drc   c                ój	  € . VP                   R R	 OV P                  NV P                  N5pV P                  V4      P	                  V4      P                  4       pV P                  V4      P	                  V4      P                  4       pV P                  V4      P	                  V4      P                  4       p\        P                  P                  P                  V P                  4      p^^^V P                  3pVP                  V4      p	W@P                  ,          V	,          pVP                   R,          w  r«V P                  V4      pV P!                  V4      pV P!                  V4      pVP                   ^,          pV( pV P!                  V4      pVP"                  ^8X  dW   VP                   ^,          VP                   ^,          ,          V P$                  8X  d   VP	                  W¯V P$                  4      pVP                   V
VV P$                  38w  d-   \'        RVP                    RV
 RV RV P$                   R2	4      hVP)                  ^4      P)                  R
4      pV P*                  P)                  ^ 4      P)                  ^ 4      P)                  ^ 4      p\        P,                  ! VVP/                  VP0                  4      4      pV P3                  WÍ4      pV P4                  P/                  VP0                  4      pVV,          p\        P6                  ! V4      pVV,          p\        P8                  ! VV\        P:                  ! VP<                  4      P>                  4      p\        P                  P                  PA                  VR	\        PB                  R7      P/                  VP<                  R7      pVP                   w  pppppVP                   R	,          pVPE                  ^ ^^^^4      P	                  R	VV4      pVPE                  ^ ^^^^4      P	                  R	VV4      p\        PF                  ! VV4      p V P	                  VVVVV4      PE                  ^ ^^^^4      p!V!P	                  V
WðPH                  ,          V P                  V P                  34      p!V!RR V13,          p!V!# )Nrš  z%Shape of extracted_valid_mask_blocks z	 is not (z, z) after potential reshape.)rê   r/  rX  rG  rü   éþÿÿÿ)%rY  r  r  rx  rJ  rœ  ry  rz  r×   rñ   rH  r{  rw  Úviewr  r¢  r±  r¬  rt  rF   r&  r�  Úlogical_andr4  r2  ru  r‚  ÚtanhÚwhereÚfinfor/  ÚminÚsoftmaxr5  r[  Úbmmrp  )"rK   r÷   r´  Ú	qkv_shapeÚquery_statesÚ
key_statesÚvalue_statesÚper_dim_scale_spÚbroadcast_shapeÚper_dim_scale_sp_broadcastr@  Úq_timeÚquery_blocksÚ
key_blocksÚvalue_blocksrA  Úoriginal_valid_maskÚextracted_valid_mask_blocksÚcondition_from_input_validityÚcondition_from_causalityÚfinal_condition_for_whereÚlogitsÚsoftcap_valÚprobabilitiesÚb_dimÚn_dimÚu_dimÚw_dimÚc_dimÚh_dimÚprob_bunÚv_bunÚ
result_bmmÚcontext_vectorss"   &&&                               rQ   r
  ÚGemma3nAudioAttention.forward$  sG  € àN�m×)Ñ)¨#¨2Ð.ÐN°·±ÐNÀÇÁÑNˆ	Ø—{‘{ =Ó1×9Ñ9¸)ÓD×OÑOÓQˆØ—[‘[ Ó/×7Ñ7¸	ÓB×MÑMÓOˆ
Ø—{‘{ =Ó1×9Ñ9¸)ÓD×OÑOÓQˆä Ÿ8™8×.Ñ.×7Ñ7¸×8JÑ8JÓKÐà˜a  D§M¡MÐ2ˆØ%5×%:Ñ%:¸?Ó%KÐ"Ø#§l¡lÕ2Ð5OÕOˆà)×/Ñ/°Õ3Ñˆ
à×-Ñ-¨lÓ;ˆØ×0Ñ0°Ó<ˆ
Ø×2Ñ2°<Ó@ˆØ'×-Ñ-¨aÕ0Ðð  $˜eÐð '+×&AÑ&AÐBUÓ&VÐ#ð (×,Ñ,°Ô1Ø+×1Ñ1°!Õ4Ð7R×7XÑ7XÐYZÕ7[Õ[Ð_c×_pÑ_pÔpà*E×*MÑ*MØ¨d×.?Ñ.?ó+Ð'ð '×,Ñ,ØØØ×Ñð1
ô 
ô
 ðØ/×5Ñ5Ð6°iÀ
¸|ð LØ$Ð% R¨×(9Ñ(9Ð':Ð:TðVóð ð )D×(MÑ(MÈaÓ(P×(ZÑ(ZÐ[]Ó(^Ð%ð $(×#?Ñ#?×#IÑ#IÈ!Ó#L×#VÑ#VÐWXÓ#Y×#cÑ#cÐdeÓ#fÐ ô
 %*×$5Ò$5Ø)Ø$×'Ñ'Ð(E×(LÑ(LÓMó%
Ð!ð ×1Ñ1°,ÓKˆð —l‘l—o‘o f§m¡mÓ4ˆØ˜+Õ%ˆÜ—’˜FÓ#ˆØ˜+Õ%ˆô —’Ð6¸ÄÇÂÈFÏLÉLÓ@Y×@]Ñ@]Ó^ˆÜŸ™×+Ñ+×3Ñ3°FÀÌ%Ï-É-Ð3ÓX×[Ñ[Ðbn×btÑbtÐ[Óuˆð -:×,?Ñ,?Ñ)ˆˆu�e˜U EØ×"Ñ" 2Õ&ˆØ ×(Ñ(¨¨A¨q°!°QÓ7×?Ñ?ÀÀEÈ5ÓQˆØ×$Ñ$ Q¨¨1¨a°Ó3×;Ñ;¸BÀÀuÓMˆÜ—Y’Y˜x¨Ó/ˆ
Ø$×,Ñ,¨U°E¸5À%ÈÓO×WÑWÐXYÐ[\Ð^_ÐabÐdeÓfˆØ)×1Ñ1àØ §?¡?Õ2Ø—‘Ø—‘ð	ó
ˆð *¨!¨W¨f¨W¨*Õ5ˆàÐrc   )rs  rp  r  rt  r  rh   ry  r�  rq  rr  r  rw  rx  r  ru  r‚  rz  )r}   r~   r   r€   rð   r€  r–  r¢  r±  r
  r‰   rŠ   rÑ   rÒ   s   @@rQ   rl  rl  «  sG   ù‡ € ÷ió iò<'÷ð ÷ð ÷,.'ð .'÷`d÷ dð drc   rl  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚGemma3nAudioCumulativeGroupNormi‹  a¼  Applies Group Normalization cumulatively over the time dimension.

This layer normalizes the input by calculating the mean and variance
cumulatively over the time dimension (dim 1). The statistics are computed
over all feature dimensions (specified by `feature_dims` and `num_channels`)
for elements marked as valid by the optional `mask`.

If a `mask` is provided (True for valid, False for invalid/padded),
invalid time steps do not contribute to the statistics calculation, and
their corresponding output values are zeroed out.

Scale and bias, if enabled, are applied per-channel (last dimension).
This behavior is similar to JAX's `GroupNormalization` with `num_groups=1`
and `cumulative=True`.
c                ó<   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[/# )re   Únum_channelsÚfeature_dimsrë   )rt   r   rw   )ry   rz   s   "€rQ   r{   Ú,Gemma3nAudioCumulativeGroupNorm.__annotate__œ  s2   ø€ ÷ Nñ NáðNñ ™s•mðNñ ñ	Nrc   c           	     ó0  <€ \         SV `  4        Wn        \        V4      V n        W0n        \        P                  ! \        P                  ! V4      4      V n
        \        \        ^^\        V P                  4      ,           ^,           4      4      V n        R# )re   N)rÁ   rð   rá  r¤   râ  rë   rñ   rò   r×   ró   rô   rH   rD   Úreduction_axes)rK   rá  râ  rë   rÂ   s   &&&&€rQ   rð   Ú(Gemma3nAudioCumulativeGroupNorm.__init__œ  sn   ø€ ô 	‰ÑÔØ(ÔÜ! ,Ó/ˆÔØŒô —l’l¤5§:¢:¨lÓ#;Ó<ˆŒô $¤E¨!¨Q´°T×5FÑ5FÓ1GÕ-GÈ!Õ-KÓ$LÓMˆÖrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   rã  ®  s'   ø€ ÷ G,ñ G,¡U§\¡\ð G,±e·l±lñ G,rc   c                óx  € V P                   V P                  3,           pVP                  R,          V8w  d#   \        RVP                  R,           RV 24      hVP                  p\
        P                  pVP                  V4      p\
        P                  ! WTR7      p\
        P                  ! WPP                  RR7      p\
        P                  ! V^R7      p\
        P                  ! W`P                  RR7      p	\
        P                  ! V	^R7      p
\
        P                  ! V
RR	7      pW‹,          pW\,
          P                  ^4      p\
        P                  ! WÐP                  RR7      p\
        P                  ! V^R7      pWû,          pW\,
          \
        P                  ! VV P                  ,           4      ,          pV P                   P                  V4      p^.VP#                  4       ^,
          ,          V P                  .,           pVVP%                  V4      ,          pVV,          pVP                  V4      # )
z¶Applies cumulative group norm, optionally using a mask.

Args:
  hidden_states: Input tensor, shape [B, T, *feature_dims, C].

Returns:
  Normalized tensor with the same shape as x.
r›  zInput tensor shape suffix z> does not match expected suffix (feature_dims + num_channels) rX  T©rê   rû   r3  r  )r½  )râ  rá  rY  rF   r/  r×   r5  r4  Ú	ones_likeÚsumrå  ÚcumsumÚclamprþ   Úrsqrtrë   rô   rê   r¸  )rK   r÷   Úexpected_input_suffixÚinput_dtypeÚ
calc_dtypeÚx_calcÚ	mask_calcÚsum_values_at_tÚcum_sum_valuesÚelements_in_group_at_tÚcum_count_elementsÚsafe_cum_count_elementsÚcum_meanÚsquared_diff_from_meanÚsum_sq_diff_at_tÚcum_sum_sq_diffÚcum_varianceÚnormalized_xÚscaleÚscale_view_shapeÚfinal_outputs   &&                   rQ   r
  Ú'Gemma3nAudioCumulativeGroupNorm.forward®  sã  € ð !%× 1Ñ 1°T×5FÑ5FÐ4HÕ HÐØ×Ñ˜rÕ"Ð&;Ô;ÜØ,¨]×-@Ñ-@ÀÕ-DÐ,Eð F9Ø9NÐ8OðQóð ð
 $×)Ñ)ˆä—]‘]ˆ
Ø×!Ñ! *Ó-ˆô —O’O FÔ=ˆ	ô  Ÿ)š) F×0CÑ0CÈTÔRˆäŸš o¸1Ô=ˆô "'§¢¨9×:MÑ:MÐW[Ô!\Ðä"Ÿ\š\Ð*@ÀaÔHÐä"'§+¢+Ð.@ÀcÔ"JÐð "Õ;ˆð
 #)Õ"3×!8Ñ!8¸Ó!;ÐÜ Ÿ9š9Ð%;×ATÑATÐ^bÔcÐô  Ÿ,š,Ð'7¸QÔ?ˆð 'Õ@ˆð Õ)¬U¯[ª[¸ÈÏÉÕ9PÓ-QÕQˆð —‘—‘˜zÓ*ˆà˜3 -×"3Ñ"3Ó"5¸Õ"9Õ:¸d×>OÑ>OÐ=PÕPÐØ# e§j¡jÐ1AÓ&BÕBˆð $ iÕ/ˆà�‰˜{Ó+Ð+rc   )rë   râ  rá  rå  rô   )r‘   ©
r}   r~   r   r€   r�   rð   r
  r‰   rŠ   rÑ   rÒ   s   @@rQ   rß  rß  ‹  s(   ù‡ € ñ÷ Nõ N÷$G,÷ G,ð G,rc   rß  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚGemma3nAudioSSCPConvBlockiø  zÍA single convolution block for the SubSampleConvProjection.

This block consists of a 2D convolution, followed by CumulativeGroupNorm,
and a ReLU activation. It handles manual padding for the convolution.
c                óP   <€ V ^8„  d   QhRS[ RS[RS[RS[S[S[S[S[3,          /# )re   r  ÚidxÚinput_freq_dimÚmanual_padding)r�   rt   r¤   )ry   rz   s   "€rQ   r{   Ú&Gemma3nAudioSSCPConvBlock.__annotate__ÿ  sA   ø€ ÷ )$ñ )$á"ð)$ñ ð)$ñ ð	)$ñ
 ™c¡3©©SÐ0Õ1ñ)$rc   c           	     ó¦  <€ \         SV `  4        Wn        W@n        V^ 8X  d   ^M#V P                  P                  V^,
          ,          pV P                  P                  V,          pV P                  P
                  V,          w  rxV P                  P                  V,          w  rš\        P                  ! VVVV3Wš3RRR7      V n	        W0P                  ^ ,          ,           V P                  ^,          ,           pW¸,
          V
,          ^,           p\        VV3V P                  P                  R7      V n        \        P                  ! 4       V n        R# )r   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr  )rá  râ  rë   N)r   r   )rÁ   rð   r  r	  r    r¢   r£   rñ   ÚConv2dÚconvrß  r¡   ÚnormÚReLUÚ
activation)rK   r  r  r  r	  r  r  Úkernel_hÚkernel_wÚstride_hÚstride_wÚf_in_paddedÚ
f_out_convrÂ   s   &&&&&        €rQ   rð   Ú"Gemma3nAudioSSCPConvBlock.__init__ÿ  s  ø€ ô 	‰ÑÔØŒØ,Ôð  !œ8‘a¨¯©×)KÑ)KÈCÐRSÍGÕ)TˆØ—{‘{×9Ñ9¸#Õ>ˆØ!Ÿ[™[×>Ñ>¸sÕCÑˆØ!Ÿ[™[×>Ñ>¸sÕCÑˆä—I’IØ#Ø%àØðð Ð'ØØô

ˆŒ	ð %×':Ñ':¸1Õ'=Õ=À×@SÑ@SÐTUÕ@VÕVˆØ!Õ,°Õ9¸AÕ=ˆ
ä3Ø%Ø$˜Ø—‘×4Ñ4ô
ˆŒ	ô Ÿ'š'›)ˆŽrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©re   Úaudio_encodingsr  rø   )ry   rz   s   "€rQ   r{   r
  *  s#   ø€ ÷ 7ñ 7¡u§|¡|ð 7¹¿¹ñ 7rc   c                ó�  € \         P                  ! WP                  R RR7      P                  V P                  P
                  P                  4      pV P	                  V4      pVP                  ^ ^^^4      P                  4       pV P                  V4      pVP                  ^ ^^^4      P                  4       pV P                  V4      # )Úconstantr@   )ÚmodeÚvalue)ÚFrI  r	  r4  r  rô   r/  r[  rœ  r  r  )rK   r  Úaudio_encodings_paddedÚaudio_encodings_convÚ
x_for_normÚx_normedÚaudio_encodings_normeds   &&     rQ   r
  Ú!Gemma3nAudioSSCPConvBlock.forward*  s®   € ô "#§¢ ×8KÑ8KÐR\ÐdgÔ!h×!kÑ!kØ�I‰I×Ñ×"Ñ"ó"
Ðð
  $Ÿy™yÐ)?Ó@Ðð *×1Ñ1°!°Q¸¸1Ó=×HÑHÓJˆ
Ø—9‘9˜ZÓ(ˆà!)×!1Ñ!1°!°Q¸¸1Ó!=×!HÑ!HÓ!JÐØ�‰Ð5Ó6Ð6rc   )r  r  r  r	  r  ))r   r   r   r   r  rÒ   s   @@rQ   r  r  ø  s$   ù‡ € ñ÷)$õ )$÷V7÷ 7ð 7rc   r  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú#Gemma3nAudioSubSampleConvProjectioni=  c                ó    <€ V ^8„  d   QhRS[ /# r  r  )ry   rz   s   "€rQ   r{   Ú0Gemma3nAudioSubSampleConvProjection.__annotate__>  s   ø€ ÷ 7mñ 7mÑ1ñ 7mrc   c                óÞ  <€ \         SV `  4        Wn        VP                  p. p. p\	        ^4       F‹  pVP
                  V,          w  rgVP                  V,          w  r‰^ p
V^,
          p^p^pVVV
V3pVP                  V4       W,,           V,           pW÷,
          V	,          ^,           pVP                  V4       TpK�  	  \        ^ VP                  VV^ ,          R7      V n	        \        ^V^ ,          VV^,          R7      V n
        VP                  R,          pVR,          pVV,          V n        \        P                  ! V P                  V P                  P                  RR7      V n        R# )re   )r  r  r  r	  Fr  Nrü   )rÁ   rð   r  r”   rH   r¢   r£   Úappendr  Úconv_0Úconv_1r    Úinput_proj_in_featuresrñ   r  rh   Úinput_proj_linear)rK   r  Úcurrent_f_for_block_inputÚcalculated_block_paddingÚcalculated_f_out_dimsrN   r  r  r  r  Ú	pad_t_topÚpad_t_bottomÚ
pad_f_leftÚpad_f_rightÚmanual_padding_tupler  Úf_out_after_convÚfinal_c_outÚfinal_f_outrÂ   s   &&                 €rQ   rð   Ú,Gemma3nAudioSubSampleConvProjection.__init__>  sr  ø€ Ü‰ÑÔØŒà$*×$:Ñ$:Ð!Ø#%Ð Ø "Ðä�q–ˆAØ!'×!=Ñ!=¸aÕ!@ÑˆHØ!'×!=Ñ!=¸aÕ!@ÑˆHð ˆIØ# a�<ˆLð ˆJØˆKð ØØØð	$Ð ð %×+Ñ+Ð,@ÔAð 4Õ@À;ÕNˆKØ +Õ 6¸8ÕCÀaÕGÐØ!×(Ñ(Ð)9Ô:Ø(8Ò%ñ= ô@ 0ØØ!×1Ñ1ØØ3°AÕ6ô	
ˆŒô 0ØØ0°Õ3ØØ3°AÕ6ô	
ˆŒð ×3Ñ3°BÕ7ˆØ+¨BÕ/ˆØ&1°KÕ&?ˆÔ#Ü!#§¢¨4×+FÑ+FÈÏÉ×H_ÑH_ÐfkÔ!lˆÖrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   r.  w  s#   ø€ ÷ ñ ¡u§|¡|ð ¹¿¹ñ rc   c                ó   € VP                  ^4      pV P                  V4      pV P                  V4      pVP                  w  rErgVP	                  ^ ^^^4      P                  4       pVP                  WFWu,          4      p	V P                  V	4      p
V
# ©r1  )r&  r1  r2  rY  r[  rœ  r¸  r4  )rK   r  Úaudio_encodings_reshapedr�  r�  Úc_outÚt_outÚf_outÚ
x_permutedÚoutput_flattenedÚoutputs   &&         rQ   r
  Ú+Gemma3nAudioSubSampleConvProjection.forwardw  s…   € ð $3×#<Ñ#<¸QÓ#?Ð Ø�K‰KÐ0Ó1ˆØ�K‰K˜‹Nˆà!"§¡Ñˆ�%à—Y‘Y˜q ! Q¨Ó*×5Ñ5Ó7ˆ
Ø%Ÿ?™?¨1°Uµ]ÓCÐØ×'Ñ'Ð(8Ó9ˆØˆrc   )r  r1  r2  r3  r4  ©	r}   r~   r   r€   rð   r
  r‰   rŠ   rÑ   rÒ   s   @@rQ   r,  r,  =  s!   ù‡ € ÷7mó 7m÷r÷ ð rc   r,  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma3nAudioConformerAttentioni†  c                ó    <€ V ^8„  d   QhRS[ /# r  r  )ry   rz   s   "€rQ   r{   Ú+Gemma3nAudioConformerAttention.__annotate__‡  s   ø€ ÷ Añ AÑ1ñ Arc   c                ó  <€ \         SV `  4        Wn        V P                  P                  V n        \
        P                  ! \        P                  ! V P                  P                  4      R R7      V n	        \        V P                  P                  4      V n        \        V4      V n        \
        P                  ! V P                  V P                  P                  R R7      V n        \        V P                  P                  4      V n        R# ©Fr  r  N)rÁ   rð   r  rh   Úpost_in_featuresrñ   r%  r×   r|  r–   rè   Úpre_attn_normrl  Úattnr  ÚpostÚ	post_norm©rK   r  rÂ   s   &&€rQ   rð   Ú'Gemma3nAudioConformerAttention.__init__‡  sª   ø€ Ü‰ÑÔØŒØ $§¡× 7Ñ 7ˆÔÜ!#§¢¬5¯<ª<¸¿¹×8UÑ8UÓ+VÐchÔ!iˆÔÜ+¨D¯K©K×,CÑ,CÓDˆÔÜ)¨&Ó1ˆŒ	Ü—I’I˜d×3Ñ3°T·[±[×5LÑ5LÐSXÔYˆŒ	Ü'¨¯©×(?Ñ(?Ó@ˆŽrc   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©re   r  rÖ   r  rµ  )ry   rz   s   "€rQ   r{   rP  ‘  s8   ø€ ÷ Oñ O¡u§|¡|ð OÁU×EUÑEUð OÑZ_×ZfÑZfñ Orc   c                ó˜  € Tp\         P                  ! WP                  ) V P                  4      pV P                  V4      pV P	                  WB4      pVP
                  w  rgr‰VP                  WgW‰,          4      p
V P                  V
4      p\         P                  ! WP                  ) V P                  4      pW0P                  V4      ,           # r  )	r×   rí  r–   rT  rU  rY  rJ  rV  rW  )rK   r  rÖ   Úaudio_encodings_input_to_attnÚaudio_encodings_normÚaudio_encodings_attn_outr�  rž  r  r  rD  s   &&&        rQ   r
  Ú&Gemma3nAudioConformerAttention.forward‘  s²   € Ø(7Ð%ÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ#×1Ñ1°/ÓBÐà#'§9¡9Ð-AÓ#RÐ ð %=×$BÑ$BÑ!ˆˆiØ#;×#CÑ#CÀAÈ)ÕJ^Ó#_Ð àŸ)™)Ð$<Ó=ˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ,¯~©~¸oÓ/NÕNÐNrc   )rU  r  r–   rV  rS  rW  rT  rL  rÒ   s   @@rQ   rN  rN  †  s#   ù‡ € ÷Aó A÷O÷ Oð Orc   rN  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú Gemma3nAudioConformerFeedForwardi¢  c                ó    <€ V ^8„  d   QhRS[ /# r  r  )ry   rz   s   "€rQ   r{   Ú-Gemma3nAudioConformerFeedForward.__annotate__£  s   ø€ ÷ 
Añ 
AÑ1ñ 
Arc   c                ó´  <€ \         SV `  4        Wn        \        P                  ! \
        P                  ! V P                  P                  4      R R7      V n        \        V P                  P                  4      V n
        \        P                  ! V P                  P                  V P                  P                  ^,          R R7      V n        \        P                  ! V P                  P                  ^,          V P                  P                  R R7      V n        \        V P                  P                  4      V n        V P                  P                  V n        R# rR  )rÁ   rð   r  rñ   r%  r×   r|  r–   rè   rh   Úpre_layer_normr  Úffw_layer_1Úffw_layer_2Úpost_layer_normrŸ   Úpost_layer_scalerX  s   &&€rQ   rð   Ú)Gemma3nAudioConformerFeedForward.__init__£  sÙ   ø€ Ü‰ÑÔØŒä!#§¢¬5¯<ª<¸¿¹×8UÑ8UÓ+VÐchÔ!iˆÔä,¨T¯[©[×-DÑ-DÓEˆÔÜŸ9š9 T§[¡[×%<Ñ%<¸d¿k¹k×>UÑ>UÐXYÕ>YÐ`eÔfˆÔÜŸ9š9 T§[¡[×%<Ñ%<¸qÕ%@À$Ç+Á+×BYÑBYÐ`eÔfˆÔÜ-¨d¯k©k×.EÑ.EÓFˆÔØ $§¡× @Ñ @ˆÖrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   rd  ¯  s'   ø€ ÷ 	Dñ 	D¡u§|¡|ð 	D¹¿¹ñ 	Drc   c                ó®  € Tp\         P                  ! WP                  ) V P                  4      pV P                  V4      pV P	                  V4      p\
        P                  P                  V4      pV P                  V4      p\         P                  ! WP                  ) V P                  4      pV P                  V4      pW!V P                  ,          ,           # r  )r×   rí  r–   rf  rg  rñ   rH  Úsilurh  ri  rj  )rK   r  Úresiduals   && rQ   r
  Ú(Gemma3nAudioConformerFeedForward.forward¯  s¯   € Ø"ˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ×-Ñ-¨oÓ>ˆØ(,×(8Ñ(8¸Ó(IˆÜŸ-™-×,Ñ,¨_Ó=ˆØ(,×(8Ñ(8¸Ó(IˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ×.Ñ.¨Ó?ˆØ¨T×-BÑ-BÕBÕCÐCrc   )r  rg  rh  r–   ri  rj  rf  rL  rÒ   s   @@rQ   rb  rb  ¢  s#   ù‡ € ÷
Aó 
A÷	D÷ 	Dð 	Drc   rb  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú Gemma3nAudioConformerLightConv1di»  c                ó    <€ V ^8„  d   QhRS[ /# r  r  )ry   rz   s   "€rQ   r{   Ú-Gemma3nAudioConformerLightConv1d.__annotate__¼  s   ø€ ÷ Dñ DÑ1ñ Drc   c           
     óð  <€ \         SV `  4        Wn        \        V P                  P                  V P                  P
                  R 7      V n        \        P                  ! V P                  P                  V P                  P                  ^,          RR7      V n	        \        P                  ! V P                  P                  V P                  P                  V P                  P                  ^^ V P                  P                  RR7      V n        \        P                  ! \        P                  ! V P                  P                   4      RR7      V n        \        V P                  P                  V P                  P
                  R 7      V n        \        P                  ! V P                  P                  V P                  P                  RR7      V n        V P                  P                  ^,
          V n        R# )©rë   Fr  )r  r  r  r  r  Úgroupsr  r  N)rÁ   rð   r  rè   rh   r•   rf  rñ   r  Úlinear_startÚConv1dr�   Údepthwise_conv1dr%  r×   r|  r–   Ú	conv_normÚ
linear_endÚcausal_paddingrX  s   &&€rQ   rð   Ú)Gemma3nAudioConformerLightConv1d.__init__¼  s?  ø€ Ü‰ÑÔØŒä,¨T¯[©[×-DÑ-DÈ$Ï+É+×JbÑJbÔcˆÔÜŸIšI d§k¡k×&=Ñ&=¸t¿{¹{×?VÑ?VÐYZÕ?ZÐafÔgˆÔÜ "§	¢	ØŸ™×/Ñ/ØŸ™×0Ñ0ØŸ™×9Ñ9ØØØ—;‘;×*Ñ*Øô!
ˆÔô "$§¢¬5¯<ª<¸¿¹×8UÑ8UÓ+VÐchÔ!iˆÔÜ'¨¯©×(?Ñ(?ÀTÇ[Á[×E]ÑE]Ô^ˆŒÜŸ)š) D§K¡K×$;Ñ$;¸T¿[¹[×=TÑ=TÐ[`ÔaˆŒà"Ÿk™k×?Ñ?À!ÕCˆÖrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   rt  Ñ  s#   ø€ ÷ ñ ¡u§|¡|ð ¹¿¹ñ rc   c                ó@  € TpV P                  V4      pV P                  V4      p\        P                  P                  P                  VRR7      pVP                  ^ ^^4      p\        P                  ! W0P                  ^ 34      pV P                  V4      pVP                  ^ ^^4      p\        P                  ! WP                  ) V P                  4      pV P                  V4      p\        P                  P                  V4      pV P                  V4      pW,           pV# )r1  r3  rü   )rf  rx  r×   rñ   rH  Úglur[  r$  rI  r}  rz  rí  r–   r{  rn  r|  )rK   r  Úaudio_encodings_residualÚaudio_encodings_permutedÚaudio_encodings_permuted_paddedrJ  s   &&    rQ   r
  Ú(Gemma3nAudioConformerLightConv1d.forwardÑ  sú   € Ø#2Ð à×-Ñ-¨oÓ>ˆØ×+Ñ+¨OÓ<ˆÜŸ(™(×-Ñ-×1Ñ1°/ÀrÐ1ÓJˆà#2×#:Ñ#:¸1¸aÀÓ#CÐ ä*+¯%ª%Ð0H×K^ÑK^Ð`aÐJbÓ*cÐ'Ø×/Ñ/Ð0OÓPˆà)×1Ñ1°!°Q¸Ó:ˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØŸ.™.¨Ó9ˆÜŸ-™-×,Ñ,¨_Ó=ˆØŸ/™/¨/Ó:ˆØ Õ;ˆØˆrc   )r}  r  r{  rz  r–   r|  rx  rf  rL  rÒ   s   @@rQ   rr  rr  »  s    ù‡ € ÷Dó D÷*÷ ð rc   rr  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma3nAudioConformerBlockiæ  c                ó    <€ V ^8„  d   QhRS[ /# r  r  )ry   rz   s   "€rQ   r{   Ú'Gemma3nAudioConformerBlock.__annotate__ç  s   ø€ ÷ 	<ñ 	<Ñ1ñ 	<rc   c                óÒ  <€ \         SV `  4        Wn        \        V P                  4      V n        \        V P                  4      V n        \        V P                  4      V n        \        V P                  4      V n	        \        P                  ! \        P                  ! V P                  P                  4      R R7      V n        \        V P                  P                   4      V n        R# )Fr  N)rÁ   rð   r  rb  Úffw_layer_startrN  Ú	attentionrr  Úlconv1dÚffw_layer_endrñ   r%  r×   r|  r–   rè   rh   r  rX  s   &&€rQ   rð   Ú#Gemma3nAudioConformerBlock.__init__ç  s’   ø€ Ü‰ÑÔØŒä?ÀÇÁÓLˆÔÜ7¸¿¹ÓDˆŒÜ7¸¿¹ÓDˆŒÜ=¸d¿k¹kÓJˆÔÜ!#§¢¬5¯<ª<¸¿¹×8UÑ8UÓ+VÐchÔ!iˆÔÜ" 4§;¡;×#:Ñ#:Ó;ˆŽ	rc   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# r[  rµ  )ry   rz   s   "€rQ   r{   r‰  ò  s3   ø€ ÷ ñ ¡u§|¡|ð ÁU×EUÑEUð ÑZ_×ZfÑZfñ rc   c                ón  € V P                  V4      pV P                  W4      pV( pWP                  R4      P                  VP                  4      ,          pV P                  V4      pV P                  V4      p\        P                  ! WP                  ) V P                  4      pV P                  V4      pV# )r1  rü   )r‹  rŒ  r&  r4  r/  r�  rŽ  r×   rí  r–   r  )rK   r  rÖ   Úvalidity_mask_for_lconvÚaudio_encodings_for_lconv_inputrJ  s   &&&   rQ   r
  Ú"Gemma3nAudioConformerBlock.forwardò  s§   € Ø×.Ñ.¨Ó?ˆØŸ.™.¨ÓIˆØ#1 /ÐØ*9×<]Ñ<]Ð^`Ó<a×<dÑ<dØ×!Ñ!ó=
õ +
Ð'ð Ÿ,™,Ð'FÓGˆà×,Ñ,¨_Ó=ˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ—‘˜?Ó+ˆØˆrc   )rŒ  r  rŽ  r‹  r–   r�  r  rL  rÒ   s   @@rQ   r‡  r‡  æ  s   ù‡ € ÷	<ó 	<÷÷ ð rc   r‡  c                   ó   € ] tR tRtRtR# )ÚGemma3nTextScaledWordEmbeddingi  rA   N©r}   r~   r   r€   r‰   rA   rc   rQ   r–  r–    ó   † Ûrc   r–  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚGemma3nTextLaurelBlocki  z Learned Augmented Residual Layerc                ó    <€ V ^8„  d   QhRS[ /# r  ©r1   )ry   rz   s   "€rQ   r{   Ú#Gemma3nTextLaurelBlock.__annotate__  s   ø€ ÷ fñ fÑ0ñ frc   c                ó¼  <€ \         SV `  4        Wn        \        P                  ! V P                  P
                  V P                  P                  R R7      V n        \        P                  ! V P                  P                  V P                  P
                  R R7      V n        \        V P                  P
                  V P                  P                  R7      V n        R# )Fr  rv  N)rÁ   rð   r  rñ   r  rh   rs   Úlinear_leftÚlinear_rightrè   r•   Úpost_laurel_normrX  s   &&€rQ   rð   ÚGemma3nTextLaurelBlock.__init__  sŠ   ø€ Ü‰ÑÔØŒäŸ9š9 T§[¡[×%<Ñ%<¸d¿k¹k×>UÑ>UÐ\aÔbˆÔÜŸIšI d§k¡k×&=Ñ&=¸t¿{¹{×?VÑ?VÐ]bÔcˆÔÜ .¨t¯{©{×/FÑ/FÈDÏKÉK×LdÑLdÔ eˆÖrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   r�    s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;rc   c                óx   € V P                  V4      pV P                  V4      pV P                  V4      pW,           # r  )rŸ  r   r¡  )rK   r÷   Úlaurel_hidden_statesÚnormed_laurel_hidden_statess   &&  rQ   r
  ÚGemma3nTextLaurelBlock.forward  s@   € Ø-1×-=Ñ-=¸mÓ-LÐØ-1×->Ñ->Ð?SÓ-TÐØ&*×&;Ñ&;Ð<PÓ&QÐ#ØÕ:Ð:rc   )r  rŸ  r   r¡  r  rÒ   s   @@rQ   rš  rš    s#   ù‡ € Ù*÷fó f÷;÷ ;ð ;rc   rš  c                   óf   a a€ ] tR tRt oR	V3R lV 3R llltV3R lR ltV3R lR ltRtVtV ;t	# )
ÚGemma3nTextMLPi  c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©re   r  Ú	layer_idx©r1   rt   )ry   rz   s   "€rQ   r{   ÚGemma3nTextMLP.__annotate__  s    ø€ ÷ Qñ QÑ0ð Q¹Sñ Qrc   c                ó†   <€ \         SV `  V4       VP                  V,          V n        VP                  V,          V n        R # r  )rÁ   rð   rC   rI   Úactivation_sparsity©rK   r  r¬  rÂ   s   &&&€rQ   rð   ÚGemma3nTextMLP.__init__  s6   ø€ Ü‰Ñ˜Ô Ø!'×!9Ñ!9¸)Õ!DˆÔØ#)×#EÑ#EÀiÕ#PˆÖ rc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   r®     s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rc   c                óÞ   € V P                  V4      pV P                  R 8”  d   V P                  V4      pV P                  V4      pV P	                  V4      pV P                  W4,          4      pV# )r@   )Ú	gate_projr°  Ú_gaussian_topkÚact_fnÚup_projÚ	down_proj)rK   r÷   rµ  Úactivationsr¸  r¹  s   &&    rQ   r
  ÚGemma3nTextMLP.forward   sa   € Ø—N‘N =Ó1ˆ	Ø×#Ñ# cÔ)Ø×+Ñ+¨IÓ6ˆIØ—k‘k )Ó,ˆØ—,‘,˜}Ó-ˆØ—N‘N ;Õ#8Ó9ˆ	ØÐrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )re   Úinputsr  rø   )ry   rz   s   "€rQ   r{   r®  )  s#   ø€ ÷ 5ñ 5¡U§\¡\ð 5±e·l±lñ 5rc   c                óò  € \         P                  ! V P                  \         P                  VP                  R 7      p\         P
                  P                  P                  ^ ^4      pVP                  V4      pVP                  VP                  4      p\         P                  ! VRRR7      p\         P                  ! VRRRR7      pWVV,          ,           p\        P                  P                  W,
          4      # )©r/  r2  Tré  F)rê   rû   Úunbiasedrü   )r×   r|  r°  r5  r2  ÚdistributionsÚnormalÚNormalÚicdfr9  r/  rÿ   Ústdrñ   rH  Úrelu)rK   r½  Útarget_sparsity_tensorÚnormal_distÚstd_multiplierÚinputs_meanÚ
inputs_stdÚcutoff_xs   &&      rQ   r¶  ÚGemma3nTextMLP._gaussian_topk)  s¶   € Ü!&§¢¨d×.FÑ.FÌeÏmÉmÐdj×dqÑdqÔ!rÐô ×)Ñ)×0Ñ0×7Ñ7¸¸1Ó=ˆØ'2×'7Ñ'7Ð8NÓ'OˆØ'×,Ñ,¨V¯\©\Ó:ˆÜ—j’j ¨R¸Ô>ˆÜ—Y’Y˜v¨2°tÀeÔLˆ
Ø¨nÕ!<Õ<ˆÜ�}‰}×!Ñ! &Õ"3Ó4Ð4rc   )r°  rC   )r   )
r}   r~   r   r€   rð   r
  r¶  r‰   rŠ   rÑ   rÒ   s   @@rQ   r©  r©    s*   ù‡ € ÷Qõ Q÷
ð ÷5÷ 5ð 5rc   r©  c                   óœ   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltV3R lR ltV3R	 lR
 ltV3R lR lt	V3R lR lt
RtVtV ;t# )ÚGemma3nTextAltUpi:  a…  Alternating Updates (AltUp)

The AltUp module wraps transformer layers. The `predict` step modifies the
input to the transformer layer, and the `correct` step propagates the output
of the transformer layer to the sparsely updated dimensions.

See more in the research paper:

https://proceedings.neurips.cc/paper_files/paper/2023/file/f2059277ac6ce66e7e5543001afa8bb5-Paper-Conference.pdf
c                ó    <€ V ^8„  d   QhRS[ /# r  rœ  )ry   rz   s   "€rQ   r{   ÚGemma3nTextAltUp.__annotate__F  s   ø€ ÷ kñ kÑ0ñ krc   c                ót  <€ \         SV `  4        Wn        \        P                  ! \
        P                  ! V P                  P                  4      4      V n        \        P                  ! V P                  P                  V P                  P                  R R7      V n        \        P                  ! V P                  P                  V P                  P                  ^,          R R7      V n        \        P                  ! V P                  P                  V P                  P                  R R7      V n        \        V P                  P                  V P                  P                  R7      V n        \        P"                  ! \
        P$                  ! V P                  P                  R,          4      R R7      V n        R# )Fr  rv  r  Nç      ð¿)rÁ   rð   r  rñ   rò   r×   rv  rh   Úcorrect_output_scaler  rq   Úcorrection_coefsÚprediction_coefsÚmodality_routerrè   r•   Úrouter_normr%  r|  Úrouter_input_scalerX  s   &&€rQ   rð   ÚGemma3nTextAltUp.__init__F  s  ø€ Ü‰ÑÔØŒÜ$&§L¢L´·²¸T¿[¹[×=TÑ=TÓ1UÓ$VˆÔ!Ü "§	¢	¨$¯+©+×*FÑ*FÈÏÉ×HdÑHdÐkpÔ qˆÔÜ "§	¢	¨$¯+©+×*FÑ*FÈÏÉ×HdÑHdÐfgÕHgÐnsÔ tˆÔÜ!Ÿyšy¨¯©×)@Ñ)@À$Ç+Á+×B^ÑB^ÐejÔkˆÔÜ)¨$¯+©+×*AÑ*AÀtÇ{Á{×G_ÑG_Ô`ˆÔÜ"$§)¢)¬E¯LªL¸¿¹×9PÑ9PÐRVÕ9VÓ,WÐdiÔ"jˆÖrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )re   r�  r  rø   )ry   rz   s   "€rQ   r{   rÑ  P  s#   ø€ ÷ 5ñ 5©5¯<©<ð 5¹E¿L¹Lñ 5rc   c                óÎ   € V P                  V4      V P                  ,          pV P                  V4      p\        P                  ! VP                  4       4      P                  V4      # r  )rØ  rÙ  r×  r×   rº  rw   r  )rK   r�  Úrouter_inputsÚrouteds   &&  rQ   Úcompute_router_modalitiesÚ*Gemma3nTextAltUp.compute_router_modalitiesP  sM   € Ø×(Ñ(¨Ó+¨d×.EÑ.EÕEˆØ×%Ñ% mÓ4ˆÜ�zŠz˜&Ÿ,™,›.Ó)×1Ñ1°!Ó4Ð4rc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rø   )ry   rz   s   "€rQ   r{   rÑ  U  s#   ø€ ÷ ?ñ ?¡U§\¡\ð ?±e·l±lñ ?rc   c                ó   € V P                  WP                  P                  ,          4      pV P                  '       dr   V P                  P                  eZ   V P
                  P                  P                  P                  V P                  P                  ) V P                  P                  4       V P                  V4      P                  ! . VP                  RR OV P                  P                  NV P                  P                  N5!  P                  ^ ^^^4      p\        P                  ! VP                  ^^^^ 4      V4      pVP                  ^^ ^^4      pWA,          pVP                  4       P!                  V4      # )a…  Predicts the output of a layer using a trainable map.

Args:
    hidden_states: A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` derived by
        stacking the input embeddings and preprocessing the last `num_altup_inputs - 1` matrices.

Returns:
    A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` containing the predictions.
Nrü   )rß  r  rn   Útrainingro   rÖ  rô   ÚdataÚclamp_rJ  rY  rq   r[  r×   r\  rœ  r  )rK   r÷   Ú
modalitiesÚ	all_coefsÚpredictionss   &&   rQ   ÚpredictÚGemma3nTextAltUp.predictU  sA  € ð ×3Ñ3°MÇ+Á+×B^ÑB^Õ4_Ó`ˆ
à�=�=ˆ=˜TŸ[™[×8Ñ8ÒDØ×!Ñ!×(Ñ(×-Ñ-×4Ñ4°d·k±k×6QÑ6QÐ5QÐSW×S^ÑS^×SnÑSnÔoð ×!Ñ! *Ó-ß‰Wðð iØ ×&Ñ& s¨Ð+ðiØ-1¯[©[×-IÑ-IðiØKOÏ;É;×KgÑKgóiç‰W�Q˜˜1˜aÓ ð 	ô —l’l =×#8Ñ#8¸¸A¸qÀ!Ó#DÀiÓPˆØ!×)Ñ)¨!¨Q°°1Ó5ˆØÕ$ˆØ×%Ñ%Ó'×/Ñ/°Ó>Ð>rc   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )re   rè  Ú	activatedr  rø   )ry   rz   s   "€rQ   r{   rÑ  q  s.   ø€ ÷ 9ñ 9¡5§<¡<ð 9¹E¿L¹Lð 9ÉUÏ\É\ñ 9rc   c                ó   € V P                  V4      pW!V P                  P                  ,          ,
          pVP                  V P                  P                  ^^^4      pV P
                  '       d›   V P                  P                  eƒ   V P                  P                  P                  V P                  P                  ) V P                  P                  4      p\        P                  P                  P                  W5RR7      R,           pMV P                  V4      R,           pVP                  ^^ ^4      P                  R4      p\        P                   ! WF4      pWq,          pVP#                  4       P%                  V4      # )a  Corrects the predictions relative to the

Args:
    predictions: A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` derived by
        stacking the input embeddings and preprocessing the last `num_altup_inputs - 1` matrices.
    activated: A 3D tensor of shape `[batch_size, num_tokens, hidden_size]` containing the activated inputs.

Returns:
    A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` correcting the original
        predictions relative to the activated input embeddings.
Nr  r  rü   )rß  r  rn   Úrepeatrq   rã  ro   rÕ  rô   rí  r×   rñ   rH  Úlinearr[  r&  Úmulrœ  r  )rK   rè  rì  ræ  Ú
innovationrô   rç  Ú	correcteds   &&&     rQ   ÚcorrectÚGemma3nTextAltUp.correctq  s&  € ð ×3Ñ3°IÓ>ˆ
Ø¨T¯[©[×-IÑ-IÕ!JÕJˆ
Ø×&Ñ& t§{¡{×'CÑ'CÀQÈÈ1ÓMˆ
à�=�=ˆ=˜TŸ[™[×8Ñ8ÒDØ×*Ñ*×1Ñ1×7Ñ7¸¿¹×9TÑ9TÐ8TÐVZ×VaÑVa×VqÑVqÓrˆFÜŸ™×+Ñ+×2Ñ2°:ÈDÐ2ÓQÐTWÕW‰Ià×-Ñ-¨jÓ9¸CÕ?ˆIð
 ×%Ñ% a¨¨AÓ.×8Ñ8¸Ó<ˆ	ä—I’I˜jÓ4ˆ	ØÕ ˆ	Ø×#Ñ#Ó%×-Ñ-¨iÓ8Ð8rc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©re   rò  r  rø   )ry   rz   s   "€rQ   r{   rÑ  �  s'   ø€ ÷ mñ m¡§¡ð m±%·,±,ñ mrc   c                óx   € VP                  V P                  4      V P                  ,          P                  V4      # )zé
This is only defined as the `forward` so that accelerate hooks can move correctly `correct_output_scale`
(which is a nn.Parameter, not a Module) between devices when offloading. It is otherwise only used in
`scale_corrected_output`
)r  rÔ  ©rK   rò  s   &&rQ   r
  ÚGemma3nTextAltUp.forward�  s2   € ð ×!Ñ! $×";Ñ";Ó<¸t×?XÑ?XÕX×aÑaÐbkÓlÐlrc   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rö  rø   )ry   rz   s   "€rQ   r{   rÑ  ˜  s#   ø€ ÷ 'ñ '±·±ð 'ÁÇÁñ 'rc   c                ó$   € V P                  V4      # )zMScales the provided 3D tensor of shape [batch_size, num_tokens, hidden_size].)r
  rø  s   &&rQ   Úscale_corrected_outputÚ'Gemma3nTextAltUp.scale_corrected_output˜  s   € à�|‰|˜IÓ&Ð&rc   )r  rÔ  rÕ  r×  rÖ  rÙ  rØ  )r}   r~   r   r€   r�   rð   rß  ré  ró  r
  rü  r‰   rŠ   rÑ   rÒ   s   @@rQ   rÏ  rÏ  :  sO   ù‡ € ñ	÷kó k÷5ð 5÷
?ð ?÷89ð 9÷>mð m÷'÷ 'ð 'rc   rÏ  c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        /# )re   r�  r8  r7  Úunsqueeze_dimrE  )ry   s   "rQ   r{   r{   �  s5   € ÷ .ñ .œEŸL™Lð .¬u¯|©|ð .Ä%Ç,Á,ð .Ô_bñ .rc   c                ó„   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           # )a$  Applies Rotary Position Embedding to the query and key tensors.

Args:
    x (`torch.Tensor`): The tensor to embed.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)r&  r"   )r�  r8  r7  rÿ  s   &&&&rQ   Úapply_rotary_pos_embr  �  s6   € ð" �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�Gœ A›¨Õ,Õ-Ð-rc   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3nTextAttentioni³  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r«  r­  )ry   rz   s   "€rQ   r{   Ú!Gemma3nTextAttention.__annotate__´  s   ø€ ÷ .
ñ .
Ñ0ð .
¹Sñ .
rc   c                ó„  <€ \         SV `  4        Wn        W n        \	        VR 4      '       d   VP
                  V,          MRV n        V P                  R8H  V n        V P                  '       d   VP                  MRV n        \        VRVP                  VP                  ,          4      V n        VP                  VP                  ,          V n        RV n        V P                  P                   V n        RV n        V P                  P$                  V P                  P&                  ,
          pY#u;8¬  ;'       d    ^ 8„  Mu V n        VP
                  RV pV P(                  '       dQ   \+        V4      ^,
          VRRR
1,          P-                  VP
                  V,          4      ,
          V n        RV n        MRRV n        V\+        V4      ^,
          VRRR
1,          P-                  VP
                  V,          4      ,
          8H  V n        \2        P4                  ! VP                  VP                  V P                  ,          VP6                  R7      V n        \;        VP                  VP<                  R7      V n        V P(                  '       gè   \2        P4                  ! VP                  VP                  V P                  ,          VP6                  R7      V n         \2        P4                  ! VP                  VP                  V P                  ,          VP6                  R7      V n!        \;        VP                  VP<                  R7      V n"        \;        VP                  VP<                  RR	7      V n#        \2        P4                  ! VP                  V P                  ,          VP                  VP6                  R7      V n$        R# )rG   Nr?   r  r  TFr  )rê   rë   )rê   rë   rì   rü   )%rÁ   rð   r  r¬  ÚhasattrrG   Ú
layer_typeÚ
is_slidingrl   Úgetattrrh   Únum_attention_headsr  rj   Únum_key_value_groupsÚscalingÚattention_dropoutÚ	is_causalrE   rr   Úis_kv_shared_layerrD   ÚindexÚkv_shared_layer_indexÚstore_full_length_kvrñ   r  Úattention_biasrx  rè   r•   Úq_normry  rz  Úk_normÚv_normÚo_proj)rK   r  r¬  Úfirst_kv_shared_layer_idxÚprev_layersrÂ   s   &&&  €rQ   rð   ÚGemma3nTextAttention.__init__´  sÉ  ø€ Ü‰ÑÔØŒØ"ŒÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŸ/™/Ð-@Ñ@ˆŒØ7;··°˜f×3Ò3ÈDˆÔä ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!ØˆŒØ!%§¡×!>Ñ!>ˆÔØˆŒà$(§K¡K×$AÑ$AÀDÇKÁK×DdÑDdÕ$dÐ!Ø"+×"LÔ"LÈ1Ô"LˆÔØ×(Ñ(Ð)CÐ*CÐDˆØ×"×"Ð"ä),¨[Ó)9¸AÕ)=ÀÉDÈbÈDÕ@Q×@WÑ@WÐX^×XjÑXjÐktÕXuÓ@vÕ)vˆDÔ&Ø(-ˆDÕ%à)-ˆDÔ&à(1´S¸Ó5EÈÕ5IÈKÑX\ÐZ\ÐX\ÕL]×LcÑLcØ×"Ñ" 9Õ-óMõ 6ñ )ˆDÔ%ô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô %¨¯©¸f×>QÑ>QÔRˆŒð ×&×&Ð&ÜŸ)š)Ø×"Ñ" F×$>Ñ$>ÀÇÁÕ$NÐU[×UjÑUjôˆDŒKô Ÿ)š)Ø×"Ñ" F×$>Ñ$>ÀÇÁÕ$NÐU[×UjÑUjôˆDŒKô )¨V¯_©_À&×BUÑBUÔVˆDŒKÜ(¨V¯_©_À&×BUÑBUÐbgÔhˆDŒKä—i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽrc   c                óZ  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[S[S[ P                  S[ P                  3,          3,          R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )	re   r÷   Úposition_embeddingsÚattention_maskNÚpast_key_valuesÚshared_kv_statesrL   r  )r×   rù   r   r¶   rt   r¤   r   r   )ry   rz   s   "€rQ   r{   r  ä  s¤   ø€ ÷ ;)ñ ;)á—|‘|ð;)ñ #Ÿ\™\ð;)ñ Ÿ™ tÕ+ð	;)ñ
  �ð;)ñ ™s¡E©%¯,©,¹¿¹Ð*DÕ$EÐEÕFÈÕMð;)ñ Ñ+Õ,ð;)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ;)rc   c           	     óî  € VP                   R R p. VORNV P                  P                  N5pVw  ršV P                  V4      P	                  V4      pV P                  V4      p\        W¹V
^R7      pVP                  ^^4      pV P                  '       dL   WPP                  ,          w  rÍVP                  VP                  4      pVP                  VP                  4      pM”V P                  V4      P	                  V4      pV P                  V4      p\        WÉV
^R7      pVP                  ^^4      pV P                  V4      P	                  V4      pV P                  V4      pVP                  ^^4      pVe1   V P                  '       g   VP!                  WÍV P"                  4      w  rÍV P$                  '       d   WÍ3WPP"                  &   \&        P(                  ! V P                  P*                  \,        4      pT! V VVVV3RV P.                  '       d   V P0                  MRRV P2                  RV P4                  /VB w  ppVP6                  ! . VORN5!  P9                  4       pV P;                  V4      pVV3# )N)rÿ  Údropoutr@   r  rl   rü   )rY  r  r  rx  r¸  r  r  Ú	transposer  r  r4  r2  ry  r  rz  r  r[   r¬  r  r   Úget_interfaceÚ_attn_implementationr!   rã  r  r  rl   rJ  rœ  r  )rK   r÷   r  r  r  r   rL   Úinput_shapeÚhidden_shaper8  r7  rÁ  rÂ  rÃ  Úattention_interfaceÚattn_outputÚattn_weightss   &&&&&&,          rQ   r
  ÚGemma3nTextAttention.forwardä  sS  € ð $×)Ñ)¨#¨2Ð.ˆØ?˜Ð? bÐ?¨$¯+©+×*>Ñ*>Ñ?ˆà&‰ˆØ—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆÜ+¨L¸sÐRSÔTˆØ#×-Ñ-¨a°Ó3ˆð
 ×"×"Ð"Ø'7×8RÑ8RÕ'SÑ$ˆJà#Ÿ™ |×':Ñ':Ó;ˆJØ'Ÿ?™?¨<×+>Ñ+>Ó?‰LàŸ™ ]Ó3×8Ñ8¸ÓFˆJØŸ™ ZÓ0ˆJÜ-¨j¸sÐRSÔTˆJØ#×-Ñ-¨a°Ó3ˆJàŸ;™; }Ó5×:Ñ:¸<ÓHˆLØŸ;™; |Ó4ˆLØ'×1Ñ1°!°QÓ7ˆLàÒ&¨t×/F×/FÐ/FØ'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJØ×$×$Ð$Ø/9Ð/GÐŸ^™^Ñ,ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð /3¯m¯m¨m�D×*Ò*Àð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rc   )r  r  r  r  r  r	  r  ry  r  r¬  r  r  r  r  rx  r  rl   r  r  rz  ©NNrL  rÒ   s   @@rQ   r  r  ³  s   ù‡ € ÷.
ó .
÷`;)÷ ;)ò ;)rc   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3nTextDecoderLayeri"  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r«  r­  )ry   rz   s   "€rQ   r{   Ú$Gemma3nTextDecoderLayer.__annotate__#  s    ø€ ÷ cñ cÑ0ð c¹Sñ crc   c                ó  <€ \         SV `  W4       \        WR 7      V n        VP                  V n        \
        VP                  ,          V n        \        V4      V n	        \        V4      V n        \        W4      V n        \        P                  ! V P                   V P                  RR7      V n        \        P                  ! V P                  V P                   RR7      V n        \'        V P                   VP(                  R7      V n        R# ))r¬  Fr  rv  N)rÁ   rð   r©  Úmlpri   r
   Úhidden_activationr·  rÏ  Úaltuprš  Úlaurelr  Ú	self_attnrñ   r  rh   Úper_layer_input_gateÚper_layer_projectionrè   r•   Úpost_per_layer_input_normr±  s   &&&€rQ   rð   Ú Gemma3nTextDecoderLayer.__init__#  sÀ   ø€ Ü‰Ñ˜Ô+Ü! &Ô>ˆŒà+1×+MÑ+MˆÔ(Ü˜V×5Ñ5Õ6ˆŒä% fÓ-ˆŒ
Ü,¨VÓ4ˆŒÜ-¨fÓ@ˆŒÜ$&§I¢I¨d×.>Ñ.>À×@`Ñ@`ÐglÔ$mˆÔ!Ü$&§I¢I¨d×.NÑ.NÐPT×P`ÑP`ÐglÔ$mˆÔ!Ü)7¸×8HÑ8HÈf×NaÑNaÔ)bˆÖ&rc   c                óÆ  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[S[S[ P                  S[ P                  3,          3,          R,          RS[ P                  R,          RS[ P
                  R,          RS[R,          R	S[S[,          R
S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /	# )re   r÷   r  Úper_layer_inputr   Nr  Úposition_idsr  rL   r  )
r×   rù   r¶   rt   r¤   Ú
LongTensorr   r   r   rà   )ry   rz   s   "€rQ   r{   r0  1  sÙ   ø€ ÷ 3%ñ 3%á—|‘|ð3%ñ #Ÿ\™\ð3%ñ Ÿ™ð	3%ñ
 ™s¡E©%¯,©,¹¿¹Ð*DÕ$EÐEÕFÈÕMð3%ñ Ÿ™ tÕ+ð3%ñ ×&Ñ&¨Õ-ð3%ñ  �ð3%ñ Ñ+Õ,ð3%ñ 
‰u�|‰|™U¡5×#4Ñ#4±e×6GÑ6GÐ#GÕHÈ4ÕOÐOÕ	Pñ3%rc   c                óÄ  € V P                   P                  V4      p	W�P                  P                  ,          p
V P	                  V
4      pV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rÞV P                  V4      pW­,           pWü,           \        P                  ! ^4      ,          pV P                  V4      pV P                  V4      pV P                  V4      pVV,           pV P                   P                  V	V4      pVV P                  P                  ,          P                  4       pV P                  P                  '       d   V P                   P!                  V4      pV P#                  V4      pV P%                  V4      p\&        P(                  ! VV4      pV P+                  V4      pV P-                  V4      pVR;;,          V,          uu&   V# )r÷   r  r   r=  r  r  :r1  NNrA   )r4  ré  r  rn   Úinput_layernormr5  r6  Úpost_attention_layernormr!  ÚsqrtÚpre_feedforward_layernormr2  Úpost_feedforward_layernormró  r}  rp   rü  r7  r·  r×   Úmultiplyr8  r9  )rK   r÷   r  r<  r   r  r=  r  rL   rè  Úactive_predictionÚactive_prediction_normedÚlaurel_outputrU  r]  Ú
attn_gatedÚattn_laurelÚ	attn_normÚattn_ffwÚattn_ffw_normÚattn_ffw_laurel_gatedÚcorrected_predictionsÚfirst_predictions   &&&&&&&&,              rQ   r
  ÚGemma3nTextDecoderLayer.forward1  sÕ  € ð —j‘j×(Ñ(¨Ó7ˆØ'¯©×(DÑ(DÕEÐà#'×#7Ñ#7Ð8IÓ#JÐ ØŸ™Ð$<Ó=ˆà—.’.ñ 
Ø2ð
à)ð
ð .ð
ð &ð	
ð
 !4ð
ð ,ð
ð ñ
‰ˆð ×,Ñ,¨TÓ2ˆà&Õ-ˆ
Ø!Õ1´T·Y²Y¸q³\ÕAˆà×2Ñ2°;Ó?ˆ	Ø—8‘8˜IÓ&ˆØ×7Ñ7¸ÓAˆØ +¨mÕ ;ÐØ $§
¡
× 2Ñ 2°;Ð@UÓ VÐà0°·±×1MÑ1MÕN×TÑTÓVÐØ�;‰;×*×*Ð*Ø#Ÿz™z×@Ñ@ÐAQÓRÐð  ×4Ñ4Ð5EÓFÐØŸ;™;Ð'7Ó8ÐÜ Ÿ>š>Ð*:¸OÓLÐð  ×4Ñ4Ð5EÓFÐØ×9Ñ9Ð:JÓKÐØ˜b×!Ð%5Õ5Ó!à$Ð$rc   )	r·  r4  ri   r5  r2  r7  r8  r9  r6  )NNNNNNrL  rÒ   s   @@rQ   r.  r.  "  s    ù‡ € ÷có c÷3%÷ 3%ò 3%rc   r.  c                   ó¼   a a€ ] tR tRt oRtRR.tR.tR]R]/t	]
P                  ! 4       R 4       tR tR	 tRV3R
 lV 3R llltRV3R lR lltV3R ltRtVtV ;t# )ÚGemma3nPreTrainedModelig  r  r   r.  r÷   Ú
attentionsc                óP	  € \         P                  ! W4       \        V\        4      '       d#   \        P
                  ! VP                  4       EM\        V\        4      '       dû   \        P                  ! VP                  4       VP                  R,          pR\        P                  P                  P                  \        P                  ! R4      4      ,          p\        P                   ! VP"                  W#,          4       \        P$                  ! VP&                  VP(                  4       \        P                   ! VP*                  VP-                  4       4       EMþ\        V\.        4      '       d.   \        P$                  ! VP0                  VP2                  4       EM»\        V\4        4      '       d_   \        P                  ! VP6                  4       \        P$                  ! VP8                  V P:                  P<                  R,          4       EMG\        V\>        4      '       dæ   RRrTVP@                  ^,          p\B        PD                  ! \G        V4      \G        V4      ,          4      \I        V^,
          ^4      ,          pV\        PJ                  ! \        PL                  ! V4      V) ,          4      ,          p\        P                   ! VPN                  VPG                  4       PQ                  ^ 4      PQ                  ^ 4      4       EML\        V\R        4      '       dp   \        P$                  ! VPT                  V P<                  R,          4       \        P$                  ! VPV                  ^\B        PX                  ! R4      ,          4       MÇ\        V\Z        4      '       d²   VP\                   F¡  p	VP^                  p
VP`                  V	,          R8w  d   \b        VP`                  V	,          ,          p
V
! VP:                  V	R7      w  r¼\        P                   ! \e        W R24      V4       \        P                   ! \e        W R24      V4       K£  	  \g        VR	4      '       d8   \        P$                  ! VPh                  V P:                  Ph                  4       R
# R
# )r�   r  r@   r8   ç       @rV   )r  Ú	_inv_freqÚ_original_inv_freqr–   Nrý   rÓ  )5r   Ú_init_weightsrB   rß  ÚinitÚones_rô   rl  Úzeros_rw  r  r×   rñ   rH  r{  r|  Úcopy_r  Ú	constant_r‚  rs  r�  r€  r–  Úembed_scaleÚscalar_embed_scalerÏ  rÔ  rÙ  r  rh   r  r  r!  r"  rw   r  r#  r$  r'  r&  ÚGemma3nTextModelÚper_layer_projection_scaleÚper_layer_input_scalerB  ÚGemma3nRotaryEmbeddingrG   Úcompute_default_rope_parametersrU   r   r
  r  r–   )rK   Úmoduler  rƒ  r(  r)  r*  r+  r'  r  Úrope_init_fnÚcurr_inv_freqr]  s   &&           rQ   rY  Ú$Gemma3nPreTrainedModel._init_weightsq  s  € ä×%Ò% dÔ3Ü�fÔ=×>Ò>Ü�JŠJ�v—}‘}Ö%Ü˜Ô 5×6Ò6Ü�KŠK˜×,Ñ,Ô-Ø—o‘o tÕ+ˆGØ¤§¡×!4Ñ!4×!=Ñ!=¼e¿lºlÈ3Ó>OÓ!PÕPˆLÜ�JŠJ�v—~‘~ wÕ'=Ô>Ü�NŠN˜6Ÿ>™>¨6×+KÑ+KÔLÜ�JŠJ�v×5Ñ5°v×7\Ñ7\Ó7^Ö_Ü˜Ô >×?Ò?Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÖIÜ˜Ô 0×1Ò1Ü�KŠK˜×3Ñ3Ô4Ü�NŠN˜6×4Ñ4°d·k±k×6MÑ6MÈtÕ6SÖTÜ˜Ô E×FÒFØ+.°˜=Ø#Ÿ_™_°Õ1ˆNÜ&*§h¢h¬u°]Ó/CÄeÈMÓFZÕ/ZÓ&[Ô^aØ Õ" Aó_õ 'Ð#ð +¬U¯YªY´u·|²|ÀNÓ7SÐWnÐVnÕ7nÓ-oÕoˆNÜ�JŠJ�v×,Ñ,¨n×.BÑ.BÓ.D×.NÑ.NÈqÓ.Q×.[Ñ.[Ð\]Ó.^Ö_Ü˜Ô 0×1Ò1Ü�NŠN˜6×<Ñ<¸d×>NÑ>NÐPTÕ>TÔUÜ�NŠN˜6×7Ñ7¸¼T¿YºYÀs»^Õ9KÕLÜ˜Ô 6×7Ò7Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ñ 1ô �6Ð.×/Ò/Ü�NŠN˜6×3Ñ3°T·[±[×5RÑ5RÖSñ 0rc   c                ó.   € V P                   P                  # r  ©Ú
base_modelÚembed_tokens_per_layer©rK   s   &rQ   Úget_per_layer_input_embeddingsÚ5Gemma3nPreTrainedModel.get_per_layer_input_embeddings™  s   € Ø�‰×5Ñ5Ð5rc   c                ó&   € WP                   n        R # r  rk  ©rK   r#  s   &&rQ   Úset_per_layer_input_embeddingsÚ5Gemma3nPreTrainedModel.set_per_layer_input_embeddingsœ  s   € Ø16�‰Ö.rc   c                ób   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[RS[P                  /# )re   Únew_num_tokensNÚpad_to_multiple_ofÚmean_resizingr  )rt   rx   rñ   Ú	Embedding)ry   rz   s   "€rQ   r{   Ú#Gemma3nPreTrainedModel.__annotate__Ÿ  s?   ø€ ÷ ñ á˜d�
ðñ   $�Jðñ ð	ñ
 
�‰ñrc   c                óP   <€ \         SV `  VVVR 7      pV P                  WV4       V# ))rv  rw  rx  )rÁ   Úresize_token_embeddingsÚ_resize_per_layer_embeddings)rK   rv  rw  rx  Úinputs_embedsrÂ   s   &&&& €rQ   r|  Ú.Gemma3nPreTrainedModel.resize_token_embeddingsŸ  s:   ø€ ô ™Ñ7Ø)Ø1Ø'ð 8ó 
ˆð
 	×)Ñ)¨.ÈmÔ\ØÐrc   c                óH   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[/# )re   rv  Nrw  rx  )rt   rx   )ry   rz   s   "€rQ   r{   rz  ­  s6   ø€ ÷ Jñ Já˜d�
ðJñ   $�JðJñ ñ	Jrc   c                ó²  € V P                   V P                  P                  4       n        V P                  P                  4       P                  '       d„   V P                  4       pV P                  WAW#4      p\        VR 4      '       d   VP                  p\        WV4       VP                  VP                  P                  4       V P                  V4       R# R# )Ú_hf_hookN)rf   r  Úget_text_configrg   ri   ro  Ú_get_resized_embeddingsr  r‚  r.   Úrequires_grad_rô   rï   rs  )rK   rv  rw  rx  rm  Únew_embeddings_per_layerÚhooks   &&&&   rQ   r}  Ú3Gemma3nPreTrainedModel._resize_per_layer_embeddings­  s²   € ð DHÇ?Á?ˆ�‰×#Ñ#Ó%Ô@Ø�;‰;×&Ñ&Ó(×D×DÐDØ%)×%HÑ%HÓ%JÐ"Ø'+×'CÑ'CØ&Ð8Jó(Ð$ô Ð-¨z×:Ò:Ø-×6Ñ6�Ü"Ð#;ÔBØ$×3Ñ3Ð4J×4QÑ4Q×4_Ñ4_Ô`Ø×/Ñ/Ð0HÖIñ Erc   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r  ©r¹   )ry   rz   s   "€rQ   r{   rz  g  s   ø‡ ‚ ÙÑò rc   rA   )ÚimageÚtextÚaudio)NNT)r}   r~   r   r€   Úinput_modalitiesÚ_skip_keys_device_placementÚ_no_split_modulesr.  r  Ú_can_record_outputsr×   Úno_gradrY  ro  rs  r|  r}  rˆ   r‰   rŠ   rÑ   rÒ   s   @@rQ   rS  rS  g  sy   ù‡ € à1ÐØ#4Ð6HÐ"IÐØ2Ð3ÐàÐ0ØÐ*ðÐð
 ‡]‚]ƒ_ñ%Tó ð%TòN6ò7÷õ ÷Jò J÷M … rc   rS  c                   ó|   a a€ ] tR tRt oRtRtRtV3R lV 3R llt]]	V3R lR l4       4       t
V3R	 ltR
tVtV ;t# )ÚGemma3nAudioEncoderiÀ  zp
An audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.
Ú	audio_melr�  c                ó    <€ V ^8„  d   QhRS[ /# r  r  )ry   rz   s   "€rQ   r{   Ú Gemma3nAudioEncoder.__annotate__Ê  s   ø€ ÷ ñ Ñ1ñ rc   c                ó
  <€ \         SV `  V4       Wn        \        V4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n
        V P                  4        R # u upi r  )rÁ   rð   r  r,  Úsubsample_conv_projectionrñ   Ú
ModuleListrH   rœ   r‡  Ú	conformerÚ	post_init)rK   r  r]  rÂ   s   && €rQ   rð   ÚGemma3nAudioEncoder.__init__Ê  si   ø€ Ü‰Ñ˜Ô ØŒä)LÈVÓ)TˆÔ&ÜŸšÜ9>¸v×?\Ñ?\Ô9]Ó^Ñ9]°AÔ'¨Ö/Ñ9]Ñ^ó
ˆŒð 	�‰Öùò _s   ÁB c          	      óz   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[S[,          /# )re   r•  rÖ   rL   r  )r×   rù   rØ   r   r   r¤   rÔ   )ry   rz   s   "€rQ   r{   r—  Ö  sC   ø€ ÷ 8
ñ 8
ÙŸ™ð8
Ù7<×7GÑ7Gð8
ÙSYÑZlÕSmð8
á	Ñ/Õ	/ñ8
rc   c                ó¾  € V P                  V4      pVP                  ^,          p^p\        \        V P                  P
                  4      4       F-  pW`P                  P
                  V,          ^ ,          ,          pK/  	  \        P                  ! WRP                  R7      V,          p\        P                  ! W‚P                  ^,          ^,
          R7      pVP                  ^8”  dE   VP                  ^8X  d4   VP                  ^ 4      P                  VP                  ^ ,          R4      pMsVP                  VP                  8X  dY   VP                  ^ ,          ^8X  dA   VP                  ^ ,          ^8w  d)   WXP                  ^ ,          8X  d   VP                  ^ 4      p\        P                  ! V^V4      p	V P                   F  p
V
! WI4      pK  	  V P                  P                  ^8”  dE   VRRRV P                  P                  13,          pV	RRRV P                  P                  13,          p	VP!                  V	P                  R4      R4      p\#        VV	R7      # )ad  Encodes a batch of MELs.

Args:
    audio_mel: a torch.Tensor of shape [batch, num_frames, num_channels,
      mel_bins].

Returns:
    audio_encodings: a torch.Tensor of shape
        `[batch_size, self.config.audio_soft_tokens_per_image,
        self.config.audio_config.hidden_size]`
    audio_mel_mask: a torch.BoolTensor of shape [batch, num_frames].
rW  )r  rG  Nr@   )Úlast_hidden_staterÖ   rü   )r™  rY  rH   rD   r  r£   r×   r$  r2  rí  r¬  r&  ÚexpandÚgatherr›  rž   Úmasked_fillrÔ   )rK   r•  rÖ   rL   r  Út_subÚtime_stride_productÚstride_pair_idxÚindicesÚcurrent_maskÚblocks   &&&,       rQ   r
  ÚGemma3nAudioEncoder.forwardÔ  sê  € ð" ×8Ñ8¸ÓCˆð  ×%Ñ% aÕ(ˆàÐÜ$¤S¨¯©×)JÑ)JÓ%KÖLˆOØ§;¡;×#DÑ#DÀ_Õ#UÐVWÕ#XÕXÒñ  Mô —,’,˜u×-BÑ-BÔCÐFYÕYˆÜ—+’+˜g×+?Ñ+?ÀÕ+BÀQÕ+FÔGˆð ×Ñ Ô" w§|¡|°qÔ'8Ø×'Ñ'¨Ó*×1Ñ1°.×2FÑ2FÀqÕ2IÈ2ÓN‰Gà×Ñ 7§<¡<Ô/Ø×$Ñ$ QÕ'¨1Ô,Ø—‘˜aÕ  AÔ%ØŸ™ qÕ)Ô)ð ×'Ñ'¨Ó*ˆGä—|’| N°A°wÓ?ˆà—^”^ˆEÙ# OÓBŠOñ $ð �;‰;×,Ñ,¨qÔ0Ø-¨aÑ1U°D·K±K×4UÑ4UÐ1UÐ.UÕVˆOà'¨Ñ+O¨d¯k©k×.OÑ.OÐ+OÐ(OÕPˆLà)×5Ñ5°l×6LÑ6LÈRÓ6PÐRUÓVˆÜ-Ø-Ø'ô
ð 	
rc   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r  r  )ry   rz   s   "€rQ   r{   r—  À  s   ø‡ ‚ ñ
 Ñò rc   )r  r›  r™  )r}   r~   r   r€   r�   Úmain_input_namerŽ  rð   r   r   r
  rˆ   r‰   rŠ   rÑ   rÒ   s   @@rQ   r”  r”  À  sD   ù‡ € ñð "€OØÐ÷ó ð  Ø÷8
ó ó  ð8
÷- … rc   r”  c                   ó   € ] tR tRtRtR# )rd  i  rA   Nr—  rA   rc   rQ   rd  rd    r˜  rc   rd  zBThe base Gemma 3n language model without a language modeling head.©Úcustom_introc                   ó´   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRV3R lR llt]]! RR	7      ]	RV3R
 lR ll4       4       4       t
V3R ltRtVtV ;t# )ra  i  c                ó    <€ V ^8„  d   QhRS[ /# r  rœ  )ry   rz   s   "€rQ   r{   ÚGemma3nTextModel.__annotate__  s   ø€ ÷ +ñ +Ñ0ñ +rc   c                ó¬  <€ \         SV `  V4       VP                  V n        VP                  V n        \	        VP
                  VP                  VP                  ,          V P                  VP                  R ,          R7      V n        \        P                  ! V P                  VP                  VP                  ,          RR7      V n        \        VP                  VP                  R7      V n        \        P                  ! \!        VP                  4       Uu. uF  p\#        W4      NK  	  up4      V n        \        VP                  VP                  R7      V n        \        P                  ! \!        ^V P(                  P*                  4       Uu. uF0  p\        P                  ! V P                  V P                  RR7      NK2  	  up4      V n        \        P                  ! \!        ^V P(                  P*                  4       Uu. uF0  p\        P                  ! V P                  V P                  RR7      NK2  	  up4      V n        \        P0                  ! \2        P4                  ! V P                  R
,          4      RR7      V n        \        P0                  ! \2        P8                  ! \2        P4                  ! R4      4      RR7      V n        . V n        \?        V P$                  4       FT  w  rEVP@                  PB                  '       g   K#  V P<                  PE                  R Uu. uF  pRV RV 2NK  	  up4       KV  	  R	# u upi u upi u upi u upi )r�   )r_  Fr  rv  r  rV  zlayers.z.self_attn.Nrý   )ry  rz  r  r  )#rÁ   rð   rh   ri   r–  rg   rE   Úpadding_idxrm  rñ   r  Úper_layer_model_projectionrè   r•   Úper_layer_projection_normrš  rH   r.  Úlayersr  r  rq   Úaltup_projectionsÚaltup_unembed_projectionsr%  r×   r|  rb  rî  rc  Ú"_keys_to_ignore_on_load_unexpectedÚ	enumerater6  r  Úextend)rK   r  r¬  r]  rN   ÚlayerÚnamerÂ   s   &&     €rQ   rð   ÚGemma3nTextModel.__init__  s„  ø€ Ü‰Ñ˜Ô à!×-Ñ-ˆÔØ+1×+MÑ+MˆÔ(ä&DØ×-Ñ-Ø×$Ñ$ v×'IÑ'IÕIØ×ÑØ×:Ñ:¸CÕ?ô	'
ˆÔ#ô +-¯)ª)Ø×ÑØ×$Ñ$ v×'IÑ'IÕIØô+
ˆÔ'ô *8¸×8ZÑ8ZÐ`f×`sÑ`sÔ)tˆÔ&Ü—m’mÜINÈv×OgÑOgÔIhÓiÑIh¸IÔ$ VÖ7ÑIhÑió
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	ä!#§¢ÜPUÐVWÐY]×YdÑYd×YuÑYuÔPvÓwÑPvÈ1ŒR�YŠY�t×'Ñ'¨×)9Ñ)9À×FÑPvÑwó"
ˆÔô *,¯ªÜPUÐVWÐY]×YdÑYd×YuÑYuÔPvÓwÑPvÈ1ŒR�YŠY�t×'Ñ'¨×)9Ñ)9À×FÑPvÑwó*
ˆÔ&ô +-¯)ª)´E·L²LÀ×AQÑAQÐSWÕAWÓ4XÐejÔ*kˆÔ'Ü%'§Y¢Y¬u¯{ª{¼5¿<º<ÈÓ;LÓ/MÐZ_Ô%`ˆÔ"ð 35ˆÔ/Ü! $§+¡+Ö.‰HˆAØ�‰×1×1Ò1Ø×7Ñ7×>Ñ>Ù@hÓiÑ@h¸�w˜q˜c ¨T¨FÓ3Ñ@hÑiöó /ùò% jùò xùò xùò js   ÄMÆ6MÈ6MÌ$M
c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )re   Ú	input_idsr  ©r×   r>  rù   )ry   rz   s   "€rQ   r{   r²  F  s%   ø€ ÷ 
ñ 
©e×.>Ñ.>ð 
Á5Ç<Á<ñ 
rc   c                óž   € V P                  V4      P                  ! . VP                  OV P                  P                  NV P
                  N5!  # r  )rm  rJ  rY  r  rE   ri   )rK   rÁ  s   &&rQ   Úget_per_layer_inputsÚ%Gemma3nTextModel.get_per_layer_inputsF  sN   € Ø×*Ñ*¨9Ó5×=Ò=ð 
Ø�_‰_ð
à�K‰K×)Ñ)ð
ð ×,Ñ,ó
ð 	
rc   c                óv   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  /# )re   r~  Úper_layer_inputsNr  rø   )ry   rz   s   "€rQ   r{   r²  M  s:   ø€ ÷ 
ñ 
á—|‘|ð
ñ  Ÿ,™,¨Õ-ð
ñ 
�‰ñ	
rc   c                ó:  € V P                  V4      pW0P                  P                  VP                  VP                  R 7      ,          pVP
                  ! . VP                  RR OV P                  P                  NV P                  N5!  pV P                  V4      pVf   V# VP                  VP                  8w  d#   VRRV P                  P                  1R3,          pW2,           V P                  P                  VP                  VP                  R 7      ,          # )r¿  N.rG  rü   )rµ  rb  r4  r/  r2  rJ  rY  r  rE   ri   r¶  rc  )rK   r~  rÇ  r8  s   &&& rQ   Úproject_per_layer_inputsÚ)Gemma3nTextModel.project_per_layer_inputsM  s&  € ð
 .2×-LÑ-LÈ]Ó-[ÐØ× ?Ñ ?× BÑ BØ×%Ñ%Ð.B×.IÑ.Ið !Có !
õ 	
Ðð  4×;Ò;ð  
Ø× Ñ   "Ð%ð 
à�K‰K×)Ñ)ð 
ð ×,Ñ,ó 
Ðð
  $×=Ñ=Ð>RÓSÐàÒ#Ø'Ð'à×%Ñ%Ð)9×)?Ñ)?Ô?à/°Ð5T°t·{±{×7TÑ7TÐ5TÐVWÐ0WÕXÐà$Õ7¸4×;UÑ;U×;XÑ;XØ×%Ñ%Ð.B×.IÑ.Ið <Yó <
õ 
ð 	
rc   F)Útie_last_hidden_statesc                ó&  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          R	S[S[,          R
S[/	# )re   rÁ  NrÇ  r  r=  r  r~  rÎ   rL   r  )	r×   r>  rù   r   rà   rx   r   r   r   )ry   rz   s   "€rQ   r{   r²  l  s±   ø€ ÷ m
ñ m
á×#Ñ# dÕ*ðm
ñ  Ÿ,™,¨Õ-ðm
ñ Ÿ™ tÕ+ð	m
ñ
 ×&Ñ&¨Õ-ðm
ñ  �ðm
ñ ×(Ñ(¨4Õ/ðm
ñ ˜$•;ðm
ñ Ñ+Õ,ðm
ñ 
!ñm
rc   c                ó¢	  € VRJ VRJ,          '       d   \        R4      hVe#   V P                  V4      pV P                  V4      pV P                  Wb4      pV'       d   Vf   \	        V P
                  R7      pVfb   Ve   VP                  4       M^ p	\        P                  ! VP                  ^,          VP                  R7      V	,           pVP                  ^ 4      p\        T;p
\        4      '       g/   RV P
                  RVRVRVR	V/pR
\        R/ VB R\        R/ VB /p
Tp\        P                   ! V^,          RRR7      R,          p\        P"                  ! R4      pV.p\%        ^V P
                  P&                  4       FÏ  pV P(                  V^,
          ,          ! V4      pVP+                  VP,                  VP                  R7      p\        P                   ! V^,          RRR7      p\        P.                  ! \        P0                  ! VVP+                  VP                  4      4      4      pVV,          V,          pVP3                  V4       KÑ  	  \        P4                  ! V^ R7      p/ p\7        V P
                  P8                  4       F  pV P;                  VVV4      VV&   K  	  \=        4       p\?        V P@                  RV P
                  PB                   4       Fk  w  ppW P
                  P8                  V,          ,          pVRRVR3,          pV! VVV P
                  P8                  V,          ,          V3RVRVR	VRV/VB pKm  	  \        P                   ! V^ ,          ^,          RRR7      R,          pV^ ,          .p\%        ^V P
                  P&                  4       FÖ  pV PD                  V^,
          ,          ! VV,          4      pVP+                  VP,                  VP                  R7      p\        P                   ! V^,          RRR7      p\        P.                  ! \        P0                  ! VVP+                  VP                  4      4      4      pVV,          V,          pVP3                  V4       KØ  	  \        P4                  ! V4      p\        P                   ! V^ R7      pV PG                  V4      p\I        VVR7      # )z�
per_layer_inputs (torch.Tensor, *optional*, defaults to None):
    Pre-computed per-layer embeddings. If None, they are derived from input_ids if provided.
Nú:You must specify exactly one of input_ids or inputs_embeds)r  rW  r  r~  r  r  r=  r>   r?   Tré  r�   gñhãˆµøä>r¿  r3  rG  r   )r   r  rA   rü   )%rF   Úembed_tokensrÄ  rÉ  r   r  Úget_seq_lengthr×   r$  rY  r2  r&  rB   r¶   r   r   rÿ   r|  rH   rq   r¸  r4  r/  rB  Úmaximumr0  ÚstackÚsetrG   Ú
rotary_embr   r»  r·  rE   r¹  r  r   )rK   rÁ  rÇ  r  r=  r  r~  rÎ   rL   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚhidden_states_0Útarget_magnitudeÚepsilon_tensorÚtemp_hidden_statesrN   Ú
altup_projÚcurrent_hidden_stateÚnew_magnituder÷   r  r  r   Údecoder_layerÚcausal_maskr<  Úaltup_unemb_projs   &&&&&&&&,                   rQ   r
  ÚGemma3nTextModel.forwardi  sY  € ð$ ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMØ#×8Ñ8¸ÓCÐà×8Ñ8¸ÓYÐç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð#Ðð (ˆô !Ÿ:š: o°qÕ&8¸bÈ$ÔOÐSVÕVÐÜŸš dÓ+ˆà-Ð.ÐÜ�q˜$Ÿ+™+×6Ñ6Ö7ˆAà×/Ñ/°°AµÖ6°ÓGˆJØ#-§=¡=°×7LÑ7LÐUe×UlÑUl =Ó#mÐ Ü!ŸJšJÐ';¸QÕ'>ÀBÐPTÔUˆMÜ!ŸJšJ¤u§}¢}°]ÀN×DUÑDUÐVf×VmÑVmÓDnÓ'oÓpˆMØ#7Ð:JÕ#JÈ]Õ#ZÐ Ø×%Ñ%Ð&:Ö;ñ 8ô ŸšÐ$6¸AÔ>ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ó+ñ 7ô $›:Ðä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAˆ}Ø-¯k©k×.EÑ.EÀaÕ.HÕIˆKØ.¨q°!°Q¸¨zÕ:ˆOá)ØØ# D§K¡K×$;Ñ$;¸AÕ$>Õ?Øñ	ð "2ð		ð
  +ð	ð *ð	ð !0ð	ð ñ	ŠMñ	 !Xô  !Ÿ:š: m°AÕ&6¸!Õ&;ÀÈTÔRÐVYÕYÐØ+¨AÕ.Ð/ÐÜ�q˜$Ÿ+™+×6Ñ6Ö7ˆAà-1×-KÑ-KÈAÐPQÍEÖ-RÐS`ÐabÕScÓ-dÐØ#3×#6Ñ#6¸_×=RÑ=RÐ[k×[rÑ[rÐ#6Ó#sÐ Ü!ŸJšJÐ';¸QÕ'>ÀBÐPTÔUˆMÜ!ŸJšJ¤u§}¢}°]ÀN×DUÑDUÐVf×VmÑVmÓDnÓ'oÓpˆMØ#7Ð:JÕ#JÈ]Õ#ZÐ Ø×%Ñ%Ð&:Ö;ñ 8ô ŸšÐ$6Ó7ˆÜŸ
š
 =°aÔ8ˆØŸ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
rc   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r  rœ  )ry   rz   s   "€rQ   r{   r²    s   ø‡ ‚ áÑò rc   )rº  r¸  r¹  rm  rh   ri   r·  r  rc  rµ  r¶  rb  r  )NNNNNNN)r}   r~   r   r€   rð   rÄ  rÉ  r   r   r   r
  rˆ   r‰   rŠ   rÑ   rÒ   s   @@rQ   ra  ra    s]   ù‡ € ÷+ó +÷Z
ð 
÷
ò 
ð8  Ù¨EÔ2Ø÷m
ñ m
ó ó 3ó  ðm
÷o … rc   ra  z?The base Gemma 3n language model with a language modeling head.c                   ó   € ] tR tRtRtR# )ÚGemma3nForCausalLMiÜ  rA   Nr—  rA   rc   rQ   rå  rå  Ü  s   † ãrc   rå  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚGemma3nMultimodalEmbedderiá  zQEmbeds token ids or soft tokens for multimodal content into language model space.c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# )re   Úmultimodal_configr»   )r�   r­   r1   )ry   rz   s   "€rQ   r{   Ú&Gemma3nMultimodalEmbedder.__annotate__ä  s)   ø€ ÷ tñ tá-Ñ0CÕCðtñ 'ñtrc   c                óx  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                  ! V P                  V P                  4      V n        \        V P                  V P
                  R 7      V n        \        V P                  V P
                  R 7      V n        \        P                  ! V P                  V P                  RR7      V n        \        V P                  V P
                  RR7      V n        R# )rv  Fr  )rë   rì   N)rÁ   rð   rh   Úmultimodal_hidden_sizer•   rë   r“   rf   Útext_hidden_sizerñ   ry  Ú	embeddingrè   Úhard_embedding_normÚsoft_embedding_normr  Úembedding_projectionÚembedding_post_projection_norm)rK   ré  r»   rÂ   s   &&&€rQ   rð   Ú"Gemma3nMultimodalEmbedder.__init__ä  sé   ø€ ô
 	‰ÑÔà&7×&CÑ&CˆÔ#Ø$×1Ñ1ˆŒØ-×:Ñ:ˆÔØ+×6Ñ6ˆŒØ +× 7Ñ 7ˆÔäŸš d§o¡o°t×7RÑ7RÓSˆŒÜ#1°$×2MÑ2MÐSW×S[ÑS[Ô#\ˆÔ Ü#1°$×2MÑ2MÐSW×S[ÑS[Ô#\ˆÔ Ü$&§I¢I¨d×.IÑ.IÈ4×K`ÑK`ÐglÔ$mˆÔ!Ü.<¸T×=RÑ=RÐX\×X`ÑX`ÐmrÔ.sˆÖ+rc   c                ó„   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  /# )re   rÁ  Nr~  r  rÂ  )ry   rz   s   "€rQ   r{   rê  ÷  sF   ø€ ÷ Bñ Bá×#Ñ# dÕ*ðBñ —|‘| dÕ*ðBñ 
�‰ñ	Brc   c                ó  € VRJ VRJ,          '       d   \        R4      hVe   V P                  V4      pM2V P                  WP                  ,
          4      pV P	                  V4      pV P                  V4      pV P                  V4      # )a­  Embeds token ids or soft tokens for multimodal content into language model space.

Args:
    input_ids: A torch.LongTensor containing the token ids to embed. Values should be in the range
        `[vocab_offset, vocab_offset + vocab_size)`.
    inputs_embeds: A torch.Tensor containing the soft tokens to embed.

Returns:
    A torch.Tensor of embeddings with  shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
NrÎ  )rF   rð  rî  r“   rï  rñ  rò  )rK   rÁ  r~  Úemb_normÚhard_embÚemb_norm_projs   &&&   rQ   r
  Ú!Gemma3nMultimodalEmbedder.forward÷  sƒ   € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ$Ø×/Ñ/°Ó>‰Hà—~‘~ i×2CÑ2CÕ&CÓDˆHØ×/Ñ/°Ó9ˆHà×1Ñ1°(Ó;ˆØ×2Ñ2°=ÓAÐArc   )
rî  rò  rñ  rë   rï  rì  rð  rí  r“   rf   r,  r  rÒ   s   @@rQ   rç  rç  á  s&   ù‡ € Ù[÷tó t÷&B÷ Bò Brc   rç  z�
    The base Gemma 3n model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c                   óè   a a€ ] tR tRt oV3R lV 3R lltR tR t]]! RR7      V3R lR	 l4       4       t	RV3R
 lR llt
]RV3R lR ll4       t]]! RR7      V3R lR l4       4       tRtVtV ;t# )ÚGemma3nModeli  c                ó    <€ V ^8„  d   QhRS[ /# r  rŠ  )ry   rz   s   "€rQ   r{   ÚGemma3nModel.__annotate__  s   ø€ ÷ ^ñ ^™}ñ ^rc   c                óB  <€ \         SV `  V4       V =V =VP                  P
                  V n        \        P                  ! VP                  4      V n	        \        VP                  VP                  4      V n        \        VP                  VP                  4      V n        R # r  )rÁ   rð   Úmulti_modal_projectorÚtext_config_dtyper»   rg   r   Úfrom_configr½   Úaudio_towerrç  r¼   Úembed_visionÚembed_audiorX  s   &&€rQ   rð   ÚGemma3nModel.__init__  s   ø€ Ü‰Ñ˜Ô ØÐ&ØÐ"Ø*0×*<Ñ*<×*WÑ*WˆÔ'Ü$×0Ò0°×1DÑ1DÓEˆÔÜ5°f×6JÑ6JÈF×L^ÑL^Ó_ˆÔÜ4°V×5HÑ5HÈ&×J\ÑJ\Ó]ˆÖrc   c                ó.   € V P                   P                  # r  ©Úlanguage_modelrm  rn  s   &rQ   ro  Ú+Gemma3nModel.get_per_layer_input_embeddings#  s   € Ø×"Ñ"×9Ñ9Ð9rc   c                ó&   € WP                   n        R # r  r  rr  s   &&rQ   rs  Ú+Gemma3nModel.set_per_layer_input_embeddings&  s   € Ø5:×ÑÖ2rc   zOProjects the last hidden state from the vision model into language model space.r®  c                ó`   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[,          /# )re   Úpixel_valuesrL   r  )r×   rà   r   r   r¤   r   )ry   rz   s   "€rQ   r{   rý  +  s;   ø€ ÷ ñ á×'Ñ'ðñ Ñ+Õ,ðñ 
Ñ+Õ	+ñ	rc   c           	     ó¤  € V P                   ! RR VRRRR/VB pVP                  pVP                  VP                  ^ ,          V P                  P
                  P                  V P                  P                  4      P                  ^ ^^4      pW@P                  P
                  P                  R,          ,          pV P                  VR7      Vn
        V# )r  r³   FÚreturn_dictTr�   ©r~  rA   )Úvision_towerr   rJ  rY  r  r¼   rh   rÆ   r[  r  Úpooler_output)rK   r  rL   Úvision_outputsr   s   &&,  rQ   Úget_image_featuresÚGemma3nModel.get_image_features)  sÅ   € ð ×*Ò*Ñs¸ÐsÐQVÐsÐdhÐsÐlrÑsˆØ*×<Ñ<Ðð .×5Ñ5Ø×#Ñ# AÕ&Ø�K‰K×%Ñ%×1Ñ1Ø�K‰K×4Ñ4ó
÷ ‰'�!�Q˜Ó
ð	 	ð 	Ÿ[™[×6Ñ6×BÑBÀCÕGÕGÐØ'+×'8Ñ'8ÐGXÐ'8Ó'YˆÔ$àÐrc   c          	      óº   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          /# )re   rÁ  Nr~  Úimage_featuresÚaudio_features)r×   r>  rà   )ry   rz   s   "€rQ   r{   rý  ?  s_   ø€ ÷ *6ñ *6á×#Ñ# dÕ*ð*6ñ ×(Ñ(¨4Õ/ð*6ñ ×)Ñ)¨DÕ0ð	*6ñ
 ×)Ñ)¨DÕ0ñ*6rc   c           
     ó¨  € VfÛ   W P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  pVP                  R4      pVV P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  P                  R4      pM0WP                  P                  8H  pWP                  P                  8H  pVP                  4       pVP                  R4      P                  VP                  4      pVed   \        WrP                  R,          ,          VP                  4       8H  RV RVP                  ^ ,          VP                  ^,          ,           24       VP                  4       pVP                  R4      P                  VP                  4      pVed   \        W‚P                  R,          ,          VP                  4       8H  RV RVP                  ^ ,          VP                  ^,          ,           24       WV3# )z×
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
equal to the length of multimodal features. If the lengths are different, an error is raised.
r¿  z6Image features and image tokens do not match, tokens: z, features: z6Audio features and audio tokens do not match, tokens: rA   rü   )Úget_input_embeddingsr×   Úfullr  rÉ   Úlongr2  ÚallrÌ   rë  r&  r4  r   rY  Únumel)	rK   rÁ  r~  r  r  Úspecial_image_maskÚspecial_audio_maskÚn_image_tokensÚn_audio_tokenss	   &&&&&    rQ   Úget_placeholder_maskÚ!Gemma3nModel.get_placeholder_mask?  s>  € ð ÒØ!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÔió3ñ "Ðð "4×!7Ñ!7¸Ó!;ÐàØ×,Ñ,Ô.Ü—J’J˜r 4§;¡;×#=Ñ#=ÄUÇZÁZÐXe×XlÑXlÔmóñ÷ ‰c�"‹gñ ð "+¯k©k×.HÑ.HÑ!HÐØ!*¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØÒ%Ü"Ø×!4Ñ!4°RÕ!8Õ8¸N×<PÑ<PÓ<RÑRØHÈÐHXÐXdÐes×eyÑeyÐz{Õe|ð  @N÷  @Tñ  @Tð  UVõ  @Wõ  fWð  eXð  Yôð
 ,×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØÒ%Ü"Ø×!4Ñ!4°RÕ!8Õ8¸N×<PÑ<PÓ<RÑRØHÈÐHXÐXdÐes×eyÑeyÐz{Õe|ð  @N÷  @Tñ  @Tð  UVõ  @Wõ  fWð  eXð  Yôð
 "Ð5Ð5rc   c                óÆ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[/# )re   rÁ  Nr  Úinput_featuresr  Úinput_features_maskr=  r  Útoken_type_idsr~  ÚlabelsrÎ   Ú	lm_kwargsr  )	r×   r>  rà   rù   r   rx   r   r   rÛ   )ry   rz   s   "€rQ   r{   rý  l  s	  ø€ ÷ F
ñ F
á×#Ñ# dÕ*ðF
ñ ×'Ñ'¨$Õ.ðF
ñ ×)Ñ)¨DÕ0ð	F
ñ
 Ÿ™ tÕ+ðF
ñ #Ÿ\™\¨DÕ0ðF
ñ ×&Ñ&¨Õ-ðF
ñ  �ðF
ñ ×(Ñ(¨4Õ/ðF
ñ ×(Ñ(¨4Õ/ðF
ñ × Ñ  4Õ'ðF
ñ ˜$•;ðF
ñ Ñ.Õ/ðF
ñ 
$ñF
rc   c                ó°	  € VRJ V	RJ,          '       d   \        R4      hVEes   V P                  4       ! V4      p	\        P                  ! V^ 8¬  WP                  8  4      p\        P
                  ! WÑ\        P                  ! V4      4      pV P                  P                  V4      p\        P                  ! WP                  P                  8¬  WP                  P                  8  4      pV P                  P                  V P                  P                  ,           ^,
          p\        P
                  ! VVV4      P                  V	P                  4      pV P                  VR7      pVP                  V	P                  V	P                  4      pVP!                  R4      p\        P
                  ! VVV	4      p	WP                  P                  8¬  pV P                  P                  V P                  P                  ,           ^,
          p\        P
                  ! VVV4      P                  V	P                  4      pV P                  VR7      pVP                  V	P                  V	P                  4      pVP!                  R4      p\        P
                  ! VVV	4      p	MRpVel   V P#                  VRR7      P$                  pVP                  V	P                  V	P                  4      pV P'                  WVR7      w  ppV	P)                  VV4      p	VEeW   VEeR   V P+                  W5( RR7      pVP$                  pVP,                  p\        P.                  ! V P                  ^,
          ..\        P0                  VP                  R7      pV P                  VR7      p \        P
                  ! VP!                  R4      V V4      pVP2                  w  p!p"p#V P4                  P6                  V",
          p$V P9                  V!V$V#4      p%\        P:                  ! VV%3^R7      pVP                  V	P                  V	P                  4      pV P'                  WVR	7      w  pp&V	P)                  V&V4      p	V P                  ! RR
RRVRVRVRVRV	RVRR/VB p'\=        V'P>                  V'       d   V'P@                  MRV'PB                  V'PD                  Ve   XMRVe   XR7      # RR7      # )aµ  
input_features_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Attention mask for `input_features` where non-zero values mark valid audio frames.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, Gemma3nForConditionalGeneration

>>> model = Gemma3nForConditionalGeneration.from_pretrained("google/gemma3n2-3b-mix-224")
>>> processor = AutoProcessor.from_pretrained("google/gemma3n2-3b-mix-224")

>>> prompt = "Where is the cat standing?"
>>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(**inputs,)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Where is the cat standing?\nsnow"
```
NrÎ  )rÁ  T)r  )r~  r  r¿  r3  )r~  r  rÁ  rÇ  r  r=  r  r~  rÎ   r  )r   r  r÷   rT  Úimage_hidden_statesrÞ   rü   rA   )#rF   r  r×   r¹  rg   r»  Ú
zeros_liker  rÄ  r  r“   r  rf   r4  r2  r/  r&  r  r  r#  Úmasked_scatterÚget_audio_featuresrÖ   r|  r  rY  r  rÅ   r¡  r6  rÛ   r   r  r÷   rT  )(rK   rÁ  r  r&  r  r'  r=  r  r(  r~  r)  rÎ   r*  Úper_layer_inputs_maskÚper_layer_inputs_tokensrÇ  Úvision_maskÚdummy_vision_token_idÚvision_input_idsÚvision_embedsÚexpanded_vision_maskÚ
audio_maskÚdummy_audio_token_idÚaudio_input_idsÚaudio_embedsÚexpanded_audio_maskr  r  r]  Úaudio_outputsr  Úaudio_padding_toksÚaudio_padding_embsÚaudio_batch_sizeÚaudio_seq_lenÚaudio_embed_dimÚextra_padding_tokensÚextra_padding_featuresr   Úoutputss(   &&&&&&&&&&&&,                           rQ   r
  ÚGemma3nModel.forwardk  s²  € ð` ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÓ Ø ×5Ñ5Ô7¸	ÓBˆMô %*×$5Ò$5°iÀ1±nÀi×RqÑRqÑFqÓ$rÐ!Ü&+§k¢kÐ2GÔTY×TdÒTdÐenÓToÓ&pÐ#Ø#×2Ñ2×GÑGÐH_Ó`Ðô  ×+Ò+Ø×.Ñ.×;Ñ;Ñ;¸Y×IYÑIY×IfÑIfÑ=fóˆKð %)×$5Ñ$5×$BÑ$BÀT×EVÑEV×EaÑEaÕ$aÐdeÕ$eÐ!Ü$Ÿ{š{¨;¸	ÐCXÓY×\Ñ\Ð]j×]qÑ]qÓrÐØ ×-Ñ-Ð8HÐ-ÓIˆMØ)×,Ñ,¨]×-AÑ-AÀ=×CVÑCVÓWˆMØ#.×#8Ñ#8¸Ó#<Ð Ü!ŸKšKÐ(<¸mÈ]Ó[ˆMð #×&6Ñ&6×&CÑ&CÑCˆJØ#'×#3Ñ#3×#@Ñ#@À4×CSÑCS×C^ÑC^Õ#^ÐabÕ#bÐ Ü#Ÿkšk¨*°iÐAUÓV×YÑYÐZg×ZnÑZnÓoˆOØ×+Ñ+°oÐ+ÓFˆLØ'Ÿ?™?¨=×+?Ñ+?À×ATÑATÓUˆLØ",×"6Ñ"6°rÓ":ÐÜ!ŸKšKÐ(;¸\È=ÓY‰Mà#Ðð Ò#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ$(×$=Ñ$=ØÀ~ð %>ó %Ñ!Ð ð *×8Ñ8Ð9KÈ^Ó\ˆMð Ó%Ð*=Ó*IØ ×3Ñ3°NÐDXÐfjÐ3ÓkˆMØ*×8Ñ8ˆNØ&×5Ñ5ˆJô "'§¢°·±À!Õ0CÐ/DÐ.EÌUÏZÉZÐ`n×`uÑ`uÔ!vÐØ!%×!1Ñ!1Ð<NÐ!1Ó!OÐÜ"Ÿ[š[¨×)=Ñ)=¸bÓ)AÐCUÐWeÓfˆNà?M×?SÑ?SÑ<Ð˜m¨_Ø#'§;¡;×#JÑ#JÈ]Õ#ZÐ Ø%7×%>Ñ%>Ð?OÐQeÐgvÓ%wÐ"ä"ŸYšY¨Ð8NÐ'OÐUVÔWˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ$(×$=Ñ$=ØÀ~ð %>ó %Ñ!ˆAÐ!ð *×8Ñ8Ð9KÈ^Ó\ˆMà×%Ò%ñ 

Øð

à-ð

ð *ð

ð &ð	

ð
 ,ð

ð (ð

ð  ð

ð ð

ð ñ

ˆô *Ø%×7Ñ7ß7@˜G×3Ò3ÀdØ!×/Ñ/Ø×)Ñ)Ø2>Ò2J¡ÐPTØ2@Ò2L ô
ð 	
ð SWô
ð 	
rc   zPProjects the last hidden state from the audio encoder into language model space.c          	      óz   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[S[,          /# )re   r&  r'  rL   r  )r×   rù   r   r   r¤   rÔ   )ry   rz   s   "€rQ   r{   rý  ö  sG   ø€ ÷ ñ áŸ™ðñ #Ÿ\™\ðñ Ñ+Õ,ð	ñ
 
Ñ/Õ	/ñrc   c                ót   € V P                   ! W3RR/VB pV P                  VP                  R7      pWTn        V# )a  
input_features (`torch.FloatTensor` of shape `(num_images, seq_length, num_features)`):
    The tensors corresponding to the input audio.
input_features_mask (`torch.FloatTensor` of shape `(num_images, seq_length)`):
    The attention mask for the input audio.
r  Tr  )r  r  r   r  )rK   r&  r'  rL   r<  r:  s   &&&,  rQ   r/  ÚGemma3nModel.get_audio_featuresô  sN   € ð 9=×8HÒ8HØñ9
Ø=Að9
ØEKñ9
ˆð ×'Ñ'°m×6UÑ6UÐ'ÓVˆØ&2Ô#àÐrc   )r  r  r  rg   )NNNN)NNNNNNNNNNN)r}   r~   r   r€   rð   ro  rs  r   r   r  r#  r
  r/  r‰   rŠ   rÑ   rÒ   s   @@rQ   rû  rû    sŽ   ù‡ € ÷^ó ^ò:ò;ð ÙÐ!rÔs÷ó tó ð÷(*6ò *6ðX ÷F
ñ F
ó ðF
ðP ÙÐ!sÔt÷ó uó ÷ð rc   rû  z†
    The base Gemma 3n model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c                   óf   a € ] tR tRt o RtR tR t]]RV 3R lR ll4       4       t	R t
R	 tR
tV tR# )ÚGemma3nForConditionalGenerationi	  Fc                ó6   € V P                   P                  4       # r  )Úmodelro  rn  s   &rQ   ro  Ú>Gemma3nForConditionalGeneration.get_per_layer_input_embeddings	  s   € Ø�z‰z×8Ñ8Ó:Ð:rc   c                ó<   € V P                   P                  V4       R # r  )rL  rs  rr  s   &&rQ   rs  Ú>Gemma3nForConditionalGeneration.set_per_layer_input_embeddings	  s   € Ø�
‰
×1Ñ1°%Ö8rc   Nc                óð  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  ,          RS[S[,          RS[	/# )re   rÁ  Nr  r&  r  r'  r=  r  r(  r~  r)  rÎ   Úlogits_to_keepr*  r  )
r×   r>  rà   rù   r   rx   rt   r   r   rä   )ry   rz   s   "€rQ   r{   Ú,Gemma3nForConditionalGeneration.__annotate__	  s  ø€ ÷ e
ñ e
á×#Ñ# dÕ*ðe
ñ ×'Ñ'¨$Õ.ðe
ñ ×)Ñ)¨DÕ0ð	e
ñ
 Ÿ™ tÕ+ðe
ñ #Ÿ\™\¨DÕ0ðe
ñ ×&Ñ&¨Õ-ðe
ñ  �ðe
ñ ×(Ñ(¨4Õ/ðe
ñ ×(Ñ(¨4Õ/ðe
ñ × Ñ  4Õ'ðe
ñ ˜$•;ðe
ñ ™eŸl™lÕ*ðe
ñ Ñ.Õ/ðe
ñ 
'ñe
rc   c                óŽ  € V P                   ! RRVRVRVRVRVRVRVRVR	V	R
V
RVRR/VB pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  4       P                  ;pe)   VV,          p\        P                  ! V4      pVV,          pRpV
e7   V P                  ! VW P                  P                  4       P                  3/ VB p\        VVVP                  VP                  VP                   VP"                  VP$                  R7      # )a<  
input_features_mask (torch.Tensor, *optional*, defaults to None):
    The attention mask for the input audio.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
    ignored (masked), the loss is only computed for the tokens with labels in
    `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

>>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
>>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

>>> messages = [
...     {
...         "role": "system",
...         "content": [
...             {"type": "text", "text": "You are a helpful assistant."}
...         ]
...     },
...     {
...         "role": "user", "content": [
...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
...             {"type": "text", "text": "Where is the cat standing?"},
...         ]
...     },
... ]

>>> inputs = processor.apply_chat_template(
...     messages,
...     tokenizer=True,
...     return_dict=True,
...     return_tensors="pt",
...     add_generation_prompt=True
... )
>>> # Generate
>>> generate_ids = model.generate(**inputs)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
```
rÁ  r  r&  r  r'  r=  r  r(  r~  r)  rÎ   r  TNrG  )ÚlossrÐ  r  r÷   rT  r,  rÞ   rA   )rL  r   rB   rt   ÚsliceÚlm_headr  rƒ  rm   r×   rº  Úloss_functionrf   rä   r  r÷   rT  r,  rÞ   )rK   rÁ  r  r&  r  r'  r=  r  r(  r~  r)  rÎ   rQ  r*  rD  r÷   Úslice_indicesrÐ  rm   rT  s   &&&&&&&&&&&&&,      rQ   r
  Ú'Gemma3nForConditionalGeneration.forward	  s�  € ðD —*’*ñ 
Øð
à%ð
ð *ð
ð *ð	
ð
 !4ð
ð &ð
ð ,ð
ð *ð
ð (ð
ð ð
ð  ð
ð ð
ð ñ
ˆð   ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ'+§{¡{×'BÑ'BÓ'D×'\Ñ'\Ð\Ð#ÒiØÐ5Õ5ˆFÜ—Z’Z Ó'ˆFØÐ5Õ5ˆFàˆØÒØ×%Ò% f¨f·k±k×6QÑ6QÓ6S×6^Ñ6^ÑlÐbkÑlˆDä,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;Ø '× ;Ñ ;ô
ð 	
rc   c                ó   € \        R 4      h)z;Do not inherit prepare_inputs_for_generation from PaliGemma©ÚNotImplementedError©rK   Úsuper_kwargss   &,rQ   Úprepare_inputs_for_generationÚ=Gemma3nForConditionalGeneration.prepare_inputs_for_generationƒ	  s   € Ü!Ð"_Ó`Ð`rc   c                ó   € \        R 4      h)z7Do not inherit create_masks_for_generate from PaliGemmar[  r]  s   &,rQ   Úcreate_masks_for_generateÚ9Gemma3nForConditionalGeneration.create_masks_for_generate†	  s   € Ü!Ð"[Ó\Ð\rc   rA   )NNNNNNNNNNNr   )r}   r~   r   r€   Úaccepts_loss_kwargsro  rs  r   r   r
  r_  rb  r‰   rŠ   r‹   s   @rQ   rJ  rJ  	  sN   ø‡ € ð  Ðò;ò9ð Ø÷e
ñ e
ó ó ðe
òNa÷]ð ]rc   rJ  rC  )
r�   r”  r¹   rå  rJ  rû  rS  r1   ra  r­   )qr!  Úcollectionsr   Úcollections.abcr   r   Údataclassesr   Útypingr   r×   Útorch.nnrñ   Útorch.nn.functionalrH  r$  Úhuggingface_hub.dataclassesr   Ú r	   rZ  rº  r
   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úautor   Úgemma2.modeling_gemma2r   r    r!   r"   Úgemma3.configuration_gemma3r#   Úgemma3.modeling_gemma3r$   r%   r&   r'   r(   Úpaligemma.modeling_paligemmar)   r*   r+   r,   Ú'timm_wrapper.configuration_timm_wrapperr-   Úaccelerate.hooksr.   Ú
get_loggerr}   r¿   r1   r�   r­   r¹   rÔ   rÛ   rä   ÚModulerè   r  rl  rß  r  r,  rN  rb  rr  r‡  r–  rš  r©  rÏ  r  r  r.  rS  r”  rd  ra  rå  rç  rû  rJ  Ú__all__rA   rc   rQ   Ú<module>r�     s�  ðó Ý  ß .Ý !Ý ã Ý ß Ð Ý .å &Ý !ß .Ý 3ß Rß SÝ 6ß FÝ &÷÷ õ 8Ý 5Ý ÷ó õ ;÷õ ÷ó õ Hñ ×ÒÝ3à	×	Ò	˜HÓ	%€ñ Ð0Ô1ØôGÐ(ó Gó ó 2ðGñT Ð0Ô1Øô_Ð)ó _ó ó 2ð_ñD Ð0Ô1Øô%#Ð+ó %#ó ó 2ð%#ñP Ð0Ô1ØôP(Ð$ó P(ó ó 2ðP(ðf Ø
ô3Ð%?ó 3ó ó ð3ô9Ð!=ô 9ô$9Ð$Cô 9ô,4�R—Y‘Yô 4ô0c)¨B¯I©Iô c)ôL]˜BŸI™Iô ]ô@j, b§i¡iô j,ôZB7 §	¡	ô B7ôJF¨"¯)©)ô FôRO R§Y¡Yô Oô8D r§y¡yô Dô2( r§y¡yô (ôV §¡ô ô<	Ð%Bô 	ô;˜RŸY™Yô ;ô$5�Yô 5ô@`'�r—y‘yô `'÷F.ô,l)˜2Ÿ9™9ô l)ô^B%Ð0ô B%ôJVJÐ2ô VJôrN
Ð0ô N
ôb	Ð2ô 	ñ ÐaÔbôC
�ó C
ó cðC
ñL Ð^Ô_ô	Ð*ó 	ó `ð	ô/B §	¡	ô /Bñd ðôôo�>ó oóðoñd ðôôv]Ð&Gó v]óðv]òr‚rc   