+
    œ°³jÊ¶ ã                   ó$  € ^ RI t ^ RIHt ^ RIHtHt ^ RIHt ^ RIt^ RI	H
t
 ^ RIH
u Ht ^RIHt ^RIHt ^RIHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHtHtH t  ^RI!H"t"H#t# ^RI$H%t%H&t& ^RI'H(t( ^RI)H*t*H+t+H,t,H-t-H.t.H/t/ ^RI0H1t1 ^RI2H3t3H4t4 ^RI5H6t6 ^RI7H8t8 ^RI9H:t:H;t;H<t<H=t= ].! 4       '       d   ^ RI>H?t? ],] ! R R]4      4       4       t@],! RR7      ] ! R R]4      4       4       tA],! RR7      ] ! R R]*4      4       4       tB ! R R ]
P†                  4      tD ! R! R"]
P†                  4      tE ! R# R$]
P†                  4      tF ! R% R&]
P†                  4      tG ! R' R(]
P†                  4      tH ! R) R*]
P†                  4      tI ! R+ R,]
P†                  4      tJ ! R- R.]
P†                  4      tK ! R/ R0]
P†                  4      tL ! R1 R2]
P†                  4      tM ! R3 R4]
Pœ                  4      tO ! R5 R6]
P†                  4      tP ! R7 R8]
P†                  4      tQ ! R9 R:]
P†                  4      tRR; tSR< R= ltTRZR> R? lltUR[R@ RA lltV ! RB RC]
P†                  4      tW ! RD RE]4      tX], ! RF RG]&4      4       tY ! RH RI]Y4      tZ ! RJ RK]
P†                  4      t[],! RLR7       ! RM RN]Y4      4       t\],! ROR7       ! RP RQ]Y]4      4       t] ! RR RS]
P†                  4      t^],! RTR7       ! RU RV]Y4      4       t_],! RWR7       ! RX RY]Y]4      4       t`. R\OtaR# )]é    N)ÚUserDict)ÚCallableÚSequence)Ú	dataclass)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_accelerate_availableÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs)Ú	AutoModel)ÚGemma3nAudioConfigÚGemma3nConfigÚGemma3nTextConfigÚGemma3nVisionConfig)Úadd_hook_to_modulec                   ó4   a € ] tR t^;t o RtRtV 3R ltRtV tR# )ÚGemma3nAudioEncoderModelOutputzm
audio_mel_mask (`torch.BoolTensor`, *optional*):
    A torch.BoolTensor of shape `(batch_size, num_frames)`
Nc                óH   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   # )é   NÚaudio_mel_mask)ÚtorchÚ
BoolTensor)ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gemma3n/modeling_gemma3n.pyÚ__annotate__Ú+Gemma3nAudioEncoderModelOutput.__annotate__;   s   ø‡ ‚ ñ ×$Ñ$ tÕ+Ñ2ò ó    © )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r+   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r/   s   @r0   r(   r(   ;   s   ø‡ € ñð
 /3€N÷ ƒ r3   r(   zL
    Base class for Gemma3n outputs, with hidden states and attentions.
    ©Úcustom_introc                   ó8   a € ] tR t^Ft o RtRtRtV 3R ltRtV t	R# )ÚGemma3nModelOutputWithPasta±  
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
audio_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
Nc                óv   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   # )r*   NÚimage_hidden_statesÚaudio_hidden_states)r,   ÚFloatTensor)r.   r/   s   "€r0   r1   Ú'Gemma3nModelOutputWithPast.__annotate__F   s5   ø‡ ‚ ñ* ×*Ñ*¨TÕ1Ñ8ñ+ ñ. ×*Ñ*¨TÕ1Ñ8ò/ r3   r4   )
r5   r6   r7   r8   r9   rC   rD   r:   r;   r<   r=   s   @r0   rA   rA   F   s   ø‡ € ñð 59Ðà48Ð÷/ ƒ r3   rA   zS
    Base class for Gemma3n causal language model (or autoregressive) outputs.
    c                   óL   a € ] tR t^`t o RtRtRtRtRtRt	Rt
RtV 3R ltRtV tR# )ÚGemma3nCausalLMOutputWithPasta
  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
audio_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
Nc                óh  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   # )	r*   NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsrC   rD   )r,   rE   r	   Útuple)r.   r/   s   "€r0   r1   Ú*Gemma3nCausalLMOutputWithPast.__annotate__`   s²   ø‡ ‚ ñ2 ×
Ñ
˜dÕ
"Ñ)ñ3 ñ4 ×Ñ Õ$Ñ+ñ5 ñ6 ˜T•\Ñ(ñ7 ñ8 ™×*Ñ*Õ+¨dÕ2Ñ9ñ9 ñ: ‘e×'Ñ'Õ(¨4Õ/Ñ6ñ; ñ< ×*Ñ*¨TÕ1Ñ8ñ= ñ@ ×*Ñ*¨TÕ1Ñ8òA r3   r4   )r5   r6   r7   r8   r9   rJ   rK   rL   rM   rN   rC   rD   r:   r;   r<   r=   s   @r0   rH   rH   `   s9   ø‡ € ñð$ &*€DØ'+€FØ$(€OØ59€MØ26€JØ48Ðà48Ð÷A ƒ r3   rH   c                   óf   a a€ ] tR t^ƒt oRV3R lV 3R llltV3R lR ltV3R lR ltRtVtV ;t	# )	ÚGemma3nRMSNormc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r*   ÚdimÚepsÚ
with_scale)ÚintÚfloatÚbool)r.   r/   s   "€r0   r1   ÚGemma3nRMSNorm.__annotate__„   s'   ø€ ÷ Lñ L™Cð L¡eð LÁñ Lr3   c                óÆ   <€ \         SV `  4        W n        W0n        V P                  '       d4   \        P
                  ! \        P                  ! V4      R R7      V n        R# R# )T)Úrequires_gradN)	ÚsuperÚ__init__rU   rV   ÚnnÚ	Parameterr,   ÚonesÚweight)ÚselfrT   rU   rV   Ú	__class__s   &&&&€r0   r^   ÚGemma3nRMSNorm.__init__„   s@   ø€ Ü‰ÑÔØŒØ$Œà�?�?ˆ?ÜŸ,š,¤u§z¢z°#£ÀdÔKˆDŽKñ r3   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r*   rM   ©r,   ÚTensor)r.   r/   s   "€r0   r1   rZ   Œ   s   ø€ ÷ =ñ =¡5§<¡<ñ =r3   c                ó¤   € VP                  ^4      P                  RRR7      V P                  ,           pV\        P                   ! VR4      ,          # )r*   T)Úkeepdiméÿÿÿÿç      à¿)ÚpowÚmeanrU   r,   )rc   rM   Úmean_squareds   && r0   Ú_normÚGemma3nRMSNorm._normŒ   sA   € Ø$×(Ñ(¨Ó+×0Ñ0°¸TÐ0ÓBÀTÇXÁXÕMˆàœuŸyšy¨°tÓ<Õ<Ð<r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r*   rM   Úreturnrg   )r.   r/   s   "€r0   r1   rZ   ‘   s#   ø€ ÷ 4ñ 4¡U§\¡\ð 4±e·l±lñ 4r3   c                óÆ   € V P                  VP                  4       4      pV P                  '       d!   W P                  P                  4       ,          pVP	                  V4      # ©N)rp   rX   rV   rb   Útype_as)rc   rM   Únormed_outputs   && r0   ÚforwardÚGemma3nRMSNorm.forward‘   sH   € ØŸ
™
 =×#6Ñ#6Ó#8Ó9ˆØ�?�?ˆ?Ø)¯K©K×,=Ñ,=Ó,?Õ?ˆMØ×$Ñ$ ]Ó3Ð3r3   )rU   rb   rV   )g�íµ ÷Æ°>T)
r5   r6   r7   r8   r^   rp   ry   r;   r<   Ú__classcell__©rd   r/   s   @@r0   rR   rR   ƒ   s*   ù‡ € ÷Lõ L÷=ð =÷
4÷ 4ð 4r3   rR   c                   ót   a a€ ] tR t^›t oV3R lV 3R lltV3R lR ltV3R lR ltV3R lR ltR	tVt	V ;t
# )
Ú%Gemma3nAudioRelativePositionEmbeddingc                ó    <€ V ^8„  d   QhRS[ /# ©r*   Úconfig©r"   )r.   r/   s   "€r0   r1   Ú2Gemma3nAudioRelativePositionEmbedding.__annotate__œ   s   ø€ ÷ lñ lÑ1ñ lr3   c                ó´  <€ \         SV `  4        Wn        V P                  P                  V n        V P                  P
                  V n        V P                  V P                  ,          V n        \        ^ V P                  P                  ^,
          4      V n
        V P                  P                  V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        RpRpV P                  ^,          p\         P"                  ! \%        V4      \%        V4      ,          4      \        V^,
          ^4      ,          pV\&        P(                  ! \&        P*                  ! V4      V) ,          4      ,          p\        P,                  ! VP%                  4       P/                  ^ 4      P/                  ^ 4      RR7      V n        R# )r   F©Úbiasç      ð?ç     ˆÃ@©Ú
persistentN)r]   r^   r�   Úconf_num_attention_headsÚ	num_headsÚhidden_sizeÚchannelsÚhead_dimÚmaxÚconf_attention_context_leftÚmax_backwardÚconf_attention_context_rightÚmax_forwardr_   ÚLinearÚpos_projÚmathÚlogrX   r,   ÚexpÚarangeÚBufferÚ	unsqueezeÚinv_timescales)rc   r�   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementr�   rd   s   &&     €r0   r^   Ú.Gemma3nAudioRelativePositionEmbedding.__init__œ   sC  ø€ Ü‰ÑÔØŒàŸ™×=Ñ=ˆŒØŸ™×/Ñ/ˆŒØŸ™¨¯©Õ7ˆŒÜ  4§;¡;×#JÑ#JÈQÕ#NÓOˆÔØŸ;™;×CÑCˆÔäŸ	š	 $§-¡-°·±À$Ç-Á-Õ1OÐV[Ô\ˆŒàˆØˆØŸ™¨!Õ+ˆÜ"&§(¢(¬5°Ó+?Ä%ÈÓBVÕ+VÓ"WÔZ]Ð^lÐopÕ^pÐrsÓZtÕ"tÐØ&¬¯ª´5·<²<ÀÓ3OÐSjÐRjÕ3jÓ)kÕkˆÜ Ÿiši¨×(<Ñ(<Ó(>×(HÑ(HÈÓ(K×(UÑ(UÐVWÓ(XÐejÔkˆÖr3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r*   ÚpositionÚdtypert   )r,   rh   r¥   )r.   r/   s   "€r0   r1   rƒ   ¯   s1   ø€ ÷ )ñ )±%·,±,ð )ÁuÇ{Á{ð )ÑW\×WcÑWcñ )r3   c                ó\  € VP                  4       P                  R4      pWP                  P                  VP                  \
        P                  R7      ,          p\
        P                  ! \
        P                  ! V4      \
        P                  ! V4      .RR7      pVP                  V4      # )é   )Údevicer¥   ©rT   rk   )rX   rœ   r�   Útor¨   r,   Úfloat32ÚcatÚsinÚcosÚtype)rc   r¤   r¥   Úscaled_timeÚtiming_signals   &&&  r0   Ú_get_timing_signal_1d_posÚ?Gemma3nAudioRelativePositionEmbedding._get_timing_signal_1d_pos¯   s{   € Ø—>‘>Ó#×-Ñ-¨bÓ1ˆØ×!4Ñ!4×!7Ñ!7¸x¿¹ÔV[×VcÑVcÐ!7Ó!dÕdˆÜŸ	š	¤5§9¢9¨[Ó#9¼5¿9º9À[Ó;QÐ"RÐXZÔ[ˆØ×!Ñ! %Ó(Ð(r3   c                ór   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[RS[RS[RS[RS[ P                  /# )	r*   Úterm_bd_before_shiftÚ
batch_sizerŒ   Únum_query_blocksÚquery_block_sizeÚkey_context_sizeÚmax_span_plus_1rt   ©r,   rh   rW   )r.   r/   s   "€r0   r1   rƒ   µ   sc   ø€ ÷ ;ñ ;á#Ÿl™lð;ñ ð;ñ ð	;ñ
 ð;ñ ð;ñ ð;ñ ð;ñ 
�‰ñ;r3   c                ó  € V^,           V,
          p^ V3p	\         P                  P                  W4      p
V
P                  VVVWV^,           ,          34      pVRRRRWV,          13,          pVP                  VVVVV34      pV# )a"  Performs the relative shift.

Args:
  term_bd_before_shift: Tensor of shape [B, N, U, W, F_span]. batch_size
    (B), num_heads (N), num_query_blocks (U), query_block_size (W),
    key_context_size (C = W+L+R), max_span_plus_1 (F_span = L+R+1).

Returns:
  Tensor of shape [B, N, U, W, C].
ºNNNN)r_   Ú
functionalÚpadÚreshape)rc   rµ   r¶   rŒ   r·   r¸   r¹   rº   Úpad_amount_last_dimÚpadding_tupleÚterm_bd_paddedÚterm_bd_reshapedÚterm_bd_slicedÚterm_bd_shifteds   &&&&&&&&      r0   Ú_relative_shiftÚ5Gemma3nAudioRelativePositionEmbedding._relative_shiftµ   s¯   € ð4  0°!Õ3°ÕFÐð Ð/Ð0ˆäŸ™×*Ñ*Ð+?ÓOˆð
 *×1Ñ1àØØ Ø °qÕ$8Õ9ð	ó
Ðð *¨!¨Q°Ð3XÐ5EÕ5XÐ3XÐ*XÕYˆð )×0Ñ0àØØ Ø Ø ðó
ˆð Ðr3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r*   ÚqueriesÚkeysrt   rg   )r.   r/   s   "€r0   r1   rƒ   ò   s3   ø€ ÷ L)ñ L)™uŸ|™|ð L)±5·<±<ð L)ÁEÇLÁLñ L)r3   c           	     ój  € VP                   w  r4rVpVP                   w   r‰ p\        P                  ! V P                  V P                  ) ^,
          RVP
                  R7      P                  ^ 4      p
V
P                   ^,          pV P                  W¡P                  R7      pV P                  V4      pVP                  ^W°P                  V P                  4      P                  ^ 4      pVP                  ^ ^^^^4      pVP                  ^ ^^^^4      p\        P                  ! VV4      pVP                  ^ ^^^^4      pVP                  ^^^ 4      pVP                  W6WE,          V4      p\        P                  ! VV4      pVP                  VVVVV4      pV P!                  VVVVVV	V4      pVV,           # )r§   ©r¨   ©r¥   rk   )Úshaper,   rš   r’   r”   r¨   rœ   r²   r¥   r–   rÀ   rŒ   r�   ÚsqueezeÚpermuteÚmatmulrÇ   )rc   rÊ   rË   r¶   r·   r¸   rŒ   r�   Ú_r¹   Úpos_indicesrº   Úsin_emb_timing_signalÚprojected_sin_embÚsin_embÚ	queries_pÚkeys_p_tÚterm_acÚ
q_permutedÚ
s_permutedÚ
q_reshapedÚterm_bd_unshifed_matmulÚterm_bd_unshifedrÆ   s   &&&                     r0   ry   Ú-Gemma3nAudioRelativePositionEmbedding.forwardò   sÄ  € ð OVÏmÉmÑKˆ
Ð&6À8Ø'+§z¡zÑ$ˆˆ1  1ô —l’l 4×#4Ñ#4°t×7GÑ7GÐ6GÈ!Õ6KÈRÐX_×XfÑXfÔg×qÑqØó
ˆð &×+Ñ+¨AÕ.ˆà $× >Ñ >ØŸ}™}ð !?ó !
Ðð
 !ŸM™MÐ*?Ó@Ðà#×+Ñ+¨A¨ÇÁÐPT×P]ÑP]Ó^×fÑfØó
ˆð —O‘O A q¨!¨Q°Ó2ˆ	Ø—<‘<  1 a¨¨AÓ.ˆÜ—,’,˜y¨(Ó3ˆð —_‘_ Q¨¨1¨a°Ó3ˆ
ð —_‘_ Q¨¨1Ó-ˆ
ð  ×'Ñ'¨
Ð?OÕ?bÐdlÓmˆ
ô
 #(§,¢,¨z¸:Ó"FÐð 3×:Ñ:ØØØØØó
Ðð ×.Ñ.ØØØØØØØó
ˆð ˜Õ(Ð(r3   )rŽ   r�   r�   r�   r’   r”   rŒ   r–   )r5   r6   r7   r8   r^   r²   rÇ   ry   r;   r<   r{   r|   s   @@r0   r~   r~   ›   s8   ù‡ € ÷ló l÷&)ð )÷;ð ;÷zL)÷ L)ð L)r3   r~   c                   óŒ   a a€ ] tR tRt oV3R lV 3R lltR tV3R lR ltV3R lR ltV3R	 lR
 ltV3R lR lt	Rt
VtV ;t# )ÚGemma3nAudioAttentioniA  c                ó    <€ V ^8„  d   QhRS[ /# r€   r‚   )r.   r/   s   "€r0   r1   Ú"Gemma3nAudioAttention.__annotate__B  s   ø€ ÷ iñ iÑ1ñ ir3   c                ó~  <€ \         SV `  4        Wn        V P                  P                  V n        V P                  P
                  V n        V P
                  V P                  ,          V n        V P                  P                  V n        V P                  P                  V n
        \        ^ V P                  P                  ^,
          4      V n        V P                  P                  V n        V P                  V P                  ,           V P                  ,           V n        \#        V4      V n        \&        P(                  ! \*        P,                  ! V P                  34      4      V n        \&        P0                  ! V P
                  V P                  V P                  ,          RR7      V n        \&        P0                  ! V P
                  V P                  V P                  ,          RR7      V n        \&        P0                  ! V P
                  V P                  V P                  ,          RR7      V n        V P                  R,          pR\*        P&                  P8                  P;                  \*        P<                  ! R4      4      ,          p\&        P>                  ! W#,          PA                  4       PC                  4       RR7      V n"        V PG                  4       p\&        P>                  ! VRR7      V n$        \&        P>                  ! \*        P<                  ! V P                  4      PK                  4       RR7      V n&        R# )r   Fr…   r‡   ç        r‰   Nrl   )'r]   r^   r�   r‹   rŒ   r�   r�   Úconf_attention_chunk_sizeÚ
chunk_sizer“   Úmax_future_horizonr�   r‘   Úmax_past_horizonÚconf_attention_logit_capÚattention_logits_soft_capÚcontext_sizer~   Úrelative_position_embeddingr_   r`   r,   ÚzerosÚper_dim_scaler•   Úq_projÚk_projÚv_projr¾   ÚsoftplusÚtensorr›   ÚcloneÚdetachÚq_scaleÚcreate_local_causal_valid_maskÚlocal_causal_valid_maskrX   Úsoftcap)rc   r�   rø   Úr_softplus_0rú   rd   s   &&   €r0   r^   ÚGemma3nAudioAttention.__init__B  s  ø€ Ü‰ÑÔØŒàŸ™×=Ñ=ˆŒØŸ;™;×2Ñ2ˆÔØ×(Ñ(¨D¯N©NÕ:ˆŒàŸ+™+×?Ñ?ˆŒØ"&§+¡+×"JÑ"JˆÔÜ # A t§{¡{×'NÑ'NÐQRÕ'RÓ SˆÔØ)-¯©×)MÑ)MˆÔ&Ø ŸO™O¨d×.CÑ.CÕCÀd×F]ÑF]Õ]ˆÔä+PÐQWÓ+XˆÔ(ÜŸ\š\¬%¯+ª+°t·}±}Ð6FÓ*GÓHˆÔä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒà—-‘- Õ%ˆØœUŸX™X×0Ñ0×9Ñ9¼%¿,º,ÀsÓ:KÓLÕLˆÜ—y’y 'Õ"8×!?Ñ!?Ó!A×!HÑ!HÓ!JÐW\Ô]ˆŒà"&×"EÑ"EÓ"GÐÜ')§y¢yÐ1HÐUZÔ'[ˆÔ$ä—y’y¤§¢¨d×.LÑ.LÓ!M×!SÑ!SÓ!UÐbgÔhˆŽr3   c                ó.  € \         P                  ! \         P                  ! V P                  V P                  3\         P
                  R 7      ^ R7      P                  p\         P                  ! \         P                  ! V P                  V P                  3\         P
                  R 7      V P                  V P                  ,           R7      p\         P                  ! V P                  V P                  3\         P
                  R 7      pW1,          V,          pV# )rÎ   )Údiagonal)	r,   Útrilra   rí   rè   rY   ÚTrê   ré   )rc   Úlower_causal_maskÚupper_causal_maskrú   s   &   r0   rù   Ú4Gemma3nAudioAttention.create_local_causal_valid_mask`  sÂ   € Ü!ŸJšJÜ�JŠJ˜×)Ñ)¨4¯?©?Ð;Ä5Ç:Á:ÔNØô
÷ ‰!ð 	ô "ŸJšJÜ�JŠJ˜Ÿ™¨×):Ñ):Ð;Ä5Ç:Á:ÔNØ×*Ñ*¨T×-DÑ-DÕDô
Ðô #(§*¢*¨d¯o©o¸t×?PÑ?PÐ-QÔY^×YcÑYcÔ"dÐØ"9Õ"MÐPaÕ"aÐØ&Ð&r3   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r*   ÚxÚpad_leftÚ	pad_rightrt   r»   )r.   r/   s   "€r0   r1   rä   m  s1   ø€ ÷ ñ ™5Ÿ<™<ð ±3ð Á3ð É5Ï<É<ñ r3   c                óª   € VP                   vrEpVP                  WB.VO54      pVP                  WC.VO54      p\        P                  ! WqV.^R7      pV# )r§   r©   )rÏ   Ú	new_zerosr,   r¬   )	rc   r  r  r  ÚbatchrÓ   Ú
tail_shapeÚleftÚrights	   &&&&     r0   Ú	_pad_dim1ÚGemma3nAudioAttention._pad_dim1m  sV   € Ø !§¡Ðˆ�:Ø�{‰{˜EÐ9¨jÑ9Ó:ˆØ—‘˜UÐ;°
Ñ;Ó<ˆÜ�IŠI�t Ð&¨AÔ.ˆØˆr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rs   rg   )r.   r/   s   "€r0   r1   rä   t  s#   ø€ ÷ ñ ©u¯|©|ð ÁÇÁñ r3   c                ód  € VP                   pVR,          w  r4W@P                  ,           ^,
          V P                  ,          pWPP                  ,          V,
          ;p^ 8”  d   V P                  V^ V4      pW5V P                  3VR,          ,           pVP                  V4      P	                  4       pV# )a  Turns a sequence to non overlapping blocks.

Args:
    hidden_states: a tensor of [batch, time, ...].

Returns:
    A tensor of [batch, num_blocks, block_size, ...], with necessary
    paddings,
    where output[:, i, ...] are x[:, i*block_size:(i+1)*block_size, ...].
ºNr*   Nºr*   NN)rÏ   rè   r  rÀ   Ú
contiguous)rc   rM   rÏ   ÚbÚtÚ
num_blocksÚpadding_lenÚpermute_dimss   &&      r0   Ú_convert_to_blockÚ'Gemma3nAudioAttention._convert_to_blockt  s•   € ð ×#Ñ#ˆØ�R�y‰ˆØŸ/™/Õ)¨AÕ-°$·/±/ÕAˆ
à%¯©Õ7¸!Õ;Ð;ˆK¸qÔ@Ø ŸN™N¨=¸!¸[ÓIˆMà t§¡Ð7¸%À½)ÕCˆØ%×-Ñ-¨lÓ;×FÑFÓHˆØÐr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rs   rg   )r.   r/   s   "€r0   r1   rä   Š  s#   ø€ ÷ .'ñ .'±E·L±Lð .'ÁUÇ\Á\ñ .'r3   c                ór  € V P                   pV P                  V P                  ,           ^,
          pV P                  WV4      pV P                  pV P                  pVP                  ^WER7      pVP                  ^8”  d+   VP                  ^8”  d   \        P                  ! VR^R7      pVP                  4       # )a“  Extracts temporal context for every block.

Args:
    hidden_states: a tensor of [batch, time, ...].

Returns:
    A tensor of [batch, num_blocks, context_size, ...], with necessary
    paddings,
    where context_size = block_size + left_context + right_context,
    and output[:, i, ...] are x[:, start-left_context:end+right_context,
    ...],
    start = i * block_size, end = (i + 1) * block_size.
)Ú	dimensionÚsizeÚstep)ÚsourceÚdestinationrk   )
rê   ré   rè   r  rí   ÚunfoldÚndimr,   Úmovedimr  )rc   rM   r  r  Ú	frame_lenÚ
frame_stepÚ
x_unfoldeds   &&     r0   Ú_extract_block_contextÚ,Gemma3nAudioAttention._extract_block_contextŠ  s£   € ð ×(Ñ(ˆð ×+Ñ+¨d¯o©oÕ=ÀÕAˆ	ØŸ™ }À	ÓJˆà×%Ñ%ˆ	Ø—_‘_ˆ
ð #×)Ñ)°A¸IÐ)ÓWˆ
ð ×Ñ Ô! j§o¡o¸Ô&9ô Ÿš z¸"È!ÔLˆJà×$Ñ$Ó&Ð&r3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r*   rM   Úmaskrt   ©r,   rh   r-   )r.   r/   s   "€r0   r1   rä   º  s5   ø€ ÷ dñ d¡U§\¡\ð d¹×9IÑ9Ið dÉeÏlÉlñ dr3   c                ój	  € . VP                   R R	 OV P                  NV P                  N5pV P                  V4      P	                  V4      P                  4       pV P                  V4      P	                  V4      P                  4       pV P                  V4      P	                  V4      P                  4       p\        P                  P                  P                  V P                  4      p^^^V P                  3pVP                  V4      p	W@P                  ,          V	,          pVP                   R,          w  r«V P                  V4      pV P!                  V4      pV P!                  V4      pVP                   ^,          pV( pV P!                  V4      pVP"                  ^8X  dW   VP                   ^,          VP                   ^,          ,          V P$                  8X  d   VP	                  W¯V P$                  4      pVP                   V
VV P$                  38w  d-   \'        RVP                    RV
 RV RV P$                   R2	4      hVP)                  ^4      P)                  R
4      pV P*                  P)                  ^ 4      P)                  ^ 4      P)                  ^ 4      p\        P,                  ! VVP/                  VP0                  4      4      pV P3                  WÍ4      pV P4                  P/                  VP0                  4      pVV,          p\        P6                  ! V4      pVV,          p\        P8                  ! VV\        P:                  ! VP<                  4      P>                  4      p\        P                  P                  PA                  VR	\        PB                  R7      P/                  VP<                  R7      pVP                   w  pppppVP                   R	,          pVPE                  ^ ^^^^4      P	                  R	VV4      pVPE                  ^ ^^^^4      P	                  R	VV4      p\        PF                  ! VV4      p V P	                  VVVVV4      PE                  ^ ^^^^4      p!V!P	                  V
WðPH                  ,          V P                  V P                  34      p!V!RR V13,          p!V!# )Nr  z%Shape of extracted_valid_mask_blocks z	 is not (z, z) after potential reshape.©rT   r¥   rÎ   r½   rk   éþÿÿÿ)%rÏ   rŒ   r�   rñ   rÀ   r  rò   ró   r,   r_   r¾   rô   rð   Úviewrø   r  r*  r%  rí   Ú
ValueErrorrœ   rú   Úlogical_andrª   r¨   rî   rû   ÚtanhÚwhereÚfinfor¥   ÚminÚsoftmaxr«   rÑ   Úbmmrè   )"rc   rM   r-  Ú	qkv_shapeÚquery_statesÚ
key_statesÚvalue_statesÚper_dim_scale_spÚbroadcast_shapeÚper_dim_scale_sp_broadcastr¶   Úq_timeÚquery_blocksÚ
key_blocksÚvalue_blocksr·   Úoriginal_valid_maskÚextracted_valid_mask_blocksÚcondition_from_input_validityÚcondition_from_causalityÚfinal_condition_for_whererK   Úsoftcap_valÚprobabilitiesÚb_dimÚn_dimÚu_dimÚw_dimÚc_dimÚh_dimÚprob_bunÚv_bunÚ
result_bmmÚcontext_vectorss"   &&&                               r0   ry   ÚGemma3nAudioAttention.forwardº  sG  € àN�m×)Ñ)¨#¨2Ð.ÐN°·±ÐNÀÇÁÑNˆ	Ø—{‘{ =Ó1×9Ñ9¸)ÓD×OÑOÓQˆØ—[‘[ Ó/×7Ñ7¸	ÓB×MÑMÓOˆ
Ø—{‘{ =Ó1×9Ñ9¸)ÓD×OÑOÓQˆä Ÿ8™8×.Ñ.×7Ñ7¸×8JÑ8JÓKÐà˜a  D§M¡MÐ2ˆØ%5×%:Ñ%:¸?Ó%KÐ"Ø#§l¡lÕ2Ð5OÕOˆà)×/Ñ/°Õ3Ñˆ
à×-Ñ-¨lÓ;ˆØ×0Ñ0°Ó<ˆ
Ø×2Ñ2°<Ó@ˆØ'×-Ñ-¨aÕ0Ðð  $˜eÐð '+×&AÑ&AÐBUÓ&VÐ#ð (×,Ñ,°Ô1Ø+×1Ñ1°!Õ4Ð7R×7XÑ7XÐYZÕ7[Õ[Ð_c×_pÑ_pÔpà*E×*MÑ*MØ¨d×.?Ñ.?ó+Ð'ð '×,Ñ,ØØØ×Ñð1
ô 
ô
 ðØ/×5Ñ5Ð6°iÀ
¸|ð LØ$Ð% R¨×(9Ñ(9Ð':Ð:TðVóð ð )D×(MÑ(MÈaÓ(P×(ZÑ(ZÐ[]Ó(^Ð%ð $(×#?Ñ#?×#IÑ#IÈ!Ó#L×#VÑ#VÐWXÓ#Y×#cÑ#cÐdeÓ#fÐ ô
 %*×$5Ò$5Ø)Ø$×'Ñ'Ð(E×(LÑ(LÓMó%
Ð!ð ×1Ñ1°,ÓKˆð —l‘l—o‘o f§m¡mÓ4ˆØ˜+Õ%ˆÜ—’˜FÓ#ˆØ˜+Õ%ˆô —’Ð6¸ÄÇÂÈFÏLÉLÓ@Y×@]Ñ@]Ó^ˆÜŸ™×+Ñ+×3Ñ3°FÀÌ%Ï-É-Ð3ÓX×[Ñ[Ðbn×btÑbtÐ[Óuˆð -:×,?Ñ,?Ñ)ˆˆu�e˜U EØ×"Ñ" 2Õ&ˆØ ×(Ñ(¨¨A¨q°!°QÓ7×?Ñ?ÀÀEÈ5ÓQˆØ×$Ñ$ Q¨¨1¨a°Ó3×;Ñ;¸BÀÀuÓMˆÜ—Y’Y˜x¨Ó/ˆ
Ø$×,Ñ,¨U°E¸5À%ÈÓO×WÑWÐXYÐ[\Ð^_ÐabÐdeÓfˆØ)×1Ñ1àØ §?¡?Õ2Ø—‘Ø—‘ð	ó
ˆð *¨!¨W¨f¨W¨*Õ5ˆàÐr3   )rì   rè   r�   rí   r�   r�   rò   rú   ré   rê   rŒ   rð   rñ   rø   rî   rû   ró   )r5   r6   r7   r8   r^   rù   r  r  r*  ry   r;   r<   r{   r|   s   @@r0   râ   râ   A  sG   ù‡ € ÷ió iò<'÷ð ÷ð ÷,.'ð .'÷`d÷ dð dr3   râ   c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚGemma3nAudioCumulativeGroupNormi!  a¼  Applies Group Normalization cumulatively over the time dimension.

This layer normalizes the input by calculating the mean and variance
cumulatively over the time dimension (dim 1). The statistics are computed
over all feature dimensions (specified by `feature_dims` and `num_channels`)
for elements marked as valid by the optional `mask`.

If a `mask` is provided (True for valid, False for invalid/padded),
invalid time steps do not contribute to the statistics calculation, and
their corresponding output values are zeroed out.

Scale and bias, if enabled, are applied per-channel (last dimension).
This behavior is similar to JAX's `GroupNormalization` with `num_groups=1`
and `cumulative=True`.
c                ó<   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[/# )r*   Únum_channelsÚfeature_dimsrU   )rW   r   rX   )r.   r/   s   "€r0   r1   Ú,Gemma3nAudioCumulativeGroupNorm.__annotate__2  s2   ø€ ÷ Nñ NáðNñ ™s•mðNñ ñ	Nr3   c           	     ó0  <€ \         SV `  4        Wn        \        V4      V n        W0n        \        P                  ! \        P                  ! V4      4      V n
        \        \        ^^\        V P                  4      ,           ^,           4      4      V n        R# )r*   N)r]   r^   r[  rO   r\  rU   r_   r`   r,   ra   rb   ÚrangeÚlenÚreduction_axes)rc   r[  r\  rU   rd   s   &&&&€r0   r^   Ú(Gemma3nAudioCumulativeGroupNorm.__init__2  sn   ø€ ô 	‰ÑÔØ(ÔÜ! ,Ó/ˆÔØŒô —l’l¤5§:¢:¨lÓ#;Ó<ˆŒô $¤E¨!¨Q´°T×5FÑ5FÓ1GÕ-GÈ!Õ-KÓ$LÓMˆÖr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rs   rg   )r.   r/   s   "€r0   r1   r]  D  s'   ø€ ÷ G,ñ G,¡U§\¡\ð G,±e·l±lñ G,r3   c                óx  € V P                   V P                  3,           pVP                  R,          V8w  d#   \        RVP                  R,           RV 24      hVP                  p\
        P                  pVP                  V4      p\
        P                  ! WTR7      p\
        P                  ! WPP                  RR7      p\
        P                  ! V^R7      p\
        P                  ! W`P                  RR7      p	\
        P                  ! V	^R7      p
\
        P                  ! V
RR	7      pW‹,          pW\,
          P                  ^4      p\
        P                  ! WÐP                  RR7      p\
        P                  ! V^R7      pWû,          pW\,
          \
        P                  ! VV P                  ,           4      ,          pV P                   P                  V4      p^.VP#                  4       ^,
          ,          V P                  .,           pVVP%                  V4      ,          pVV,          pVP                  V4      # )
z¶Applies cumulative group norm, optionally using a mask.

Args:
  hidden_states: Input tensor, shape [B, T, *feature_dims, C].

Returns:
  Normalized tensor with the same shape as x.
r  zInput tensor shape suffix z> does not match expected suffix (feature_dims + num_channels) rÎ   T©rT   rj   r©   r‡   )r8  )r\  r[  rÏ   r3  r¥   r,   r«   rª   Ú	ones_likeÚsumra  ÚcumsumÚclamprm   ÚrsqrtrU   rb   rT   r2  )rc   rM   Úexpected_input_suffixÚinput_dtypeÚ
calc_dtypeÚx_calcÚ	mask_calcÚsum_values_at_tÚcum_sum_valuesÚelements_in_group_at_tÚcum_count_elementsÚsafe_cum_count_elementsÚcum_meanÚsquared_diff_from_meanÚsum_sq_diff_at_tÚcum_sum_sq_diffÚcum_varianceÚnormalized_xÚscaleÚscale_view_shapeÚfinal_outputs   &&                   r0   ry   Ú'Gemma3nAudioCumulativeGroupNorm.forwardD  sã  € ð !%× 1Ñ 1°T×5FÑ5FÐ4HÕ HÐØ×Ñ˜rÕ"Ð&;Ô;ÜØ,¨]×-@Ñ-@ÀÕ-DÐ,Eð F9Ø9NÐ8OðQóð ð
 $×)Ñ)ˆä—]‘]ˆ
Ø×!Ñ! *Ó-ˆô —O’O FÔ=ˆ	ô  Ÿ)š) F×0CÑ0CÈTÔRˆäŸš o¸1Ô=ˆô "'§¢¨9×:MÑ:MÐW[Ô!\Ðä"Ÿ\š\Ð*@ÀaÔHÐä"'§+¢+Ð.@ÀcÔ"JÐð "Õ;ˆð
 #)Õ"3×!8Ñ!8¸Ó!;ÐÜ Ÿ9š9Ð%;×ATÑATÐ^bÔcÐô  Ÿ,š,Ð'7¸QÔ?ˆð 'Õ@ˆð Õ)¬U¯[ª[¸ÈÏÉÕ9PÓ-QÕQˆð —‘—‘˜zÓ*ˆà˜3 -×"3Ñ"3Ó"5¸Õ"9Õ:¸d×>OÑ>OÐ=PÕPÐØ# e§j¡jÐ1AÓ&BÕBˆð $ iÕ/ˆà�‰˜{Ó+Ð+r3   )rU   r\  r[  ra  rb   )gü©ñÒMbP?©
r5   r6   r7   r8   r9   r^   ry   r;   r<   r{   r|   s   @@r0   rY  rY  !  s(   ù‡ € ñ÷ Nõ N÷$G,÷ G,ð G,r3   rY  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚGemma3nAudioSSCPConvBlockiŽ  zÍA single convolution block for the SubSampleConvProjection.

This block consists of a 2D convolution, followed by CumulativeGroupNorm,
and a ReLU activation. It handles manual padding for the convolution.
c                óP   <€ V ^8„  d   QhRS[ RS[RS[RS[S[S[S[S[3,          /# )r*   r�   ÚidxÚinput_freq_dimÚmanual_padding)r"   rW   rO   )r.   r/   s   "€r0   r1   Ú&Gemma3nAudioSSCPConvBlock.__annotate__•  sA   ø€ ÷ )$ñ )$á"ð)$ñ ð)$ñ ð	)$ñ
 ™c¡3©©SÐ0Õ1ñ)$r3   c           	     ó¦  <€ \         SV `  4        Wn        W@n        V^ 8X  d   ^M#V P                  P                  V^,
          ,          pV P                  P                  V,          pV P                  P
                  V,          w  rxV P                  P                  V,          w  rš\        P                  ! VVVV3Wš3RRR7      V n	        W0P                  ^ ,          ,           V P                  ^,          ,           pW¸,
          V
,          ^,           p\        VV3V P                  P                  R7      V n        \        P                  ! 4       V n        R# )r   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr†   )r[  r\  rU   N)r   r   )r]   r^   r�   r…  Ússcp_conv_channel_sizeÚsscp_conv_kernel_sizeÚsscp_conv_stride_sizer_   ÚConv2dÚconvrY  Ússcp_conv_group_norm_epsÚnormÚReLUÚ
activation)rc   r�   rƒ  r„  r…  rˆ  r‰  Úkernel_hÚkernel_wÚstride_hÚstride_wÚf_in_paddedÚ
f_out_convrd   s   &&&&&        €r0   r^   Ú"Gemma3nAudioSSCPConvBlock.__init__•  s  ø€ ô 	‰ÑÔØŒØ,Ôð  !œ8‘a¨¯©×)KÑ)KÈCÐRSÍGÕ)TˆØ—{‘{×9Ñ9¸#Õ>ˆØ!Ÿ[™[×>Ñ>¸sÕCÑˆØ!Ÿ[™[×>Ñ>¸sÕCÑˆä—I’IØ#Ø%àØðð Ð'ØØô

ˆŒ	ð %×':Ñ':¸1Õ'=Õ=À×@SÑ@SÐTUÕ@VÕVˆØ!Õ,°Õ9¸AÕ=ˆ
ä3Ø%Ø$˜Ø—‘×4Ñ4ô
ˆŒ	ô Ÿ'š'›)ˆŽr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r*   Úaudio_encodingsrt   rg   )r.   r/   s   "€r0   r1   r†  À  s#   ø€ ÷ 7ñ 7¡u§|¡|ð 7¹¿¹ñ 7r3   c                ó�  € \         P                  ! WP                  R RR7      P                  V P                  P
                  P                  4      pV P	                  V4      pVP                  ^ ^^^4      P                  4       pV P                  V4      pVP                  ^ ^^^4      P                  4       pV P                  V4      # )Úconstantræ   )ÚmodeÚvalue)ÚFr¿   r…  rª   r‘  rb   r¥   rÑ   r  r“  r•  )rc   rŸ  Úaudio_encodings_paddedÚaudio_encodings_convÚ
x_for_normÚx_normedÚaudio_encodings_normeds   &&     r0   ry   Ú!Gemma3nAudioSSCPConvBlock.forwardÀ  s®   € ô "#§¢ ×8KÑ8KÐR\ÐdgÔ!h×!kÑ!kØ�I‰I×Ñ×"Ñ"ó"
Ðð
  $Ÿy™yÐ)?Ó@Ðð *×1Ñ1°!°Q¸¸1Ó=×HÑHÓJˆ
Ø—9‘9˜ZÓ(ˆà!)×!1Ñ!1°!°Q¸¸1Ó!=×!HÑ!HÓ!JÐØ�‰Ð5Ó6Ð6r3   )r•  r�   r‘  r…  r“  ))r   r   r   r   r  r|   s   @@r0   r�  r�  Ž  s$   ù‡ € ñ÷)$õ )$÷V7÷ 7ð 7r3   r�  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú#Gemma3nAudioSubSampleConvProjectioniÓ  c                ó    <€ V ^8„  d   QhRS[ /# r€   r‚   )r.   r/   s   "€r0   r1   Ú0Gemma3nAudioSubSampleConvProjection.__annotate__Ô  s   ø€ ÷ 7mñ 7mÑ1ñ 7mr3   c                óÞ  <€ \         SV `  4        Wn        VP                  p. p. p\	        ^4       F‹  pVP
                  V,          w  rgVP                  V,          w  r‰^ p
V^,
          p^p^pVVV
V3pVP                  V4       W,,           V,           pW÷,
          V	,          ^,           pVP                  V4       TpK�  	  \        ^ VP                  VV^ ,          R7      V n	        \        ^V^ ,          VV^,          R7      V n
        VP                  R,          pVR,          pVV,          V n        \        P                  ! V P                  V P                  P                  RR7      V n        R# )r*   )rƒ  r„  r�   r…  Fr…   Nrk   )r]   r^   r�   Úinput_feat_sizer_  rŽ  r�  Úappendr�  Úconv_0Úconv_1r�  Úinput_proj_in_featuresr_   r•   r�   Úinput_proj_linear)rc   r�   Úcurrent_f_for_block_inputÚcalculated_block_paddingÚcalculated_f_out_dimsÚir–  r—  r˜  r™  Ú	pad_t_topÚpad_t_bottomÚ
pad_f_leftÚpad_f_rightÚmanual_padding_tuplerš  Úf_out_after_convÚfinal_c_outÚfinal_f_outrd   s   &&                 €r0   r^   Ú,Gemma3nAudioSubSampleConvProjection.__init__Ô  sr  ø€ Ü‰ÑÔØŒà$*×$:Ñ$:Ð!Ø#%Ð Ø "Ðä�q–ˆAØ!'×!=Ñ!=¸aÕ!@ÑˆHØ!'×!=Ñ!=¸aÕ!@ÑˆHð ˆIØ# a�<ˆLð ˆJØˆKð ØØØð	$Ð ð %×+Ñ+Ð,@ÔAð 4Õ@À;ÕNˆKØ +Õ 6¸8ÕCÀaÕGÐØ!×(Ñ(Ð)9Ô:Ø(8Ò%ñ= ô@ 0ØØ!×1Ñ1ØØ3°AÕ6ô	
ˆŒô 0ØØ0°Õ3ØØ3°AÕ6ô	
ˆŒð ×3Ñ3°BÕ7ˆØ+¨BÕ/ˆØ&1°KÕ&?ˆÔ#Ü!#§¢¨4×+FÑ+FÈÏÉ×H_ÑH_ÐfkÔ!lˆÖr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rž  rg   )r.   r/   s   "€r0   r1   r®    s#   ø€ ÷ ñ ¡u§|¡|ð ¹¿¹ñ r3   c                ó   € VP                  ^4      pV P                  V4      pV P                  V4      pVP                  w  rErgVP	                  ^ ^^^4      P                  4       pVP                  WFWu,          4      p	V P                  V	4      p
V
# ©r§   )rœ   r²  r³  rÏ   rÑ   r  r2  rµ  )rc   rŸ  Úaudio_encodings_reshapedr  r  Úc_outÚt_outÚf_outÚ
x_permutedÚoutput_flattenedÚoutputs   &&         r0   ry   Ú+Gemma3nAudioSubSampleConvProjection.forward  s…   € ð $3×#<Ñ#<¸QÓ#?Ð Ø�K‰KÐ0Ó1ˆØ�K‰K˜‹Nˆà!"§¡Ñˆ�%à—Y‘Y˜q ! Q¨Ó*×5Ñ5Ó7ˆ
Ø%Ÿ?™?¨1°Uµ]ÓCÐØ×'Ñ'Ð(8Ó9ˆØˆr3   )r�   r²  r³  r´  rµ  ©	r5   r6   r7   r8   r^   ry   r;   r<   r{   r|   s   @@r0   r¬  r¬  Ó  s!   ù‡ € ÷7mó 7m÷r÷ ð r3   r¬  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma3nAudioConformerAttentioni  c                ó    <€ V ^8„  d   QhRS[ /# r€   r‚   )r.   r/   s   "€r0   r1   Ú+Gemma3nAudioConformerAttention.__annotate__  s   ø€ ÷ Añ AÑ1ñ Ar3   c                ó  <€ \         SV `  4        Wn        V P                  P                  V n        \
        P                  ! \        P                  ! V P                  P                  4      R R7      V n	        \        V P                  P                  4      V n        \        V4      V n        \
        P                  ! V P                  V P                  P                  R R7      V n        \        V P                  P                  4      V n        R# ©Fr‰   r…   N)r]   r^   r�   r�   Úpost_in_featuresr_   r›   r,   rõ   Úgradient_clippingrR   Úpre_attn_normrâ   Úattnr•   ÚpostÚ	post_norm©rc   r�   rd   s   &&€r0   r^   Ú'Gemma3nAudioConformerAttention.__init__  sª   ø€ Ü‰ÑÔØŒØ $§¡× 7Ñ 7ˆÔÜ!#§¢¬5¯<ª<¸¿¹×8UÑ8UÓ+VÐchÔ!iˆÔÜ+¨D¯K©K×,CÑ,CÓDˆÔÜ)¨&Ó1ˆŒ	Ü—I’I˜d×3Ñ3°T·[±[×5LÑ5LÐSXÔYˆŒ	Ü'¨¯©×(?Ñ(?Ó@ˆŽr3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©r*   rŸ  r+   rt   r.  )r.   r/   s   "€r0   r1   rÒ  '  s8   ø€ ÷ Oñ O¡u§|¡|ð OÁU×EUÑEUð OÑZ_×ZfÑZfñ Or3   c                ó˜  € Tp\         P                  ! WP                  ) V P                  4      pV P                  V4      pV P	                  WB4      pVP
                  w  rgr‰VP                  WgW‰,          4      p
V P                  V
4      p\         P                  ! WP                  ) V P                  4      pW0P                  V4      ,           # rv   )	r,   ri  rÖ  r×  rØ  rÏ   rÀ   rÙ  rÚ  )rc   rŸ  r+   Úaudio_encodings_input_to_attnÚaudio_encodings_normÚaudio_encodings_attn_outr  r  rŒ   r�   rÆ  s   &&&        r0   ry   Ú&Gemma3nAudioConformerAttention.forward'  s²   € Ø(7Ð%ÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ#×1Ñ1°/ÓBÐà#'§9¡9Ð-AÓ#RÐ ð %=×$BÑ$BÑ!ˆˆiØ#;×#CÑ#CÀAÈ)ÕJ^Ó#_Ð àŸ)™)Ð$<Ó=ˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ,¯~©~¸oÓ/NÕNÐNr3   )rØ  r�   rÖ  rÙ  rÕ  rÚ  r×  rÎ  r|   s   @@r0   rÐ  rÐ    s#   ù‡ € ÷Aó A÷O÷ Oð Or3   rÐ  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú Gemma3nAudioConformerFeedForwardi8  c                ó    <€ V ^8„  d   QhRS[ /# r€   r‚   )r.   r/   s   "€r0   r1   Ú-Gemma3nAudioConformerFeedForward.__annotate__9  s   ø€ ÷ 
Añ 
AÑ1ñ 
Ar3   c                ó´  <€ \         SV `  4        Wn        \        P                  ! \
        P                  ! V P                  P                  4      R R7      V n        \        V P                  P                  4      V n
        \        P                  ! V P                  P                  V P                  P                  ^,          R R7      V n        \        P                  ! V P                  P                  ^,          V P                  P                  R R7      V n        \        V P                  P                  4      V n        V P                  P                  V n        R# rÔ  )r]   r^   r�   r_   r›   r,   rõ   rÖ  rR   r�   Úpre_layer_normr•   Úffw_layer_1Úffw_layer_2Úpost_layer_normÚconf_residual_weightÚpost_layer_scalerÛ  s   &&€r0   r^   Ú)Gemma3nAudioConformerFeedForward.__init__9  sÙ   ø€ Ü‰ÑÔØŒä!#§¢¬5¯<ª<¸¿¹×8UÑ8UÓ+VÐchÔ!iˆÔä,¨T¯[©[×-DÑ-DÓEˆÔÜŸ9š9 T§[¡[×%<Ñ%<¸d¿k¹k×>UÑ>UÐXYÕ>YÐ`eÔfˆÔÜŸ9š9 T§[¡[×%<Ñ%<¸qÕ%@À$Ç+Á+×BYÑBYÐ`eÔfˆÔÜ-¨d¯k©k×.EÑ.EÓFˆÔØ $§¡× @Ñ @ˆÖr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rž  rg   )r.   r/   s   "€r0   r1   rç  E  s'   ø€ ÷ 	Dñ 	D¡u§|¡|ð 	D¹¿¹ñ 	Dr3   c                ó®  € Tp\         P                  ! WP                  ) V P                  4      pV P                  V4      pV P	                  V4      p\
        P                  P                  V4      pV P                  V4      p\         P                  ! WP                  ) V P                  4      pV P                  V4      pW!V P                  ,          ,           # rv   )r,   ri  rÖ  ré  rê  r_   r¾   Úsilurë  rì  rî  )rc   rŸ  Úresiduals   && r0   ry   Ú(Gemma3nAudioConformerFeedForward.forwardE  s¯   € Ø"ˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ×-Ñ-¨oÓ>ˆØ(,×(8Ñ(8¸Ó(IˆÜŸ-™-×,Ñ,¨_Ó=ˆØ(,×(8Ñ(8¸Ó(IˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ×.Ñ.¨Ó?ˆØ¨T×-BÑ-BÕBÕCÐCr3   )r�   rê  rë  rÖ  rì  rî  ré  rÎ  r|   s   @@r0   rå  rå  8  s#   ù‡ € ÷
Aó 
A÷	D÷ 	Dð 	Dr3   rå  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú Gemma3nAudioConformerLightConv1diQ  c                ó    <€ V ^8„  d   QhRS[ /# r€   r‚   )r.   r/   s   "€r0   r1   Ú-Gemma3nAudioConformerLightConv1d.__annotate__R  s   ø€ ÷ Dñ DÑ1ñ Dr3   c           
     óð  <€ \         SV `  4        Wn        \        V P                  P                  V P                  P
                  R 7      V n        \        P                  ! V P                  P                  V P                  P                  ^,          RR7      V n	        \        P                  ! V P                  P                  V P                  P                  V P                  P                  ^^ V P                  P                  RR7      V n        \        P                  ! \        P                  ! V P                  P                   4      RR7      V n        \        V P                  P                  V P                  P
                  R 7      V n        \        P                  ! V P                  P                  V P                  P                  RR7      V n        V P                  P                  ^,
          V n        R# )©rU   Fr…   )rˆ  r‰  rŠ  r‹  rŒ  Úgroupsr†   r‰   N)r]   r^   r�   rR   r�   Úrms_norm_epsré  r_   r•   Úlinear_startÚConv1dÚconf_conv_kernel_sizeÚdepthwise_conv1dr›   r,   rõ   rÖ  Ú	conv_normÚ
linear_endÚcausal_paddingrÛ  s   &&€r0   r^   Ú)Gemma3nAudioConformerLightConv1d.__init__R  s?  ø€ Ü‰ÑÔØŒä,¨T¯[©[×-DÑ-DÈ$Ï+É+×JbÑJbÔcˆÔÜŸIšI d§k¡k×&=Ñ&=¸t¿{¹{×?VÑ?VÐYZÕ?ZÐafÔgˆÔÜ "§	¢	ØŸ™×/Ñ/ØŸ™×0Ñ0ØŸ™×9Ñ9ØØØ—;‘;×*Ñ*Øô!
ˆÔô "$§¢¬5¯<ª<¸¿¹×8UÑ8UÓ+VÐchÔ!iˆÔÜ'¨¯©×(?Ñ(?ÀTÇ[Á[×E]ÑE]Ô^ˆŒÜŸ)š) D§K¡K×$;Ñ$;¸T¿[¹[×=TÑ=TÐ[`ÔaˆŒà"Ÿk™k×?Ñ?À!ÕCˆÖr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rž  rg   )r.   r/   s   "€r0   r1   rø  g  s#   ø€ ÷ ñ ¡u§|¡|ð ¹¿¹ñ r3   c                ó@  € TpV P                  V4      pV P                  V4      p\        P                  P                  P                  VRR7      pVP                  ^ ^^4      p\        P                  ! W0P                  ^ 34      pV P                  V4      pVP                  ^ ^^4      p\        P                  ! WP                  ) V P                  4      pV P                  V4      p\        P                  P                  V4      pV P                  V4      pW,           pV# )r§   r©   rk   )ré  rý  r,   r_   r¾   ÚglurÑ   r¤  r¿   r  r   ri  rÖ  r  rò  r  )rc   rŸ  Úaudio_encodings_residualÚaudio_encodings_permutedÚaudio_encodings_permuted_paddedrÌ  s   &&    r0   ry   Ú(Gemma3nAudioConformerLightConv1d.forwardg  sú   € Ø#2Ð à×-Ñ-¨oÓ>ˆØ×+Ñ+¨OÓ<ˆÜŸ(™(×-Ñ-×1Ñ1°/ÀrÐ1ÓJˆà#2×#:Ñ#:¸1¸aÀÓ#CÐ ä*+¯%ª%Ð0H×K^ÑK^Ð`aÐJbÓ*cÐ'Ø×/Ñ/Ð0OÓPˆà)×1Ñ1°!°Q¸Ó:ˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØŸ.™.¨Ó9ˆÜŸ-™-×,Ñ,¨_Ó=ˆØŸ/™/¨/Ó:ˆØ Õ;ˆØˆr3   )r  r�   r  r   rÖ  r  rý  ré  rÎ  r|   s   @@r0   rö  rö  Q  s    ù‡ € ÷Dó D÷*÷ ð r3   rö  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma3nAudioConformerBlocki|  c                ó    <€ V ^8„  d   QhRS[ /# r€   r‚   )r.   r/   s   "€r0   r1   Ú'Gemma3nAudioConformerBlock.__annotate__}  s   ø€ ÷ 	<ñ 	<Ñ1ñ 	<r3   c                óÒ  <€ \         SV `  4        Wn        \        V P                  4      V n        \        V P                  4      V n        \        V P                  4      V n        \        V P                  4      V n	        \        P                  ! \        P                  ! V P                  P                  4      R R7      V n        \        V P                  P                   4      V n        R# ©Fr‰   N)r]   r^   r�   rå  Úffw_layer_startrÐ  Ú	attentionrö  Úlconv1dÚffw_layer_endr_   r›   r,   rõ   rÖ  rR   r�   r“  rÛ  s   &&€r0   r^   Ú#Gemma3nAudioConformerBlock.__init__}  s’   ø€ Ü‰ÑÔØŒä?ÀÇÁÓLˆÔÜ7¸¿¹ÓDˆŒÜ7¸¿¹ÓDˆŒÜ=¸d¿k¹kÓJˆÔÜ!#§¢¬5¯<ª<¸¿¹×8UÑ8UÓ+VÐchÔ!iˆÔÜ" 4§;¡;×#:Ñ#:Ó;ˆŽ	r3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rÞ  r.  )r.   r/   s   "€r0   r1   r  ˆ  s3   ø€ ÷ ñ ¡u§|¡|ð ÁU×EUÑEUð ÑZ_×ZfÑZfñ r3   c                ón  € V P                  V4      pV P                  W4      pV( pWP                  R4      P                  VP                  4      ,          pV P                  V4      pV P                  V4      p\        P                  ! WP                  ) V P                  4      pV P                  V4      pV# )r§   rk   )r  r  rœ   rª   r¥   r  r  r,   ri  rÖ  r“  )rc   rŸ  r+   Úvalidity_mask_for_lconvÚaudio_encodings_for_lconv_inputrÌ  s   &&&   r0   ry   Ú"Gemma3nAudioConformerBlock.forwardˆ  s§   € Ø×.Ñ.¨Ó?ˆØŸ.™.¨ÓIˆØ#1 /ÐØ*9×<]Ñ<]Ð^`Ó<a×<dÑ<dØ×!Ñ!ó=
õ +
Ð'ð Ÿ,™,Ð'FÓGˆà×,Ñ,¨_Ó=ˆÜŸ+š+ o×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ—‘˜?Ó+ˆØˆr3   )r  r�   r  r  rÖ  r  r“  rÎ  r|   s   @@r0   r  r  |  s   ù‡ € ÷	<ó 	<÷÷ ð r3   r  c                   ó^   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lV 3R lltRtVtV ;t	# )	ÚGemma3nTextScaledWordEmbeddingi—  zT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r*   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scale)rW   rX   )r.   r/   s   "€r0   r1   Ú+Gemma3nTextScaledWordEmbedding.__annotate__œ  s0   ø€ ÷ Rñ R¡sð R¹3ð RÉSð RÑ_dñ Rr3   c                ó–   <€ \         SV `  WV4       W@n        \        P                  ! \
        P                  ! V4      R R7      V n        R# r  )r]   r^   Úscalar_embed_scaler_   r›   r,   rõ   r"  )rc   r  r   r!  r"  rd   s   &&&&&€r0   r^   Ú'Gemma3nTextScaledWordEmbedding.__init__œ  s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÔQˆÖr3   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r*   Ú	input_idsrg   )r.   r/   s   "€r0   r1   r#  ¡  s   ø€ ÷ Sñ S¡§¡ñ Sr3   c                óŠ   <€ \         SV `  V4      V P                  P                  V P                  P
                  4      ,          # rv   )r]   ry   r"  rª   rb   r¥   )rc   r(  rd   s   &&€r0   ry   Ú&Gemma3nTextScaledWordEmbedding.forward¡  s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÕRÐRr3   )r"  r%  )r‡   r  r|   s   @@r0   r  r  —  s(   ù‡ € ñ÷Rõ R÷
S÷ Só Sr3   r  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚGemma3nTextLaurelBlocki¥  z Learned Augmented Residual Layerc                ó    <€ V ^8„  d   QhRS[ /# r€   ©r$   )r.   r/   s   "€r0   r1   Ú#Gemma3nTextLaurelBlock.__annotate__¨  s   ø€ ÷ fñ fÑ0ñ fr3   c                ó¼  <€ \         SV `  4        Wn        \        P                  ! V P                  P
                  V P                  P                  R R7      V n        \        P                  ! V P                  P                  V P                  P
                  R R7      V n        \        V P                  P
                  V P                  P                  R7      V n        R# )Fr…   rú  N)r]   r^   r�   r_   r•   r�   Úlaurel_rankÚlinear_leftÚlinear_rightrR   rü  Úpost_laurel_normrÛ  s   &&€r0   r^   ÚGemma3nTextLaurelBlock.__init__¨  sŠ   ø€ Ü‰ÑÔØŒäŸ9š9 T§[¡[×%<Ñ%<¸d¿k¹k×>UÑ>UÐ\aÔbˆÔÜŸIšI d§k¡k×&=Ñ&=¸t¿{¹{×?VÑ?VÐ]bÔcˆÔÜ .¨t¯{©{×/FÑ/FÈDÏKÉK×LdÑLdÔ eˆÖr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rs   rg   )r.   r/   s   "€r0   r1   r/  °  s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r3   c                óx   € V P                  V4      pV P                  V4      pV P                  V4      pW,           # rv   )r2  r3  r4  )rc   rM   Úlaurel_hidden_statesÚnormed_laurel_hidden_statess   &&  r0   ry   ÚGemma3nTextLaurelBlock.forward°  s@   € Ø-1×-=Ñ-=¸mÓ-LÐØ-1×->Ñ->Ð?SÓ-TÐØ&*×&;Ñ&;Ð<PÓ&QÐ#ØÕ:Ð:r3   )r�   r2  r3  r4  r  r|   s   @@r0   r,  r,  ¥  s#   ù‡ € Ù*÷fó f÷;÷ ;ð ;r3   r,  c                   óf   a a€ ] tR tRt oR	V3R lV 3R llltV3R lR ltV3R lR ltRtVtV ;t	# )
ÚGemma3nTextMLPi·  c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r*   r�   Ú	layer_idx©r$   rW   )r.   r/   s   "€r0   r1   ÚGemma3nTextMLP.__annotate__¸  s    ø€ ÷ 	Qñ 	QÑ0ð 	Q¹Sñ 	Qr3   c                ó  <€ \         SV `  4        Wn        VP                  V n        VP                  V,          V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        VP                  V,          V n        R# ©Fr…   N)r]   r^   r�   r�   Úintermediate_sizer_   r•   Ú	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fnÚactivation_sparsity_patternÚactivation_sparsity©rc   r�   r?  rd   s   &&&€r0   r^   ÚGemma3nTextMLP.__init__¸  s¼   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9¸)Õ!DˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Õ6ˆŒØ#)×#EÑ#EÀiÕ#PˆÖ r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rs   rg   )r.   r/   s   "€r0   r1   rA  Ã  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                óÞ   € V P                  V4      pV P                  R 8”  d   V P                  V4      pV P                  V4      pV P	                  V4      pV P                  W4,          4      pV# )ræ   )rE  rK  Ú_gaussian_topkrI  rF  rG  )rc   rM   rE  ÚactivationsrF  rG  s   &&    r0   ry   ÚGemma3nTextMLP.forwardÃ  sa   € Ø—N‘N =Ó1ˆ	Ø×#Ñ# cÔ)Ø×+Ñ+¨IÓ6ˆIØ—k‘k )Ó,ˆØ—,‘,˜}Ó-ˆØ—N‘N ;Õ#8Ó9ˆ	ØÐr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r*   Úinputsrt   rg   )r.   r/   s   "€r0   r1   rA  Ì  s#   ø€ ÷ 5ñ 5¡U§\¡\ð 5±e·l±lñ 5r3   c                óò  € \         P                  ! V P                  \         P                  VP                  R 7      p\         P
                  P                  P                  ^ ^4      pVP                  V4      pVP                  VP                  4      p\         P                  ! VRRR7      p\         P                  ! VRRRR7      pWVV,          ,           p\        P                  P                  W,
          4      # )©r¥   r¨   Tre  F)rT   rj   Úunbiasedrk   )r,   rõ   rK  r«   r¨   ÚdistributionsÚnormalÚNormalÚicdfr¯   r¥   rn   Ústdr_   r¾   Úrelu)rc   rT  Útarget_sparsity_tensorÚnormal_distÚstd_multiplierÚinputs_meanÚ
inputs_stdÚcutoff_xs   &&      r0   rP  ÚGemma3nTextMLP._gaussian_topkÌ  s¶   € Ü!&§¢¨d×.FÑ.FÌeÏmÉmÐdj×dqÑdqÔ!rÐô ×)Ñ)×0Ñ0×7Ñ7¸¸1Ó=ˆØ'2×'7Ñ'7Ð8NÓ'OˆØ'×,Ñ,¨V¯\©\Ó:ˆÜ—j’j ¨R¸Ô>ˆÜ—Y’Y˜v¨2°tÀeÔLˆ
Ø¨nÕ!<Õ<ˆÜ�}‰}×!Ñ! &Õ"3Ó4Ð4r3   )rI  rK  r�   rG  rE  r�   rD  rF  )r   )
r5   r6   r7   r8   r^   ry   rP  r;   r<   r{   r|   s   @@r0   r<  r<  ·  s*   ù‡ € ÷	Qõ 	Q÷ð ÷5÷ 5ð 5r3   r<  c                   óœ   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltV3R lR ltV3R	 lR
 ltV3R lR lt	V3R lR lt
RtVtV ;t# )ÚGemma3nTextAltUpiÝ  a…  Alternating Updates (AltUp)

The AltUp module wraps transformer layers. The `predict` step modifies the
input to the transformer layer, and the `correct` step propagates the output
of the transformer layer to the sparsely updated dimensions.

See more in the research paper:

https://proceedings.neurips.cc/paper_files/paper/2023/file/f2059277ac6ce66e7e5543001afa8bb5-Paper-Conference.pdf
c                ó    <€ V ^8„  d   QhRS[ /# r€   r.  )r.   r/   s   "€r0   r1   ÚGemma3nTextAltUp.__annotate__é  s   ø€ ÷ kñ kÑ0ñ kr3   c                ót  <€ \         SV `  4        Wn        \        P                  ! \
        P                  ! V P                  P                  4      4      V n        \        P                  ! V P                  P                  V P                  P                  R R7      V n        \        P                  ! V P                  P                  V P                  P                  ^,          R R7      V n        \        P                  ! V P                  P                  V P                  P                  R R7      V n        \        V P                  P                  V P                  P                  R7      V n        \        P"                  ! \
        P$                  ! V P                  P                  R,          4      R R7      V n        R# )Fr…   rú  r‰   Nç      ð¿)r]   r^   r�   r_   r`   r,   rï   r�   Úcorrect_output_scaler•   Úaltup_num_inputsÚcorrection_coefsÚprediction_coefsÚmodality_routerrR   rü  Úrouter_normr›   rõ   Úrouter_input_scalerÛ  s   &&€r0   r^   ÚGemma3nTextAltUp.__init__é  s  ø€ Ü‰ÑÔØŒÜ$&§L¢L´·²¸T¿[¹[×=TÑ=TÓ1UÓ$VˆÔ!Ü "§	¢	¨$¯+©+×*FÑ*FÈÏÉ×HdÑHdÐkpÔ qˆÔÜ "§	¢	¨$¯+©+×*FÑ*FÈÏÉ×HdÑHdÐfgÕHgÐnsÔ tˆÔÜ!Ÿyšy¨¯©×)@Ñ)@À$Ç+Á+×B^ÑB^ÐejÔkˆÔÜ)¨$¯+©+×*AÑ*AÀtÇ{Á{×G_ÑG_Ô`ˆÔÜ"$§)¢)¬E¯LªL¸¿¹×9PÑ9PÐRVÕ9VÓ,WÐdiÔ"jˆÖr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r*   r  rt   rg   )r.   r/   s   "€r0   r1   rh  ó  s#   ø€ ÷ 5ñ 5©5¯<©<ð 5¹E¿L¹Lñ 5r3   c                óÎ   € V P                  V4      V P                  ,          pV P                  V4      p\        P                  ! VP                  4       4      P                  V4      # rv   )rp  rq  ro  r,   r5  rX   rw   )rc   r  Úrouter_inputsÚrouteds   &&  r0   Úcompute_router_modalitiesÚ*Gemma3nTextAltUp.compute_router_modalitiesó  sM   € Ø×(Ñ(¨Ó+¨d×.EÑ.EÕEˆØ×%Ñ% mÓ4ˆÜ�zŠz˜&Ÿ,™,›.Ó)×1Ñ1°!Ó4Ð4r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rs   rg   )r.   r/   s   "€r0   r1   rh  ø  s#   ø€ ÷ ?ñ ?¡U§\¡\ð ?±e·l±lñ ?r3   c                ó   € V P                  WP                  P                  ,          4      pV P                  '       dr   V P                  P                  eZ   V P
                  P                  P                  P                  V P                  P                  ) V P                  P                  4       V P                  V4      P                  ! . VP                  RR OV P                  P                  NV P                  P                  N5!  P                  ^ ^^^4      p\        P                  ! VP                  ^^^^ 4      V4      pVP                  ^^ ^^4      pWA,          pVP                  4       P!                  V4      # )a…  Predicts the output of a layer using a trainable map.

Args:
    hidden_states: A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` derived by
        stacking the input embeddings and preprocessing the last `num_altup_inputs - 1` matrices.

Returns:
    A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` containing the predictions.
Nrk   )rw  r�   Úaltup_active_idxÚtrainingÚaltup_coef_cliprn  rb   ÚdataÚclamp_rÀ   rÏ   rl  rÑ   r,   rÒ   r  rw   )rc   rM   Ú
modalitiesÚ	all_coefsÚpredictionss   &&   r0   ÚpredictÚGemma3nTextAltUp.predictø  sA  € ð ×3Ñ3°MÇ+Á+×B^ÑB^Õ4_Ó`ˆ
à�=�=ˆ=˜TŸ[™[×8Ñ8ÒDØ×!Ñ!×(Ñ(×-Ñ-×4Ñ4°d·k±k×6QÑ6QÐ5QÐSW×S^ÑS^×SnÑSnÔoð ×!Ñ! *Ó-ß‰Wðð iØ ×&Ñ& s¨Ð+ðiØ-1¯[©[×-IÑ-IðiØKOÏ;É;×KgÑKgóiç‰W�Q˜˜1˜aÓ ð 	ô —l’l =×#8Ñ#8¸¸A¸qÀ!Ó#DÀiÓPˆØ!×)Ñ)¨!¨Q°°1Ó5ˆØÕ$ˆØ×%Ñ%Ó'×/Ñ/°Ó>Ð>r3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r*   r‚  Ú	activatedrt   rg   )r.   r/   s   "€r0   r1   rh    s.   ø€ ÷ 9ñ 9¡5§<¡<ð 9¹E¿L¹Lð 9ÉUÏ\É\ñ 9r3   c                ó   € V P                  V4      pW!V P                  P                  ,          ,
          pVP                  V P                  P                  ^^^4      pV P
                  '       d›   V P                  P                  eƒ   V P                  P                  P                  V P                  P                  ) V P                  P                  4      p\        P                  P                  P                  W5RR7      R,           pMV P                  V4      R,           pVP                  ^^ ^4      P                  R4      p\        P                   ! WF4      pWq,          pVP#                  4       P%                  V4      # )a  Corrects the predictions relative to the

Args:
    predictions: A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` derived by
        stacking the input embeddings and preprocessing the last `num_altup_inputs - 1` matrices.
    activated: A 3D tensor of shape `[batch_size, num_tokens, hidden_size]` containing the activated inputs.

Returns:
    A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` correcting the original
        predictions relative to the activated input embeddings.
Nr…   r‡   rk   )rw  r�   r{  Úrepeatrl  r|  r}  rm  rb   ri  r,   r_   r¾   ÚlinearrÑ   rœ   Úmulr  rw   )rc   r‚  r†  r€  Ú
innovationrb   r�  Ú	correcteds   &&&     r0   ÚcorrectÚGemma3nTextAltUp.correct  s&  € ð ×3Ñ3°IÓ>ˆ
Ø¨T¯[©[×-IÑ-IÕ!JÕJˆ
Ø×&Ñ& t§{¡{×'CÑ'CÀQÈÈ1ÓMˆ
à�=�=ˆ=˜TŸ[™[×8Ñ8ÒDØ×*Ñ*×1Ñ1×7Ñ7¸¿¹×9TÑ9TÐ8TÐVZ×VaÑVa×VqÑVqÓrˆFÜŸ™×+Ñ+×2Ñ2°:ÈDÐ2ÓQÐTWÕW‰Ià×-Ñ-¨jÓ9¸CÕ?ˆIð
 ×%Ñ% a¨¨AÓ.×8Ñ8¸Ó<ˆ	ä—I’I˜jÓ4ˆ	ØÕ ˆ	Ø×#Ñ#Ó%×-Ñ-¨iÓ8Ð8r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r*   rŒ  rt   rg   )r.   r/   s   "€r0   r1   rh  3  s'   ø€ ÷ mñ m¡§¡ð m±%·,±,ñ mr3   c                óx   € VP                  V P                  4      V P                  ,          P                  V4      # )zé
This is only defined as the `forward` so that accelerate hooks can move correctly `correct_output_scale`
(which is a nn.Parameter, not a Module) between devices when offloading. It is otherwise only used in
`scale_corrected_output`
)rw   rk  ©rc   rŒ  s   &&r0   ry   ÚGemma3nTextAltUp.forward3  s2   € ð ×!Ñ! $×";Ñ";Ó<¸t×?XÑ?XÕX×aÑaÐbkÓlÐlr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r�  rg   )r.   r/   s   "€r0   r1   rh  ;  s#   ø€ ÷ 'ñ '±·±ð 'ÁÇÁñ 'r3   c                ó$   € V P                  V4      # )zMScales the provided 3D tensor of shape [batch_size, num_tokens, hidden_size].)ry   r’  s   &&r0   Úscale_corrected_outputÚ'Gemma3nTextAltUp.scale_corrected_output;  s   € à�|‰|˜IÓ&Ð&r3   )r�   rk  rm  ro  rn  rq  rp  )r5   r6   r7   r8   r9   r^   rw  rƒ  r�  ry   r–  r;   r<   r{   r|   s   @@r0   rf  rf  Ý  sO   ù‡ € ñ	÷kó k÷5ð 5÷
?ð ?÷89ð 9÷>mð m÷'÷ 'ð 'r3   rf  c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr©   rk   )rÏ   r,   r¬   )r  Úx1Úx2s   &  r0   Úrotate_halfr›  @  sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r3   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r*   rM   Ún_reprt   r»   )r.   s   "r0   r1   r1   G  s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur3   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r½   r½   Nr½   r½   )rÏ   ÚexpandrÀ   )rM   r�  r  Únum_key_value_headsÚslenr�   s   &&    r0   Ú	repeat_kvr¢  G  sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr3   c                ór  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        \
        ,          R\        R,          R	\        R,          R
\        \        P                  \        P                  3,          /	# )r*   ÚmoduleÚqueryÚkeyr£  Úattention_maskNÚdropoutÚscalingrû   rt   )r_   ÚModuler,   rh   rX   rW   rO   )r.   s   "r0   r1   r1   S  s™   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ”S�[ð%ô �T�\ð%ô �T�\ð%ô Œ5�<‰<œŸ™Ð%Õ&ñ%r3   c                 ó¤  € Vf   V P                   R,          p\        W P                  4      p	\        W0P                  4      p
\        P                  ! WP                  ^^4      4      V,          pVe'   W·,          p\        P                  ! V4      pW·,          pVe	   W´,           p\        P                  P                  VR\        P                  R7      P                  VP                  4      p\        P                  P                  WµV P                  R7      p\        P                  ! Wº4      pVP                  ^^4      P                  4       pWË3# )Nr0  )Úpr|  rl   rk   )r�   r¢  Únum_key_value_groupsr,   rÒ   Ú	transposer5  r_   r¾   r9  r«   rª   r¥   r¨  r|  r  )r¤  r¥  r¦  r£  r§  r¨  r©  rû   Úkwargsr=  r>  Úattn_weightsÚattn_outputs   &&&&&&&&,    r0   Úeager_attention_forwardr²  S  s  € ð ‚Ø—/‘/ 4Õ'ˆä˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LàÒØ#Õ-ˆÜ—z’z ,Ó/ˆØ#Õ-ˆØÒ!Ø#Õ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$r3   c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        /# )r*   r  r®   r­   Úunsqueeze_dimr»   )r.   s   "r0   r1   r1   u  s5   € ÷ .ñ .œEŸL™Lð .¬u¯|©|ð .Ä%Ç,Á,ð .Ô_bñ .r3   c                ó„   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           # )a$  Applies Rotary Position Embedding to the query and key tensors.

Args:
    x (`torch.Tensor`): The tensor to embed.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)rœ   r›  )r  r®   r­   r´  s   &&&&r0   Úapply_rotary_pos_embr¶  u  s6   € ð" �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�Gœ A›¨Õ,Õ-Ð-r3   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3nTextAttentioni‹  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r>  r@  )r.   r/   s   "€r0   r1   Ú!Gemma3nTextAttention.__annotate__Œ  s   ø€ ÷ .
ñ .
Ñ0ð .
¹Sñ .
r3   c                ó„  <€ \         SV `  4        Wn        W n        \	        VR 4      '       d   VP
                  V,          MRV n        V P                  R8H  V n        V P                  '       d   VP                  MRV n        \        VRVP                  VP                  ,          4      V n        VP                  VP                  ,          V n        RV n        V P                  P                   V n        RV n        V P                  P$                  V P                  P&                  ,
          pY#u;8¬  ;'       d    ^ 8„  Mu V n        VP
                  RV pV P(                  '       dQ   \+        V4      ^,
          VRRR
1,          P-                  VP
                  V,          4      ,
          V n        RV n        MRRV n        V\+        V4      ^,
          VRRR
1,          P-                  VP
                  V,          4      ,
          8H  V n        \2        P4                  ! VP                  VP                  V P                  ,          VP6                  R7      V n        \;        VP                  VP<                  R7      V n        V P(                  '       gè   \2        P4                  ! VP                  VP                  V P                  ,          VP6                  R7      V n         \2        P4                  ! VP                  VP                  V P                  ,          VP6                  R7      V n!        \;        VP                  VP<                  R7      V n"        \;        VP                  VP<                  RR	7      V n#        \2        P4                  ! VP                  V P                  ,          VP                  VP6                  R7      V n$        R# )Úlayer_typesNÚsliding_attentionr�   r‡   TFr…   )rT   rU   )rT   rU   rV   rk   )%r]   r^   r�   r?  Úhasattrr¼  Ú
layer_typeÚ
is_slidingÚsliding_windowÚgetattrr�   Únum_attention_headsr�   r   r­  r©  Úattention_dropoutÚ	is_causalÚnum_hidden_layersÚnum_kv_shared_layersÚis_kv_shared_layerr`  ÚindexÚkv_shared_layer_indexÚstore_full_length_kvr_   r•   Úattention_biasrñ   rR   rü  Úq_normrò   ró   Úk_normÚv_normÚo_proj)rc   r�   r?  Úfirst_kv_shared_layer_idxÚprev_layersrd   s   &&&  €r0   r^   ÚGemma3nTextAttention.__init__Œ  sÉ  ø€ Ü‰ÑÔØŒØ"ŒÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŸ/™/Ð-@Ñ@ˆŒØ7;··°˜f×3Ò3ÈDˆÔä ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!ØˆŒØ!%§¡×!>Ñ!>ˆÔØˆŒà$(§K¡K×$AÑ$AÀDÇKÁK×DdÑDdÕ$dÐ!Ø"+×"LÔ"LÈ1Ô"LˆÔØ×(Ñ(Ð)CÐ*CÐDˆØ×"×"Ð"ä),¨[Ó)9¸AÕ)=ÀÉDÈbÈDÕ@Q×@WÑ@WÐX^×XjÑXjÐktÕXuÓ@vÕ)vˆDÔ&Ø(-ˆDÕ%à)-ˆDÔ&à(1´S¸Ó5EÈÕ5IÈKÑX\ÐZ\ÐX\ÕL]×LcÑLcØ×"Ñ" 9Õ-óMõ 6ñ )ˆDÔ%ô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô %¨¯©¸f×>QÑ>QÔRˆŒð ×&×&Ð&ÜŸ)š)Ø×"Ñ" F×$>Ñ$>ÀÇÁÕ$NÐU[×UjÑUjôˆDŒKô Ÿ)š)Ø×"Ñ" F×$>Ñ$>ÀÇÁÕ$NÐU[×UjÑUjôˆDŒKô )¨V¯_©_À&×BUÑBUÔVˆDŒKÜ(¨V¯_©_À&×BUÑBUÐbgÔhˆDŒKä—i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽr3   c                óZ  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[S[S[ P                  S[ P                  3,          3,          R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )	r*   rM   Úposition_embeddingsr§  NrL   Úshared_kv_statesr¯  rt   )r,   rh   r	   ÚdictrW   rO   r   r   )r.   r/   s   "€r0   r1   rº  ¼  s¤   ø€ ÷ ;)ñ ;)á—|‘|ð;)ñ #Ÿ\™\ð;)ñ Ÿ™ tÕ+ð	;)ñ
  �ð;)ñ ™s¡E©%¯,©,¹¿¹Ð*DÕ$EÐEÕFÈÕMð;)ñ Ñ+Õ,ð;)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ;)r3   c           	     óî  € VP                   R R p. VORNV P                  P                  N5pVw  ršV P                  V4      P	                  V4      pV P                  V4      p\        W¹V
^R7      pVP                  ^^4      pV P                  '       dL   WPP                  ,          w  rÍVP                  VP                  4      pVP                  VP                  4      pM”V P                  V4      P	                  V4      pV P                  V4      p\        WÉV
^R7      pVP                  ^^4      pV P                  V4      P	                  V4      pV P                  V4      pVP                  ^^4      pVe1   V P                  '       g   VP!                  WÍV P"                  4      w  rÍV P$                  '       d   WÍ3WPP"                  &   \&        P(                  ! V P                  P*                  \,        4      pT! V VVVV3RV P.                  '       d   V P0                  MRRV P2                  RV P4                  /VB w  ppVP6                  ! . VORN5!  P9                  4       pV P;                  V4      pVV3# )N)r´  r¨  ræ   r©  rÁ  rk   )rÏ   r�   r�   rñ   r2  rÍ  r¶  r®  rÈ  rÊ  rª   r¨   rò   rÎ  ró   rÏ  Úupdater?  rË  r   Úget_interfaceÚ_attn_implementationr²  r|  rÄ  r©  rÁ  rÀ   r  rÐ  )rc   rM   rÕ  r§  rL   rÖ  r¯  Úinput_shapeÚhidden_shaper®   r­   r<  r=  r>  Úattention_interfacer±  r°  s   &&&&&&,          r0   ry   ÚGemma3nTextAttention.forward¼  sS  € ð $×)Ñ)¨#¨2Ð.ˆØ?˜Ð? bÐ?¨$¯+©+×*>Ñ*>Ñ?ˆà&‰ˆØ—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆÜ+¨L¸sÐRSÔTˆØ#×-Ñ-¨a°Ó3ˆð
 ×"×"Ð"Ø'7×8RÑ8RÕ'SÑ$ˆJà#Ÿ™ |×':Ñ':Ó;ˆJØ'Ÿ?™?¨<×+>Ñ+>Ó?‰LàŸ™ ]Ó3×8Ñ8¸ÓFˆJØŸ™ ZÓ0ˆJÜ-¨j¸sÐRSÔTˆJØ#×-Ñ-¨a°Ó3ˆJàŸ;™; }Ó5×:Ñ:¸<ÓHˆLØŸ;™; |Ó4ˆLØ'×1Ñ1°!°QÓ7ˆLàÒ&¨t×/F×/FÐ/FØ'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJØ×$×$Ð$Ø/9Ð/GÐŸ^™^Ñ,ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð /3¯m¯m¨m�D×*Ò*Àð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r3   )rÄ  r�   r�   rÅ  rÈ  rÀ  rÎ  rò   rÊ  r?  r¿  r­  rÐ  rÍ  rñ   r©  rÁ  rË  rÏ  ró   ©NNrÎ  r|   s   @@r0   r¸  r¸  ‹  s   ù‡ € ÷.
ó .
÷`;)÷ ;)ò ;)r3   r¸  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3nTextDecoderLayeriú  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r>  r@  )r.   r/   s   "€r0   r1   Ú$Gemma3nTextDecoderLayer.__annotate__û  s    ø€ ÷ cñ cÑ0ð c¹Sñ cr3   c                ó~  <€ \         SV `  4        Wn        VP                  V n        W n        \        W4      V n        \        WR 7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        VP                  V n        \         VP"                  ,          V n        \'        V4      V n        \+        V4      V n        \.        P0                  ! V P                  V P                  RR7      V n        \.        P0                  ! V P                  V P                  RR7      V n        \        V P                  VP                  R7      V n        R# ))r?  rú  Fr…   N)r]   r^   r�   r�   r?  r¸  Ú	self_attnr<  ÚmlprR   rü  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormÚhidden_size_per_layer_inputr   rH  rI  rf  Úaltupr,  Úlaurelr_   r•   Úper_layer_input_gateÚper_layer_projectionÚpost_per_layer_input_normrL  s   &&&€r0   r^   Ú Gemma3nTextDecoderLayer.__init__û  sB  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ"ŒÜ-¨fÓ@ˆŒÜ! &Ô>ˆŒÜ-¨d×.>Ñ.>ÀF×DWÑDWÔXˆÔÜ(6°t×7GÑ7GÈV×M`ÑM`Ô(aˆÔ%Ü)7¸×8HÑ8HÈf×NaÑNaÔ)bˆÔ&Ü*8¸×9IÑ9IÈv×ObÑObÔ*cˆÔ'à+1×+MÑ+MˆÔ(Ü˜V×5Ñ5Õ6ˆŒä% fÓ-ˆŒ
Ü,¨VÓ4ˆŒÜ$&§I¢I¨d×.>Ñ.>À×@`Ñ@`ÐglÔ$mˆÔ!Ü$&§I¢I¨d×.NÑ.NÐPT×P`ÑP`ÐglÔ$mˆÔ!Ü)7¸×8HÑ8HÈf×NaÑNaÔ)bˆÖ&r3   c                óÆ  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[S[S[ P                  S[ P                  3,          3,          R,          RS[ P                  R,          RS[ P
                  R,          RS[R,          R	S[S[,          R
S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /	# )r*   rM   rÕ  Úper_layer_inputrÖ  Nr§  Úposition_idsrL   r¯  rt   )
r,   rh   r×  rW   rO   Ú
LongTensorr	   r   r   rE   )r.   r/   s   "€r0   r1   rä    sÙ   ø€ ÷ 3%ñ 3%á—|‘|ð3%ñ #Ÿ\™\ð3%ñ Ÿ™ð	3%ñ
 ™s¡E©%¯,©,¹¿¹Ð*DÕ$EÐEÕFÈÕMð3%ñ Ÿ™ tÕ+ð3%ñ ×&Ñ&¨Õ-ð3%ñ  �ð3%ñ Ñ+Õ,ð3%ñ 
‰u�|‰|™U¡5×#4Ñ#4±e×6GÑ6GÐ#GÕHÈ4ÕOÐOÕ	Pñ3%r3   c                óÄ  € V P                   P                  V4      p	W�P                  P                  ,          p
V P	                  V
4      pV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rÞV P                  V4      pW­,           pWü,           \        P                  ! ^4      ,          pV P                  V4      pV P                  V4      pV P                  V4      pVV,           pV P                   P                  V	V4      pVV P                  P                  ,          P                  4       pV P                  P                  '       d   V P                   P!                  V4      pV P#                  V4      pV P%                  V4      p\&        P(                  ! VV4      pV P+                  V4      pV P-                  V4      pVR;;,          V,          uu&   V# )rM   r§  rÖ  rõ  rÕ  rL   :r§   NNr4   )rí  rƒ  r�   r{  rè  rî  ræ  ré  r—   Úsqrtrê  rç  rë  r�  rö   Úaltup_correct_scaler–  rï  rI  r,   Úmultiplyrð  rñ  )rc   rM   rÕ  rô  rÖ  r§  rõ  rL   r¯  r‚  Úactive_predictionÚactive_prediction_normedÚlaurel_outputrØ  rÓ   Ú
attn_gatedÚattn_laurelÚ	attn_normÚattn_ffwÚattn_ffw_normÚattn_ffw_laurel_gatedÚcorrected_predictionsÚfirst_predictions   &&&&&&&&,              r0   ry   ÚGemma3nTextDecoderLayer.forward  sÕ  € ð —j‘j×(Ñ(¨Ó7ˆØ'¯©×(DÑ(DÕEÐà#'×#7Ñ#7Ð8IÓ#JÐ ØŸ™Ð$<Ó=ˆà—.’.ñ 
Ø2ð
à)ð
ð .ð
ð &ð	
ð
 !4ð
ð ,ð
ð ñ
‰ˆð ×,Ñ,¨TÓ2ˆà&Õ-ˆ
Ø!Õ1´T·Y²Y¸q³\ÕAˆà×2Ñ2°;Ó?ˆ	Ø—8‘8˜IÓ&ˆØ×7Ñ7¸ÓAˆØ +¨mÕ ;ÐØ $§
¡
× 2Ñ 2°;Ð@UÓ VÐà0°·±×1MÑ1MÕN×TÑTÓVÐØ�;‰;×*×*Ð*Ø#Ÿz™z×@Ñ@ÐAQÓRÐð  ×4Ñ4Ð5EÓFÐØŸ;™;Ð'7Ó8ÐÜ Ÿ>š>Ð*:¸OÓLÐð  ×4Ñ4Ð5EÓFÐØ×9Ñ9Ð:JÓKÐØ˜b×!Ð%5Õ5Ó!à$Ð$r3   )rI  rí  r�   r�   rì  rè  rî  r?  rç  rï  rð  ré  rë  rñ  rê  ræ  )NNNNNNrÎ  r|   s   @@r0   râ  râ  ú  s    ù‡ € ÷có c÷*3%÷ 3%ò 3%r3   râ  c                   óÞ   a a€ ] tR tRt oRtRtR.tRR.tRtRt	Rt
RtRtR]R]/tRt]P$                  ! 4       V 3R	 l4       tR
 tR tRV3R lV 3R llltRV3R lR lltV3R ltRtVtV ;t# )ÚGemma3nPreTrainedModeliF  ÚmodelTrâ  rL   rÖ  rM   rN   c                óD	  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       EM\        V\        4      '       dû   \        P                  ! VP                  4       VP                  R,          pR\        P                  P                  P                  \        P                  ! R4      4      ,          p\        P                   ! VP"                  W#,          4       \        P$                  ! VP&                  VP(                  4       \        P                   ! VP*                  VP-                  4       4       EMþ\        V\.        4      '       d.   \        P$                  ! VP0                  VP2                  4       EM»\        V\4        4      '       d_   \        P                  ! VP6                  4       \        P$                  ! VP8                  V P:                  P<                  R,          4       EMG\        V\>        4      '       dæ   RRrTVP@                  ^,          p\B        PD                  ! \G        V4      \G        V4      ,          4      \I        V^,
          ^4      ,          pV\        PJ                  ! \        PL                  ! V4      V) ,          4      ,          p\        P                   ! VPN                  VPG                  4       PQ                  ^ 4      PQ                  ^ 4      4       EML\        V\R        4      '       dp   \        P$                  ! VPT                  V P<                  R,          4       \        P$                  ! VPV                  ^\B        PX                  ! R4      ,          4       MÇ\        V\Z        4      '       d²   VP\                   F¡  p	VP^                  p
VP`                  V	,          R8w  d   \b        VP`                  V	,          ,          p
V
! VP:                  V	R7      w  r¼\        P                   ! \e        W R24      V4       \        P                   ! \e        W R24      V4       K£  	  \g        VR	4      '       d8   \        P$                  ! VPh                  V P:                  Ph                  4       R
# R
# )ç      à?r‡   ræ   rˆ   ç       @Údefault©r¿  Ú	_inv_freqÚ_original_inv_freqrÖ  Nrl   rj  )5r]   Ú_init_weightsÚ
isinstancerY  ÚinitÚones_rb   râ   Úzeros_rð   r�   r,   r_   r¾   rô   rõ   Úcopy_rø   Ú	constant_rû   rì   rú   rù   r  r"  r%  rf  rk  rq  r�   r�   r~   rŽ   r—   r˜   rX   r�   r™   rš   r�   rœ   ÚGemma3nTextModelÚper_layer_projection_scaleÚper_layer_input_scalerø  ÚGemma3nRotaryEmbeddingr¼  Úcompute_default_rope_parametersÚ	rope_typer   rÂ  r¾  rÖ  )rc   r¤  rø   rü   rž   rŸ   r    r¡   r�   r¿  Úrope_init_fnÚcurr_inv_freqrÓ   rd   s   &&           €r0   r  Ú$Gemma3nPreTrainedModel._init_weightsY  s  ø€ ä‰Ñ˜fÔ%Ü�fÔ=×>Ò>Ü�JŠJ�v—}‘}Ö%Ü˜Ô 5×6Ò6Ü�KŠK˜×,Ñ,Ô-Ø—o‘o tÕ+ˆGØ¤§¡×!4Ñ!4×!=Ñ!=¼e¿lºlÈ3Ó>OÓ!PÕPˆLÜ�JŠJ�v—~‘~ wÕ'=Ô>Ü�NŠN˜6Ÿ>™>¨6×+KÑ+KÔLÜ�JŠJ�v×5Ñ5°v×7\Ñ7\Ó7^Ö_Ü˜Ô >×?Ò?Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÖIÜ˜Ô 0×1Ò1Ü�KŠK˜×3Ñ3Ô4Ü�NŠN˜6×4Ñ4°d·k±k×6MÑ6MÈtÕ6SÖTÜ˜Ô E×FÒFØ+.°˜=Ø#Ÿ_™_°Õ1ˆNÜ&*§h¢h¬u°]Ó/CÄeÈMÓFZÕ/ZÓ&[Ô^aØ Õ" Aó_õ 'Ð#ð +¬U¯YªY´u·|²|ÀNÓ7SÐWnÐVnÕ7nÓ-oÕoˆNÜ�JŠJ�v×,Ñ,¨n×.BÑ.BÓ.D×.NÑ.NÈqÓ.Q×.[Ñ.[Ð\]Ó.^Ö_Ü˜Ô 0×1Ò1Ü�NŠN˜6×<Ñ<¸d×>NÑ>NÐPTÕ>TÔUÜ�NŠN˜6×7Ñ7¸¼T¿YºYÀs»^Õ9KÕLÜ˜Ô 6×7Ò7Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ñ 1ô �6Ð.×/Ò/Ü�NŠN˜6×3Ñ3°T·[±[×5RÑ5RÖSñ 0r3   c                ó.   € V P                   P                  # rv   ©Ú
base_modelÚembed_tokens_per_layer©rc   s   &r0   Úget_per_layer_input_embeddingsÚ5Gemma3nPreTrainedModel.get_per_layer_input_embeddings�  s   € Ø�‰×5Ñ5Ð5r3   c                ó&   € WP                   n        R # rv   r"  ©rc   r£  s   &&r0   Úset_per_layer_input_embeddingsÚ5Gemma3nPreTrainedModel.set_per_layer_input_embeddings„  s   € Ø16�‰Ö.r3   c                ób   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[RS[P                  /# )r*   Únew_num_tokensNÚpad_to_multiple_ofÚmean_resizingrt   )rW   rY   r_   Ú	Embedding)r.   r/   s   "€r0   r1   Ú#Gemma3nPreTrainedModel.__annotate__‡  s?   ø€ ÷ ñ á˜d�
ðñ   $�Jðñ ð	ñ
 
�‰ñr3   c                óP   <€ \         SV `  VVVR 7      pV P                  WV4       V# ))r-  r.  r/  )r]   Úresize_token_embeddingsÚ_resize_per_layer_embeddings)rc   r-  r.  r/  Úinputs_embedsrd   s   &&&& €r0   r3  Ú.Gemma3nPreTrainedModel.resize_token_embeddings‡  s:   ø€ ô ™Ñ7Ø)Ø1Ø'ð 8ó 
ˆð
 	×)Ñ)¨.ÈmÔ\ØÐr3   c                óH   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[/# )r*   r-  Nr.  r/  )rW   rY   )r.   r/   s   "€r0   r1   r1  •  s6   ø€ ÷ Jñ Já˜d�
ðJñ   $�JðJñ ñ	Jr3   c                ó²  € V P                   V P                  P                  4       n        V P                  P                  4       P                  '       d„   V P                  4       pV P                  WAW#4      p\        VR 4      '       d   VP                  p\        WV4       VP                  VP                  P                  4       V P                  V4       R# R# )Ú_hf_hookN)Ú
vocab_sizer�   Úget_text_configÚvocab_size_per_layer_inputrì  r&  Ú_get_resized_embeddingsr¾  r9  r&   Úrequires_grad_rb   r\   r*  )rc   r-  r.  r/  r$  Únew_embeddings_per_layerÚhooks   &&&&   r0   r4  Ú3Gemma3nPreTrainedModel._resize_per_layer_embeddings•  s²   € ð DHÇ?Á?ˆ�‰×#Ñ#Ó%Ô@Ø�;‰;×&Ñ&Ó(×D×DÐDØ%)×%HÑ%HÓ%JÐ"Ø'+×'CÑ'CØ&Ð8Jó(Ð$ô Ð-¨z×:Ò:Ø-×6Ñ6�Ü"Ð#;ÔBØ$×3Ñ3Ð4J×4QÑ4Q×4_Ñ4_Ô`Ø×/Ñ/Ð0HÖIñ Er3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r€   ©r#   )r.   r/   s   "€r0   r1   r1  F  s   ø‡ ‚ áÑò r3   r4   )ÚimageÚtextÚaudio)NNT)r5   r6   r7   r8   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrâ  r¸  Ú_can_record_outputsÚinput_modalitiesr,   Úno_gradr  r&  r*  r3  r4  r:   r;   r<   r{   r|   s   @@r0   r  r  F  s¤   ù‡ € ð  ÐØ&*Ð#Ø2Ð3ÐØ#4Ð6HÐ"IÐØÐØ€NØÐà!ÐØ"&ÐàÐ0ØÐ*ðÐð 2Ðà
‡]‚]ƒ_ô%Tó ð%TòN6ò7÷õ ÷Jò J÷_ … r3   r  c                   ó|   a a€ ] tR tRt oRtRtRtV3R lV 3R llt]]	V3R lR l4       4       t
V3R	 ltR
tVtV ;t# )ÚGemma3nAudioEncoderi¨  zp
An audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.
Ú	audio_melrF  c                ó    <€ V ^8„  d   QhRS[ /# r€   r‚   )r.   r/   s   "€r0   r1   Ú Gemma3nAudioEncoder.__annotate__²  s   ø€ ÷ ñ Ñ1ñ r3   c                ó
  <€ \         SV `  V4       Wn        \        V4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n
        V P                  4        R # u upi rv   )r]   r^   r�   r¬  Úsubsample_conv_projectionr_   Ú
ModuleListr_  Úconf_num_hidden_layersr  Ú	conformerÚ	post_init)rc   r�   rÓ   rd   s   && €r0   r^   ÚGemma3nAudioEncoder.__init__²  si   ø€ Ü‰Ñ˜Ô ØŒä)LÈVÓ)TˆÔ&ÜŸšÜ9>¸v×?\Ñ?\Ô9]Ó^Ñ9]°AÔ'¨Ö/Ñ9]Ñ^ó
ˆŒð 	�‰Öùò _s   ÁB c          	      óz   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[S[,          /# )r*   rU  r+   r¯  rt   )r,   rh   r-   r   r   rO   r(   )r.   r/   s   "€r0   r1   rW  ¾  sC   ø€ ÷ 8
ñ 8
ÙŸ™ð8
Ù7<×7GÑ7Gð8
ÙSYÑZlÕSmð8
á	Ñ/Õ	/ñ8
r3   c                ó¾  € V P                  V4      pVP                  ^,          p^p\        \        V P                  P
                  4      4       F-  pW`P                  P
                  V,          ^ ,          ,          pK/  	  \        P                  ! WRP                  R7      V,          p\        P                  ! W‚P                  ^,          ^,
          R7      pVP                  ^8”  dE   VP                  ^8X  d4   VP                  ^ 4      P                  VP                  ^ ,          R4      pMsVP                  VP                  8X  dY   VP                  ^ ,          ^8X  dA   VP                  ^ ,          ^8w  d)   WXP                  ^ ,          8X  d   VP                  ^ 4      p\        P                  ! V^V4      p	V P                   F  p
V
! WI4      pK  	  V P                  P                  ^8”  dE   VRRRV P                  P                  13,          pV	RRRV P                  P                  13,          p	VP!                  V	P                  R4      R4      p\#        VV	R7      # )ad  Encodes a batch of MELs.

Args:
    audio_mel: a torch.Tensor of shape [batch, num_frames, num_channels,
      mel_bins].

Returns:
    audio_encodings: a torch.Tensor of shape
        `[batch_size, self.config.audio_soft_tokens_per_image,
        self.config.audio_config.hidden_size]`
    audio_mel_mask: a torch.BoolTensor of shape [batch, num_frames].
rÍ   )r�   r½   Nræ   )Úlast_hidden_stater+   rk   )rY  rÏ   r_  r`  r�   r�  r,   rš   r¨   ri  r%  rœ   rŸ  Úgatherr\  Úconf_reduction_factorÚmasked_fillr(   )rc   rU  r+   r¯  rŸ  Út_subÚtime_stride_productÚstride_pair_idxÚindicesÚcurrent_maskÚblocks   &&&,       r0   ry   ÚGemma3nAudioEncoder.forward¼  sê  € ð" ×8Ñ8¸ÓCˆð  ×%Ñ% aÕ(ˆàÐÜ$¤S¨¯©×)JÑ)JÓ%KÖLˆOØ§;¡;×#DÑ#DÀ_Õ#UÐVWÕ#XÕXÒñ  Mô —,’,˜u×-BÑ-BÔCÐFYÕYˆÜ—+’+˜g×+?Ñ+?ÀÕ+BÀQÕ+FÔGˆð ×Ñ Ô" w§|¡|°qÔ'8Ø×'Ñ'¨Ó*×1Ñ1°.×2FÑ2FÀqÕ2IÈ2ÓN‰Gà×Ñ 7§<¡<Ô/Ø×$Ñ$ QÕ'¨1Ô,Ø—‘˜aÕ  AÔ%ØŸ™ qÕ)Ô)ð ×'Ñ'¨Ó*ˆGä—|’| N°A°wÓ?ˆà—^”^ˆEÙ# OÓBŠOñ $ð �;‰;×,Ñ,¨qÔ0Ø-¨aÑ1U°D·K±K×4UÑ4UÐ1UÐ.UÕVˆOà'¨Ñ+O¨d¯k©k×.OÑ.OÐ+OÐ(OÕPˆLà)×5Ñ5°l×6LÑ6LÈRÓ6PÐRUÓVˆÜ-Ø-Ø'ô
ð 	
r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r€   r‚   )r.   r/   s   "€r0   r1   rW  ¨  s   ø‡ ‚ ñ
 Ñò r3   )r�   r\  rY  )r5   r6   r7   r8   r9   Úmain_input_namerQ  r^   r   r    ry   r:   r;   r<   r{   r|   s   @@r0   rT  rT  ¨  sD   ù‡ € ñð "€OØÐ÷ó ð  Ø÷8
ó ó  ð8
÷- … r3   rT  c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )r  iù  r¨   z5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# r€   r.  )r.   r/   s   "€r0   r1   Ú#Gemma3nRotaryEmbedding.__annotate__û  s   ø€ ÷ Uñ UÑ0ñ Ur3   c           
     óÞ  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n        / V n	        V P                   Fû  pV P
                  P                  V,          pVf   K&  VR,          V P                  V&   V P                  pV P                  V,          R8w  d   \        V P                  V,          ,          pV! V P
                  W#R7      w  rg\        W R2\        P                  ! VRR7      4       \        W R2\        P                  ! VP!                  4       RR7      4       \        W R2V4       Ký  	  R # )	Nr  r  r  r  Fr‰   r  Ú_attention_scaling)r]   r^   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr�   ÚsortedÚsetr¼  r  Úrope_parametersr  r   Úsetattrr_   r›   rö   )	rc   r�   r¨   r¿  Úrope_paramsr  r  Úcurr_attention_scalingrd   s	   &&&     €r0   r^   ÚGemma3nRotaryEmbedding.__init__ú  s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà)4°[Õ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÕ)¨YÔ6Ü2°4·>±>À*Õ3MÕN�Ù4@ÀÇÁÈfÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó +r3   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r*   r�   r¿  Nrt   )r$   ÚstrrO   r,   rh   rX   )r.   r/   s   "€r0   r1   rq    s7   ø€ ÷ 5ñ 5Ù!ð5Ù<?À$½Jð5á	‰u�|‰|™UÐ"Õ	#ñ5r3   c           	     ó>  € V P                   V,          R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetar�   Nr‡   rÎ   )ry  rÂ  r�   rÃ  r,   rš   rX   rª   )r�   r¨   r¿  r¯  ÚbaserT   Úattention_factorÚinv_freqs   &&&,    r0   r  Ú6Gemma3nRotaryEmbedding.compute_default_rope_parameters  sƒ   € ð& ×%Ñ% jÕ1°,Õ?ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r3   c                óZ  € \        W R 24      p\        W R24      pVR
,          P                  VP                  ^ ,          R^4      P                  \        P
                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	\        P                  ! W™3RR7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP                  VP                  R	7      XP                  VP                  R	7      3#   + '       g   i     LH; i)r  rs  NrV  ÚmpsÚcpuF)Údevice_typeÚenabledr©   rÎ   )Nr½   Nrk   )r½   Nr½   )rÂ  rŸ  rÏ   rª   r,   rX   r¨   r  r¯   r  r   r®  r¬   r®   r­   r¥   )rc   r  rõ  r¿  r„  Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr‰  ÚfreqsÚembr®   r­   s   &&&&         r0   ry   ÚGemma3nRotaryEmbedding.forward+  sJ  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã:A FÆF*	)r�   r¼  ru  rv  r  rv   rà  )r5   r6   r7   r8   r   r^   Ústaticmethodr  r,   rR  r   ry   r;   r<   r{   r|   s   @@r0   r  r  ù  su   ù‡ € Ù�X vÔ.÷Uô Uó /ðUð* Ù�X vÔ.÷5ñ 5ó /ó ð5ð2 ‡]‚]ƒ_Øñ<ó ó ÷<ð <r3   r  zBThe base Gemma 3n language model without a language modeling head.c                   ó¸   a a€ ] tR tRt oRtV3R lV 3R llt]]! RR7      ]RV3R lR ll4       4       4       t	V3R lR	 lt
RV3R
 lR lltV3R ltRtVtV ;t# )r  iA  c                ó    <€ V ^8„  d   QhRS[ /# r€   r.  )r.   r/   s   "€r0   r1   ÚGemma3nTextModel.__annotate__F  s   ø€ ÷ 7ñ 7Ñ0ñ 7r3   c                óØ  <€ \         SV `  V4       VP                  V n        VP                  V n        \        VP                  VP                  V P                  V P                  P                  R ,          R7      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                   R7      V n        \%        V4      V n        RV n        VP                  V n        VP*                  V n        \        VP,                  VP                  VP*                  ,          V P                  VP*                  R ,          R7      V n        \        P0                  ! V P                  VP                  VP*                  ,          RR7      V n        \        VP*                  VP                   R7      V n        \        P                  ! \        ^V P                  P6                  4       Uu. uF0  p\        P0                  ! V P                  V P                  RR7      NK2  	  up4      V n        \        P                  ! \        ^V P                  P6                  4       Uu. uF0  p\        P0                  ! V P                  V P                  RR7      NK2  	  up4      V n        \        P<                  ! \>        P@                  ! V P                  R
,          4      RR7      V n!        \        P<                  ! \>        PD                  ! \>        P@                  ! R4      4      RR7      V n#        . V n$        \K        V P                  4       FT  w  rEVPL                  PN                  '       g   K#  V PH                  PQ                  R Uu. uF  pRV RV 2NK  	  up4       KV  	  V PS                  4        R	# u upi u upi u upi u upi )r  )r"  rú  Fr…   r‰   r  zlayers.z.self_attn.Nrl   )rò   ró   rÎ  rÏ  )*r]   r^   Úpad_token_idr!  r:  r  r�   r�   Úembed_tokensr_   rZ  r_  rÆ  râ  ÚlayersrR   rü  r“  r  Ú
rotary_embÚgradient_checkpointingrì  r<  r$  r•   Úper_layer_model_projectionÚper_layer_projection_normrl  Úaltup_projectionsÚaltup_unembed_projectionsr›   r,   rõ   r  rj  r  Ú"_keys_to_ignore_on_load_unexpectedÚ	enumerateræ  rÈ  Úextendr]  )rc   r�   r?  rÓ   r¹  ÚlayerÚnamerd   s   &&     €r0   r^   ÚGemma3nTextModel.__init__F  sÿ  ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒô ;Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÕQmô
ˆÔô —m’mÜINÈv×OgÑOgÔIhÓiÑIh¸IÔ$ VÖ7ÑIhÑió
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü0°Ó8ˆŒØ&+ˆÔ#à!×-Ñ-ˆÔØ+1×+MÑ+MˆÔ(ä&DØ×-Ñ-Ø×$Ñ$ v×'IÑ'IÕIØ×ÑØ×:Ñ:¸CÕ?ô	'
ˆÔ#ô +-¯)ª)Ø×ÑØ×$Ñ$ v×'IÑ'IÕIØô+
ˆÔ'ô *8¸×8ZÑ8ZÐ`f×`sÑ`sÔ)tˆÔ&ä!#§¢ÜPUÐVWÐY]×YdÑYd×YuÑYuÔPvÓwÑPvÈ1ŒR�YŠY�t×'Ñ'¨×)9Ñ)9À×FÑPvÑwó"
ˆÔô *,¯ªÜPUÐVWÐY]×YdÑYd×YuÑYuÔPvÓwÑPvÈ1ŒR�YŠY�t×'Ñ'¨×)9Ñ)9À×FÑPvÑwó*
ˆÔ&ô +-¯)ª)´E·L²LÀ×AQÑAQÐSWÕAWÓ4XÐejÔ*kˆÔ'Ü%'§Y¢Y¬u¯{ª{¼5¿<º<ÈÓ;LÓ/MÐZ_Ô%`ˆÔ"ð 35ˆÔ/Ü! $§+¡+Ö.‰HˆAØ�‰×1×1Ò1Ø×7Ñ7×>Ñ>Ù@hÓiÑ@h¸�w˜q˜c ¨T¨FÓ3Ñ@hÑiöñ /ð 	�‰Öùò[ jùò4 xùò xùò js   Â'OÈ6OÊ	6O"Î*O'
F)Útie_last_hidden_statesc                ó&  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          R	S[S[,          R
S[/	# )r*   r(  NÚper_layer_inputsr§  rõ  rL   r5  Ú	use_cacher¯  rt   )	r,   rö  rh   r	   rE   rY   r   r   r   )r.   r/   s   "€r0   r1   r”  ‚  s±   ø€ ÷ m
ñ m
á×#Ñ# dÕ*ðm
ñ  Ÿ,™,¨Õ-ðm
ñ Ÿ™ tÕ+ð	m
ñ
 ×&Ñ&¨Õ-ðm
ñ  �ðm
ñ ×(Ñ(¨4Õ/ðm
ñ ˜$•;ðm
ñ Ñ+Õ,ðm
ñ 
!ñm
r3   c                ó¢	  € VRJ VRJ,          '       d   \        R4      hVe#   V P                  V4      pV P                  V4      pV P                  Wb4      pV'       d   Vf   \	        V P
                  R7      pVfb   Ve   VP                  4       M^ p	\        P                  ! VP                  ^,          VP                  R7      V	,           pVP                  ^ 4      p\        T;p
\        4      '       g/   RV P
                  RVRVRVR	V/pR
\        R/ VB R\        R/ VB /p
Tp\        P                   ! V^,          RRR7      R,          p\        P"                  ! R4      pV.p\%        ^V P
                  P&                  4       FÏ  pV P(                  V^,
          ,          ! V4      pVP+                  VP,                  VP                  R7      p\        P                   ! V^,          RRR7      p\        P.                  ! \        P0                  ! VVP+                  VP                  4      4      4      pVV,          V,          pVP3                  V4       KÑ  	  \        P4                  ! V^ R7      p/ p\7        V P
                  P8                  4       F  pV P;                  VVV4      VV&   K  	  \=        4       p\?        V P@                  RV P
                  PB                   4       Fk  w  ppW P
                  P8                  V,          ,          pVRRVR3,          pV! VVV P
                  P8                  V,          ,          V3RVRVR	VRV/VB pKm  	  \        P                   ! V^ ,          ^,          RRR7      R,          pV^ ,          .p\%        ^V P
                  P&                  4       FÖ  pV PD                  V^,
          ,          ! VV,          4      pVP+                  VP,                  VP                  R7      p\        P                   ! V^,          RRR7      p\        P.                  ! \        P0                  ! VVP+                  VP                  4      4      4      pVV,          V,          pVP3                  V4       KØ  	  \        P4                  ! V4      p\        P                   ! V^ R7      pV PG                  V4      p\I        VVR7      # )z�
per_layer_inputs (torch.Tensor, *optional*, defaults to None):
    Pre-computed per-layer embeddings. If None, they are derived from input_ids if provided.
Nú:You must specify exactly one of input_ids or inputs_embeds©r�   rÍ   r�   r5  r§  rL   rõ  Úfull_attentionr½  Tre  r  gñhãˆµøä>rV  r©   r½   rÖ  )ra  rL   r4   rk   )%r3  r—  Úget_per_layer_inputsÚproject_per_layer_inputsr
   r�   Úget_seq_lengthr,   rš   rÏ   r¨   rœ   r  r×  r   r   rn   rõ   r_  rl  r�  rª   r¥   rø  Úmaximumr±  Ústackrx  r¼  r™  r   r   r˜  rÆ  rž  r“  r   )rc   r(  r§  r§  rõ  rL   r5  r¨  r¯  Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚhidden_states_0Útarget_magnitudeÚepsilon_tensorÚtemp_hidden_statesr¹  Ú
altup_projÚcurrent_hidden_stateÚnew_magnituderM   rÕ  r¿  rÖ  Údecoder_layerÚcausal_maskrô  Úaltup_unemb_projs   &&&&&&&&,                   r0   ry   ÚGemma3nTextModel.forward  sY  € ð$ ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMØ#×8Ñ8¸ÓCÐà×8Ñ8¸ÓYÐç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð#Ðð (ˆô !Ÿ:š: o°qÕ&8¸bÈ$ÔOÐSVÕVÐÜŸš dÓ+ˆà-Ð.ÐÜ�q˜$Ÿ+™+×6Ñ6Ö7ˆAà×/Ñ/°°AµÖ6°ÓGˆJØ#-§=¡=°×7LÑ7LÐUe×UlÑUl =Ó#mÐ Ü!ŸJšJÐ';¸QÕ'>ÀBÐPTÔUˆMÜ!ŸJšJ¤u§}¢}°]ÀN×DUÑDUÐVf×VmÑVmÓDnÓ'oÓpˆMØ#7Ð:JÕ#JÈ]Õ#ZÐ Ø×%Ñ%Ð&:Ö;ñ 8ô ŸšÐ$6¸AÔ>ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ó+ñ 7ô $›:Ðä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAˆ}Ø-¯k©k×.EÑ.EÀaÕ.HÕIˆKØ.¨q°!°Q¸¨zÕ:ˆOá)ØØ# D§K¡K×$;Ñ$;¸AÕ$>Õ?Øñ	ð "2ð		ð
  +ð	ð *ð	ð !0ð	ð ñ	ŠMñ	 !Xô  !Ÿ:š: m°AÕ&6¸!Õ&;ÀÈTÔRÐVYÕYÐØ+¨AÕ.Ð/ÐÜ�q˜$Ÿ+™+×6Ñ6Ö7ˆAà-1×-KÑ-KÈAÐPQÍEÖ-RÐS`ÐabÕScÓ-dÐØ#3×#6Ñ#6¸_×=RÑ=RÐ[k×[rÑ[rÐ#6Ó#sÐ Ü!ŸJšJÐ';¸QÕ'>ÀBÐPTÔUˆMÜ!ŸJšJ¤u§}¢}°]ÀN×DUÑDUÐVf×VmÑVmÓDnÓ'oÓpˆMØ#7Ð:JÕ#JÈ]Õ#ZÐ Ø×%Ñ%Ð&:Ö;ñ 8ô ŸšÐ$6Ó7ˆÜŸ
š
 =°aÔ8ˆØŸ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r*   r(  rt   ©r,   rö  rh   )r.   r/   s   "€r0   r1   r”  ñ  s%   ø€ ÷ 
ñ 
©e×.>Ñ.>ð 
Á5Ç<Á<ñ 
r3   c                óž   € V P                  V4      P                  ! . VP                  OV P                  P                  NV P
                  N5!  # rv   )r$  rÀ   rÏ   r�   rÆ  rì  )rc   r(  s   &&r0   r­  Ú%Gemma3nTextModel.get_per_layer_inputsñ  sN   € Ø×*Ñ*¨9Ó5×=Ò=ð 
Ø�_‰_ð
à�K‰K×)Ñ)ð
ð ×,Ñ,ó
ð 	
r3   c                óv   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  /# )r*   r5  r§  Nrt   rg   )r.   r/   s   "€r0   r1   r”  ø  s:   ø€ ÷ 
ñ 
á—|‘|ð
ñ  Ÿ,™,¨Õ-ð
ñ 
�‰ñ	
r3   c                ó:  € V P                  V4      pW0P                  P                  VP                  VP                  R 7      ,          pVP
                  ! . VP                  RR OV P                  P                  NV P                  N5!  pV P                  V4      pVf   V# VP                  VP                  8w  d#   VRRV P                  P                  1R3,          pW2,           V P                  P                  VP                  VP                  R 7      ,          # )rV  N.r½   rk   )r›  r  rª   r¥   r¨   rÀ   rÏ   r�   rÆ  rì  rœ  r  )rc   r5  r§  rð  s   &&& r0   r®  Ú)Gemma3nTextModel.project_per_layer_inputsø  s&  € ð
 .2×-LÑ-LÈ]Ó-[ÐØ× ?Ñ ?× BÑ BØ×%Ñ%Ð.B×.IÑ.Ið !Có !
õ 	
Ðð  4×;Ò;ð  
Ø× Ñ   "Ð%ð 
à�K‰K×)Ñ)ð 
ð ×,Ñ,ó 
Ðð
  $×=Ñ=Ð>RÓSÐàÒ#Ø'Ð'à×%Ñ%Ð)9×)?Ñ)?Ô?à/°Ð5T°t·{±{×7TÑ7TÐ5TÐVWÐ0WÕXÐà$Õ7¸4×;UÑ;U×;XÑ;XØ×%Ñ%Ð.B×.IÑ.Ið <Yó <
õ 
ð 	
r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r€   r.  )r.   r/   s   "€r0   r1   r”  A  s   ø‡ ‚ áÑò r3   )rŸ  r�  rž  r—  r$  rš  r�   rì  r˜  r“  r!  r  r›  rœ  r  r™  r:  )rE  )NNNNNNNrv   )r5   r6   r7   r8   rQ  r^   r   r    r   ry   r­  r®  r:   r;   r<   r{   r|   s   @@r0   r  r  A  sf   ù‡ € ð !Ð÷7ó 7ðr  Ù¨EÔ2Ø÷m
ñ m
ó ó 3ó  ðm
÷^
ð 
÷
ò 
÷o … r3   r  z?The base Gemma 3n language model with a language modeling head.c                   óœ   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV3R	 lV 3R
 llt]	]
RV3R lR ll4       4       tV3R ltRtVtV ;t# )ÚGemma3nForCausalLMi  úlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrM   rK   Úkeep_full_weightc                ó    <€ V ^8„  d   QhRS[ /# r€   r.  )r.   r/   s   "€r0   r1   ÚGemma3nForCausalLM.__annotate__  s   ø€ ÷ ñ Ñ0ñ r3   c                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# rC  )
r]   r^   r  r	  r:  r_   r•   r�   rË  r]  rÛ  s   &&€r0   r^   ÚGemma3nForCausalLM.__init__  sU   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör3   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r*   r(  Nr§  rõ  rL   r5  Úlabelsr¨  Úlogits_to_keepr¯  rt   )
r,   rö  rh   r	   rE   rY   rW   r   r   r   )r.   r/   s   "€r0   r1   rÏ  '  s»   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ × Ñ  4Õ'ð;
ñ ˜$•;ð;
ñ ™eŸl™lÕ*ð;
ñ Ñ+Õ,ð;
ñ 
 ñ;
r3   c	                óF  € V P                   ! R
RVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  eO   WÐP                  P                  ,          p\        P                  ! V4      pWÐP                  P                  ,          pRpVe   V P                  ! WÖV P                  3/ V	B p\        VVV
P                  V
P                  V
P                  R	7      # )a$  
Example:

```python
>>> from transformers import AutoTokenizer, Gemma3nForCausalLM

>>> model = Gemma3nForCausalLM.from_pretrained("google/gemma-2-9b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```r(  r§  rõ  rL   r5  r¨  Nr½   )rJ   rK   rL   rM   rN   r4   )r	  ra  r  rW   ÚslicerË  r�   Úfinal_logit_softcappingr,   r5  Úloss_functionr:  r   rL   rM   rN   )rc   r(  r§  rõ  rL   r5  rÓ  r¨  rÔ  r¯  ÚoutputsrM   Úslice_indicesrK   rJ   s   &&&&&&&&&,     r0   ry   ÚGemma3nForCausalLM.forward%  s0  € ð@ ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ�;‰;×.Ñ.Ò:ØŸk™k×AÑAÕAˆFÜ—Z’Z Ó'ˆFØŸk™k×AÑAÕAˆFàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r€   r.  )r.   r/   s   "€r0   r1   rÏ    s   ø‡ ‚ ñ Ñò r3   )rË  r	  r:  )NNNNNNNr   )r5   r6   r7   r8   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr^   r   r   ry   r:   r;   r<   r{   r|   s   @@r0   rÉ  rÉ    sm   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€J÷ó ð Ø÷;
ñ ;
ó ó ð;
÷' … r3   rÉ  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚGemma3nMultimodalEmbedderie  zQEmbeds token ids or soft tokens for multimodal content into language model space.c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# )r*   Úmultimodal_configÚtext_config)r"   r%   r$   )r.   r/   s   "€r0   r1   Ú&Gemma3nMultimodalEmbedder.__annotate__h  s)   ø€ ÷ tñ tá-Ñ0CÕCðtñ 'ñtr3   c                óx  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                  ! V P                  V P                  4      V n        \        V P                  V P
                  R 7      V n        \        V P                  V P
                  R 7      V n        \        P                  ! V P                  V P                  RR7      V n        \        V P                  V P
                  RR7      V n        R# )rú  Fr…   )rU   rV   N)r]   r^   r�   Úmultimodal_hidden_sizerü  rU   Úvocab_offsetr:  Útext_hidden_sizer_   r0  Ú	embeddingrR   Úhard_embedding_normÚsoft_embedding_normr•   Úembedding_projectionÚembedding_post_projection_norm)rc   rä  rå  rd   s   &&&€r0   r^   Ú"Gemma3nMultimodalEmbedder.__init__h  sé   ø€ ô
 	‰ÑÔà&7×&CÑ&CˆÔ#Ø$×1Ñ1ˆŒØ-×:Ñ:ˆÔØ+×6Ñ6ˆŒØ +× 7Ñ 7ˆÔäŸš d§o¡o°t×7RÑ7RÓSˆŒÜ#1°$×2MÑ2MÐSW×S[ÑS[Ô#\ˆÔ Ü#1°$×2MÑ2MÐSW×S[ÑS[Ô#\ˆÔ Ü$&§I¢I¨d×.IÑ.IÈ4×K`ÑK`ÐglÔ$mˆÔ!Ü.<¸T×=RÑ=RÐX\×X`ÑX`ÐmrÔ.sˆÖ+r3   c                ó„   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  /# )r*   r(  Nr5  rt   rÁ  )r.   r/   s   "€r0   r1   ræ  {  sF   ø€ ÷ Bñ Bá×#Ñ# dÕ*ðBñ —|‘| dÕ*ðBñ 
�‰ñ	Br3   c                ó  € VRJ VRJ,          '       d   \        R4      hVe   V P                  V4      pM2V P                  WP                  ,
          4      pV P	                  V4      pV P                  V4      pV P                  V4      # )a­  Embeds token ids or soft tokens for multimodal content into language model space.

Args:
    input_ids: A torch.LongTensor containing the token ids to embed. Values should be in the range
        `[vocab_offset, vocab_offset + vocab_size)`.
    inputs_embeds: A torch.Tensor containing the soft tokens to embed.

Returns:
    A torch.Tensor of embeddings with  shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
Nrª  )r3  rí  rë  ré  rì  rî  rï  )rc   r(  r5  Úemb_normÚhard_embÚemb_norm_projs   &&&   r0   ry   Ú!Gemma3nMultimodalEmbedder.forward{  sƒ   € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ$Ø×/Ñ/°Ó>‰Hà—~‘~ i×2CÑ2CÕ&CÓDˆHØ×/Ñ/°Ó9ˆHà×1Ñ1°(Ó;ˆØ×2Ñ2°=ÓAÐAr3   )
rë  rï  rî  rU   rì  rè  rí  rê  ré  r:  rà  r  r|   s   @@r0   râ  râ  e  s&   ù‡ € Ù[÷tó t÷&B÷ Bò Br3   râ  z�
    The base Gemma 3n model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c                   óì   a a€ ] tR tRt oRtV3R lV 3R llt]]! RR7      V3R lR l4       4       tRV3R	 lR
 llt	]RV3R lR ll4       t
R tR t]]! RR7      V3R lR l4       4       tRtVtV ;t# )ÚGemma3nModeli—  Fc                ó    <€ V ^8„  d   QhRS[ /# r€   rC  )r.   r/   s   "€r0   r1   ÚGemma3nModel.__annotate__¡  s   ø€ ÷ ñ ™}ñ r3   c                ó*  <€ \         SV `  V4       \        P                  ! VP                  R 7      V n        VP                  P                  V n        \        P                  ! VP                  R 7      pW n        VP                  P                  V n	        \        P                  ! VP                  4      V n        \        VP                  VP                  4      V n        \        VP                  VP                  4      V n        V P                  4        R# )r«  N)r]   r^   r!   Úfrom_configÚvision_configÚvision_towerrå  r:  Úlanguage_modelr<  Úaudio_configÚaudio_towerrâ  Úembed_visionÚembed_audior]  )rc   r�   rÿ  rd   s   && €r0   r^   ÚGemma3nModel.__init__¡  sÅ   ø€ Ü‰Ñ˜Ô Ü%×1Ò1¸×9MÑ9MÔNˆÔØ ×,Ñ,×7Ñ7ˆŒä"×.Ò.°f×6HÑ6HÔIˆØ,ÔØ*0×*<Ñ*<×*WÑ*WˆÔ'Ü$×0Ò0°×1DÑ1DÓEˆÔÜ5°f×6JÑ6JÈF×L^ÑL^Ó_ˆÔÜ4°V×5HÑ5HÈ&×J\ÑJ\Ó]ˆÔØ�‰Ör3   zOProjects the last hidden state from the vision model into language model space.r>   c                ó`   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[,          /# )r*   Úpixel_valuesr¯  rt   )r,   rE   r   r   rO   r   )r.   r/   s   "€r0   r1   rú  °  s;   ø€ ÷ ñ á×'Ñ'ðñ Ñ+Õ,ðñ 
Ñ+Õ	+ñ	r3   c           	     ó¤  € V P                   ! RR VRRRR/VB pVP                  pVP                  VP                  ^ ,          V P                  P
                  P                  V P                  P                  4      P                  ^ ^^4      pW@P                  P
                  P                  R,          ,          pV P                  VR7      Vn
        V# )r  Ú
do_poolingFÚreturn_dictTr  ©r5  r4   )rþ  ra  rÀ   rÏ   r�   rý  r�   Úvision_soft_tokens_per_imagerÑ   r  Úpooler_output)rc   r  r¯  Úvision_outputsra  s   &&,  r0   Úget_image_featuresÚGemma3nModel.get_image_features®  sÅ   € ð ×*Ò*Ñs¸ÐsÐQVÐsÐdhÐsÐlrÑsˆØ*×<Ñ<Ðð .×5Ñ5Ø×#Ñ# AÕ&Ø�K‰K×%Ñ%×1Ñ1Ø�K‰K×4Ñ4ó
÷ ‰'�!�Q˜Ó
ð	 	ð 	Ÿ[™[×6Ñ6×BÑBÀCÕGÕGÐØ'+×'8Ñ'8ÐGXÐ'8Ó'YˆÔ$àÐr3   c          	      óº   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          /# )r*   r(  Nr5  Úimage_featuresÚaudio_features)r,   rö  rE   )r.   r/   s   "€r0   r1   rú  Ä  s_   ø€ ÷ *6ñ *6á×#Ñ# dÕ*ð*6ñ ×(Ñ(¨4Õ/ð*6ñ ×)Ñ)¨DÕ0ð	*6ñ
 ×)Ñ)¨DÕ0ñ*6r3   c           
     ó¨  € VfÛ   W P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  pVP                  R4      pVV P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  P                  R4      pM0WP                  P                  8H  pWP                  P                  8H  pVP                  4       pVP                  R4      P                  VP                  4      pVed   \        WrP                  R,          ,          VP                  4       8H  RV RVP                  ^ ,          VP                  ^,          ,           24       VP                  4       pVP                  R4      P                  VP                  4      pVed   \        W‚P                  R,          ,          VP                  4       8H  RV RVP                  ^ ,          VP                  ^,          ,           24       WV3# )z×
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
equal to the length of multimodal features. If the lengths are different, an error is raised.
rV  z6Image features and image tokens do not match, tokens: z, features: z6Audio features and audio tokens do not match, tokens: r4   rk   )Úget_input_embeddingsr,   Úfullr�   Úimage_token_idÚlongr¨   ÚallÚaudio_token_idrg  rœ   rª   r   rÏ   Únumel)	rc   r(  r5  r  r  Úspecial_image_maskÚspecial_audio_maskÚn_image_tokensÚn_audio_tokenss	   &&&&&    r0   Úget_placeholder_maskÚ!Gemma3nModel.get_placeholder_maskÄ  s>  € ð ÒØ!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÔió3ñ "Ðð "4×!7Ñ!7¸Ó!;ÐàØ×,Ñ,Ô.Ü—J’J˜r 4§;¡;×#=Ñ#=ÄUÇZÁZÐXe×XlÑXlÔmóñ÷ ‰c�"‹gñ ð "+¯k©k×.HÑ.HÑ!HÐØ!*¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØÒ%Ü"Ø×!4Ñ!4°RÕ!8Õ8¸N×<PÑ<PÓ<RÑRØHÈÐHXÐXdÐes×eyÑeyÐz{Õe|ð  @N÷  @Tñ  @Tð  UVõ  @Wõ  fWð  eXð  Yôð
 ,×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØÒ%Ü"Ø×!4Ñ!4°RÕ!8Õ8¸N×<PÑ<PÓ<RÑRØHÈÐHXÐXdÐes×eyÑeyÐz{Õe|ð  @N÷  @Tñ  @Tð  UVõ  @Wõ  fWð  eXð  Yôð
 "Ð5Ð5r3   c                óÆ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[/# )r*   r(  Nr  Úinput_featuresr§  Úinput_features_maskrõ  rL   Útoken_type_idsr5  rÓ  r¨  Ú	lm_kwargsrt   )	r,   rö  rE   rh   r	   rY   r   r   rA   )r.   r/   s   "€r0   r1   rú  ñ  s	  ø€ ÷ F
ñ F
á×#Ñ# dÕ*ðF
ñ ×'Ñ'¨$Õ.ðF
ñ ×)Ñ)¨DÕ0ð	F
ñ
 Ÿ™ tÕ+ðF
ñ #Ÿ\™\¨DÕ0ðF
ñ ×&Ñ&¨Õ-ðF
ñ  �ðF
ñ ×(Ñ(¨4Õ/ðF
ñ ×(Ñ(¨4Õ/ðF
ñ × Ñ  4Õ'ðF
ñ ˜$•;ðF
ñ Ñ.Õ/ðF
ñ 
$ñF
r3   c                ó°	  € VRJ V	RJ,          '       d   \        R4      hVEes   V P                  4       ! V4      p	\        P                  ! V^ 8¬  WP                  8  4      p\        P
                  ! WÑ\        P                  ! V4      4      pV P                  P                  V4      p\        P                  ! WP                  P                  8¬  WP                  P                  8  4      pV P                  P                  V P                  P                  ,           ^,
          p\        P
                  ! VVV4      P                  V	P                  4      pV P                  VR7      pVP                  V	P                  V	P                  4      pVP!                  R4      p\        P
                  ! VVV	4      p	WP                  P                  8¬  pV P                  P                  V P                  P                  ,           ^,
          p\        P
                  ! VVV4      P                  V	P                  4      pV P                  VR7      pVP                  V	P                  V	P                  4      pVP!                  R4      p\        P
                  ! VVV	4      p	MRpVel   V P#                  VRR7      P$                  pVP                  V	P                  V	P                  4      pV P'                  WVR7      w  ppV	P)                  VV4      p	VEeW   VEeR   V P+                  W5( RR7      pVP$                  pVP,                  p\        P.                  ! V P                  ^,
          ..\        P0                  VP                  R7      pV P                  VR7      p \        P
                  ! VP!                  R4      V V4      pVP2                  w  p!p"p#V P4                  P6                  V",
          p$V P9                  V!V$V#4      p%\        P:                  ! VV%3^R7      pVP                  V	P                  V	P                  4      pV P'                  WVR	7      w  pp&V	P)                  V&V4      p	V P                  ! RR
RRVRVRVRVRV	RVRR/VB p'\=        V'P>                  V'       d   V'P@                  MRV'PB                  V'PD                  Ve   XMRVe   XR7      # RR7      # )aµ  
input_features_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Attention mask for `input_features` where non-zero values mark valid audio frames.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, Gemma3nForConditionalGeneration

>>> model = Gemma3nForConditionalGeneration.from_pretrained("google/gemma3n2-3b-mix-224")
>>> processor = AutoProcessor.from_pretrained("google/gemma3n2-3b-mix-224")

>>> prompt = "Where is the cat standing?"
>>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(**inputs,)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Where is the cat standing?\nsnow"
```
Nrª  )r(  T)r	  )r5  r  rV  r©   )r5  r  r(  r§  r§  rõ  rL   r5  r¨  r	  )ra  rL   rM   rN   rC   rD   rk   r4   )#r3  r  r,   r4  r<  r6  Ú
zeros_likerÿ  r­  r  ré  r  r:  rª   r¨   r¥   rœ   r  r  r  Úmasked_scatterÚget_audio_featuresr+   rõ   r  rÏ   r�   Úaudio_soft_tokens_per_imagerŸ  r¬   rA   ra  rL   rM   rN   )(rc   r(  r  r"  r§  r#  rõ  rL   r$  r5  rÓ  r¨  r%  Úper_layer_inputs_maskÚper_layer_inputs_tokensr§  Úvision_maskÚdummy_vision_token_idÚvision_input_idsÚvision_embedsÚexpanded_vision_maskÚ
audio_maskÚdummy_audio_token_idÚaudio_input_idsÚaudio_embedsÚexpanded_audio_maskr  r  rÓ   Úaudio_outputsr  Úaudio_padding_toksÚaudio_padding_embsÚaudio_batch_sizeÚaudio_seq_lenÚaudio_embed_dimÚextra_padding_tokensÚextra_padding_featuresr  rÙ  s(   &&&&&&&&&&&&,                           r0   ry   ÚGemma3nModel.forwardð  s²  € ð` ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÓ Ø ×5Ñ5Ô7¸	ÓBˆMô %*×$5Ò$5°iÀ1±nÀi×RqÑRqÑFqÓ$rÐ!Ü&+§k¢kÐ2GÔTY×TdÒTdÐenÓToÓ&pÐ#Ø#×2Ñ2×GÑGÐH_Ó`Ðô  ×+Ò+Ø×.Ñ.×;Ñ;Ñ;¸Y×IYÑIY×IfÑIfÑ=fóˆKð %)×$5Ñ$5×$BÑ$BÀT×EVÑEV×EaÑEaÕ$aÐdeÕ$eÐ!Ü$Ÿ{š{¨;¸	ÐCXÓY×\Ñ\Ð]j×]qÑ]qÓrÐØ ×-Ñ-Ð8HÐ-ÓIˆMØ)×,Ñ,¨]×-AÑ-AÀ=×CVÑCVÓWˆMØ#.×#8Ñ#8¸Ó#<Ð Ü!ŸKšKÐ(<¸mÈ]Ó[ˆMð #×&6Ñ&6×&CÑ&CÑCˆJØ#'×#3Ñ#3×#@Ñ#@À4×CSÑCS×C^ÑC^Õ#^ÐabÕ#bÐ Ü#Ÿkšk¨*°iÐAUÓV×YÑYÐZg×ZnÑZnÓoˆOØ×+Ñ+°oÐ+ÓFˆLØ'Ÿ?™?¨=×+?Ñ+?À×ATÑATÓUˆLØ",×"6Ñ"6°rÓ":ÐÜ!ŸKšKÐ(;¸\È=ÓY‰Mà#Ðð Ò#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ$(×$=Ñ$=ØÀ~ð %>ó %Ñ!Ð ð *×8Ñ8Ð9KÈ^Ó\ˆMð Ó%Ð*=Ó*IØ ×3Ñ3°NÐDXÐfjÐ3ÓkˆMØ*×8Ñ8ˆNØ&×5Ñ5ˆJô "'§¢°·±À!Õ0CÐ/DÐ.EÌUÏZÉZÐ`n×`uÑ`uÔ!vÐØ!%×!1Ñ!1Ð<NÐ!1Ó!OÐÜ"Ÿ[š[¨×)=Ñ)=¸bÓ)AÐCUÐWeÓfˆNà?M×?SÑ?SÑ<Ð˜m¨_Ø#'§;¡;×#JÑ#JÈ]Õ#ZÐ Ø%7×%>Ñ%>Ð?OÐQeÐgvÓ%wÐ"ä"ŸYšY¨Ð8NÐ'OÐUVÔWˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ$(×$=Ñ$=ØÀ~ð %>ó %Ñ!ˆAÐ!ð *×8Ñ8Ð9KÈ^Ó\ˆMà×%Ò%ñ 

Øð

à-ð

ð *ð

ð &ð	

ð
 ,ð

ð (ð

ð  ð

ð ð

ð ñ

ˆô *Ø%×7Ñ7ß7@˜G×3Ò3ÀdØ!×/Ñ/Ø×)Ñ)Ø2>Ò2J¡ÐPTØ2@Ò2L ô
ð 	
ð SWô
ð 	
r3   c                ó.   € V P                   P                  # rv   ©rÿ  r$  r%  s   &r0   r&  Ú+Gemma3nModel.get_per_layer_input_embeddingsy  s   € Ø×"Ñ"×9Ñ9Ð9r3   c                ó&   € WP                   n        R # rv   rA  r)  s   &&r0   r*  Ú+Gemma3nModel.set_per_layer_input_embeddings|  s   € Ø5:×ÑÖ2r3   zPProjects the last hidden state from the audio encoder into language model space.c          	      óz   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[S[,          /# )r*   r"  r#  r¯  rt   )r,   rh   r   r   rO   r(   )r.   r/   s   "€r0   r1   rú  �  sG   ø€ ÷ ñ áŸ™ðñ #Ÿ\™\ðñ Ñ+Õ,ð	ñ
 
Ñ/Õ	/ñr3   c                ót   € V P                   ! W3RR/VB pV P                  VP                  R7      pWTn        V# )a  
input_features (`torch.FloatTensor` of shape `(num_images, seq_length, num_features)`):
    The tensors corresponding to the input audio.
input_features_mask (`torch.FloatTensor` of shape `(num_images, seq_length)`):
    The attention mask for the input audio.
r	  Tr
  )r  r  ra  r  )rc   r"  r#  r¯  r7  r5  s   &&&,  r0   r)  ÚGemma3nModel.get_audio_features  sN   € ð 9=×8HÒ8HØñ9
Ø=Að9
ØEKñ9
ˆð ×'Ñ'°m×6UÑ6UÐ'ÓVˆØ&2Ô#àÐr3   )r  r  r  rÿ  rþ  r:  r<  )NNNN)NNNNNNNNNNN)r5   r6   r7   r8   Úaccepts_loss_kwargsr^   r   r   r  r  ry   r&  r*  r)  r;   r<   r{   r|   s   @@r0   rø  rø  —  s”   ù‡ € ð  Ð÷ó ð ÙÐ!rÔs÷ó tó ð÷(*6ò *6ðX ÷F
ñ F
ó ðF
òP:ò;ð ÙÐ!sÔt÷ó uó ÷ð r3   rø  z†
    The base Gemma 3n model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c                   óœ   a a€ ] tR tRt oRR/tRtV3R lV 3R llt]V3R lR l4       t]	]RV3R	 lR
 ll4       4       t
R tR tRtVtV ;t# )ÚGemma3nForConditionalGenerationi–  rÊ  z(model.language_model.embed_tokens.weightFc                ó    <€ V ^8„  d   QhRS[ /# r€   rC  )r.   r/   s   "€r0   r1   Ú,Gemma3nForConditionalGeneration.__annotate__   s   ø€ ÷ ñ ™}ñ r3   c                óò   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  P                  VP                  P                  R R7      V n	        V P                  4        R# rC  )r]   r^   rø  r	  r_   r•   rå  r�   r:  rË  r]  rÛ  s   &&€r0   r^   Ú(Gemma3nForConditionalGeneration.__init__   sS   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ü—y’y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰Ör3   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r*   r  r¯  )r,   rE   r   r   )r.   r/   s   "€r0   r1   rL  §  s+   ø€ ÷ Eñ E©u×/@Ñ/@ð EÉFÑSeÕLfñ Er3   c                ó<   € V P                   P                  ! V3/ VB # rv   )r	  r  )rc   r  r¯  s   &&,r0   r  Ú2Gemma3nForConditionalGeneration.get_image_features¦  s   € à�z‰z×,Ò,¨\ÑD¸VÑDÐDr3   c                óð  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  ,          RS[S[,          RS[	/# )r*   r(  Nr  r"  r§  r#  rõ  rL   r$  r5  rÓ  r¨  rÔ  r%  rt   )
r,   rö  rE   rh   r	   rY   rW   r   r   rH   )r.   r/   s   "€r0   r1   rL  ¬  s  ø€ ÷ e
ñ e
á×#Ñ# dÕ*ðe
ñ ×'Ñ'¨$Õ.ðe
ñ ×)Ñ)¨DÕ0ð	e
ñ
 Ÿ™ tÕ+ðe
ñ #Ÿ\™\¨DÕ0ðe
ñ ×&Ñ&¨Õ-ðe
ñ  �ðe
ñ ×(Ñ(¨4Õ/ðe
ñ ×(Ñ(¨4Õ/ðe
ñ × Ñ  4Õ'ðe
ñ ˜$•;ðe
ñ ™eŸl™lÕ*ðe
ñ Ñ.Õ/ðe
ñ 
'ñe
r3   c                óŽ  € V P                   ! RRVRVRVRVRVRVRVRVR	V	R
V
RVRR/VB pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  4       P                  ;pe)   VV,          p\        P                  ! V4      pVV,          pRpV
e7   V P                  ! VW P                  P                  4       P                  3/ VB p\        VVVP                  VP                  VP                   VP"                  VP$                  R7      # )a<  
input_features_mask (torch.Tensor, *optional*, defaults to None):
    The attention mask for the input audio.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
    ignored (masked), the loss is only computed for the tokens with labels in
    `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

>>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
>>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

>>> messages = [
...     {
...         "role": "system",
...         "content": [
...             {"type": "text", "text": "You are a helpful assistant."}
...         ]
...     },
...     {
...         "role": "user", "content": [
...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
...             {"type": "text", "text": "Where is the cat standing?"},
...         ]
...     },
... ]

>>> inputs = processor.apply_chat_template(
...     messages,
...     tokenizer=True,
...     return_dict=True,
...     return_tensors="pt",
...     add_generation_prompt=True
... )
>>> # Generate
>>> generate_ids = model.generate(**inputs)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
```
r(  r  r"  r§  r#  rõ  rL   r$  r5  rÓ  r¨  r	  TNr½   )rJ   rK   rL   rM   rN   rC   rD   r4   )r	  ra  r  rW   rÖ  rË  r�   r;  r×  r,   r5  rØ  r:  rH   rL   rM   rN   rC   rD   )rc   r(  r  r"  r§  r#  rõ  rL   r$  r5  rÓ  r¨  rÔ  r%  rÙ  rM   rÚ  rK   r×  rJ   s   &&&&&&&&&&&&&,      r0   ry   Ú'Gemma3nForConditionalGeneration.forwardª  s�  € ðD —*’*ñ 
Øð
à%ð
ð *ð
ð *ð	
ð
 !4ð
ð &ð
ð ,ð
ð *ð
ð (ð
ð ð
ð  ð
ð ð
ð ñ
ˆð   ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ'+§{¡{×'BÑ'BÓ'D×'\Ñ'\Ð\Ð#ÒiØÐ5Õ5ˆFÜ—Z’Z Ó'ˆFØÐ5Õ5ˆFàˆØÒØ×%Ò% f¨f·k±k×6QÑ6QÓ6S×6^Ñ6^ÑlÐbkÑlˆDä,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;Ø '× ;Ñ ;ô
ð 	
r3   c                ó6   € V P                   P                  4       # rv   )r	  r&  r%  s   &r0   r&  Ú>Gemma3nForConditionalGeneration.get_per_layer_input_embeddings	  s   € Ø�z‰z×8Ñ8Ó:Ð:r3   c                ó<   € V P                   P                  V4       R # rv   )r	  r*  r)  s   &&r0   r*  Ú>Gemma3nForConditionalGeneration.set_per_layer_input_embeddings	  s   € Ø�
‰
×1Ñ1°%Ö8r3   )rË  r	  )NNNNNNNNNNNr   )r5   r6   r7   r8   rÝ  rH  r^   r   r  r   ry   r&  r*  r;   r<   r{   r|   s   @@r0   rJ  rJ  –  sm   ù‡ € ð +Ð,VÐWÐØÐ÷ó ð ÷Eó ðEð Ø÷e
ñ e
ó ó ðe
òN;÷9ò 9r3   rJ  )ræ   NNrÅ  )rT  rÉ  rJ  rø  r  r  )br—   Úcollectionsr   Úcollections.abcr   r   Údataclassesr   r,   Útorch.nnr_   Útorch.nn.functionalr¾   r¤  Ú r   r  rQ  r   Úcache_utilsr	   r
   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr    Úautor!   Úconfiguration_gemma3nr"   r#   r$   r%   Úaccelerate.hooksr&   r(   rA   rH   rª  rR   r~   râ   rY  r�  r¬  rÐ  rå  rö  r  r0  r  r,  r<  rf  r›  r¢  r²  r¶  r¸  râ  r  rT  r  r  rÉ  râ  rø  rJ  Ú__all__r4   r3   r0   Ú<module>ro     s'  ðó* Ý  ß .Ý !ã Ý ß Ð å &Ý !ß .Ý )ß RÝ 9ß kÑ kß Kß FÝ &÷÷ õ 1ß GÝ 5Ý ß lÓ lñ ×ÒÝ3ð Ø
ô3Ð%?ó 3ó ó ð3ñ ðôð
 ô9Ð!8ó 9ó óð9ñ( ðôð
 ô9 Kó 9ó óð9ô:4�R—Y‘Yô 4ô0c)¨B¯I©Iô c)ôL]˜BŸI™Iô ]ô@j, b§i¡iô j,ôZB7 §	¡	ô B7ôJF¨"¯)©)ô FôRO R§Y¡Yô Oô8D r§y¡yô Dô2( r§y¡yô (ôV §¡ô ô6S R§\¡\ô Sô;˜RŸY™Yô ;ô$#5�R—Y‘Yô #5ôL`'�r—y‘yô `'òF(õ	U÷%÷D.ô,l)˜2Ÿ9™9ô l)ô^I%Ð8ô I%ðX ô^J˜_ó ^Jó ð^JôBN
Ð0ô N
ôbE<˜RŸY™Yô E<ñP ÐaÔbôO
Ð-ó O
ó cðO
ñd Ð^Ô_ôM
Ð/°ó M
ó `ðM
ô`/B §	¡	ô /Bñd ðôôvÐ)ó vóðvñr ðôô{9Ð&<¸oó {9óð{9ò|‚r3   