+
    œ°³jGå ã                   óè  € ^ RI t ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RI	t	^ RI	H
t
 ^ RIHt ^RIHt ^R	IHt ^R
IHtHt ^RIHt ^RIHt ^RIHt ^RIHtHtHtH t H!t!H"t"H#t#H$t$ ^RI%H&t& ^RI'H(t( ^RI)H*t*H+t+ ^RI,H-t-H.t. ^RI/H0t0H1t1 ^RI2H3t3 ^RI4H5t5H6t6H7t7H8t8H9t9H:t: ^RI;H<t< ^RI=H>t>H?t? ^RI@HAtAHBtB ^RICHDtD ^RIEHFtFHGtGHHtHHItI ]9! 4       '       d   ^ RIJHKtK ]7! RR7      ] ! R R]*4      4       4       tL]7! R R7      ] ! R! R"]54      4       4       tM] ! R# R$]*4      4       tN]7] ! R% R&]+4      4       4       tO ! R' R(]
P                   4      tQ ! R) R*]
P                   4      tR ! R+ R,]
P                   4      tS ! R- R.]
P                   4      tT ! R/ R0]
P                   4      tU ! R1 R2]
P                   4      tV ! R3 R4]
P                   4      tW ! R5 R6]
P°                  4      tY ! R7 R8]
P                   4      tZ ! R9 R:]
P                   4      t[ ! R; R<]
P                   4      t\ ! R= R>]
P                   4      t] ! R? R@]
P                   4      t^ ! RA RB]
P                   4      t_RC t`RzRD RE lltaRF RG ltbR{RH RI lltcR|RJ RK lltd ! RL RM]
P                   4      te ! RN RO](4      tf ! RP RQ]
P                   4      tg ! RR RS]
P                   4      th ! RT RU]
P                   4      ti ! RV RW]
P                   4      tj] ! RX RY]
P                   4      4       tk ! RZ R[]
P                   4      tl ! R\ R]](4      tm ! R^ R_]
PÜ                  4      to]7 ! R` Ra]14      4       tp]7! RbR7       ! Rc Rd]p4      4       tq]7! ReR7       ! Rf Rg]p]4      4       trRh Ri lts ! Rj Rk]p4      tt ! Rl Rm]p4      tu ! Rn Ro]
P                   4      tvRp Rq ltwRr Rs ltx]7! RtR7       ! Ru Rv]p4      4       ty]7! RwR7       ! Rx Ry]p]4      4       tz. R}Ot{R# )~é    N)ÚUserDict)ÚCallable)Ú	dataclass)Úcached_property)Únn)Ú
functional)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_experts_implementation)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_bidirectional_maskÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_accelerate_availableÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)Ú	AutoModel)ÚGemma4AudioConfigÚGemma4ConfigÚGemma4TextConfigÚGemma4VisionConfig)Úadd_hook_to_modulezK
    Base class for Gemma4 outputs, with hidden states and attentions.
    ©Úcustom_introc                   ó<   a € ] tR t^Ht o RtRtRtRtV 3R ltRt	V t
R# )ÚGemma4ModelOutputWithPastaw  
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
audio_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
shared_kv_states (`dict`, *optional*):
    Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
    Used to pass shared KV states between layers during KV sharing.
Nc                óä   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[S[S[ P
                  S[ P
                  3,          3,          R,          ;R&   # )é   NÚimage_hidden_statesÚaudio_hidden_statesÚshared_kv_states)ÚtorchÚFloatTensorÚdictÚstrÚtupleÚTensor)ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gemma4/modeling_gemma4.pyÚ__annotate__Ú&Gemma4ModelOutputWithPast.__annotate__H   sc   ø‡ ‚ ñ0 ×*Ñ*¨TÕ1Ñ8ñ1 ñ4 ×*Ñ*¨TÕ1Ñ8ñ5 ñ8 ™3¡¡e§l¡l±E·L±LÐ&@Õ AÐAÕBÀTÕIÑPò9 ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r8   r9   r:   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©rB   s   @rC   r5   r5   H   s%   ø‡ € ñð" 59Ðà48ÐàLPÐ÷9 ƒ rF   r5   zR
    Base class for Gemma4 causal language model (or autoregressive) outputs.
    c                   óP   a € ] tR t^gt o RtRtRtRtRtRt	Rt
RtRtV 3R ltRtV tR# )ÚGemma4CausalLMOutputWithPastaÐ  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
audio_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
shared_kv_states (`dict`, *optional*):
    Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
    Used to pass shared KV states between layers during KV sharing.
Nc                óÖ  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[S[S[ P                  S[ P                  3,          3,          R,          ;R	&   # )
r7   NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr8   r9   r:   )r;   r<   r   r?   r=   r>   r@   )rA   rB   s   "€rC   rD   Ú)Gemma4CausalLMOutputWithPast.__annotate__g   sæ   ø‡ ‚ ñ8 ×
Ñ
˜dÕ
"Ñ)ñ9 ñ: ×Ñ Õ$Ñ+ñ; ñ< ˜T•\Ñ(ñ= ñ> ™×*Ñ*Õ+¨dÕ2Ñ9ñ? ñ@ ‘e×'Ñ'Õ(¨4Õ/Ñ6ñA ñB ×*Ñ*¨TÕ1Ñ8ñC ñF ×*Ñ*¨TÕ1Ñ8ñG ñJ ™3¡¡e§l¡l±E·L±LÐ&@Õ AÐAÕBÀTÕIÑPòK rF   rG   )rH   rI   rJ   rK   rL   rT   rU   rV   rW   rX   r8   r9   r:   rM   rN   rO   rP   s   @rC   rR   rR   g   s?   ø‡ € ñð* &*€DØ'+€FØ$(€OØ59€MØ26€JØ48Ðà48ÐàLPÐ÷K ƒ rF   rR   c                   ó4   a € ] tR t^�t o RtRtV 3R ltRtV tR# )ÚGemma4TextModelOutputWithPasta!  
BaseModelOutputWithPast extended with shared_kv_states for KV sharing.

Args:
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
Nc                óˆ   <€ V ^8„  d   Qh/ S[ S[S[S[P                  S[P                  3,          3,          R,          ;R&   # )r7   Nr:   )r=   r>   r?   r;   r@   )rA   rB   s   "€rC   rD   Ú*Gemma4TextModelOutputWithPast.__annotate__�   s3   ø‡ ‚ ñ ™3¡¡e§l¡l±E·L±LÐ&@Õ AÐAÕBÀTÕIÑPò rF   rG   )	rH   rI   rJ   rK   rL   r:   rM   rN   rO   rP   s   @rC   r[   r[   �   s   ø‡ € ñð MQÐ÷ ƒ rF   r[   c                   ó4   a € ] tR t^�t o RtRtV 3R ltRtV tR# )ÚGemma4AudioModelOutputz›
attention_mask (`torch.BoolTensor`, *optional*):
    A torch.BoolTensor of shape `(batch_size, num_frames)`. True for valid positions, False for padding.
Nc                óH   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   # )r7   NÚattention_mask)r;   Ú
BoolTensor)rA   rB   s   "€rC   rD   Ú#Gemma4AudioModelOutput.__annotate__�   s   ø‡ ‚ ñ ×$Ñ$ tÕ+Ñ2ò rF   rG   )	rH   rI   rJ   rK   rL   ra   rM   rN   rO   rP   s   @rC   r_   r_   �   s   ø‡ € ñð
 /3€N÷ ƒ rF   r_   c                   óP   a a€ ] tR t^¨t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma4ClippableLinearc                ó@   <€ V ^8„  d   QhRS[ S[,          RS[RS[RR/# )r7   ÚconfigÚin_featuresÚout_featuresÚreturnN)r0   r-   Úint)rA   rB   s   "€rC   rD   Ú"Gemma4ClippableLinear.__annotate__©   s?   ø€ ÷ Dñ Dá"Ñ%6Õ6ðDñ ðDñ ð	Dð
 
ñDrF   c                ól  <€ \         SV `  4        VP                  V n        \        P                  ! W#R R7      V n        V P                  '       då   \        P                  ! \        P                  ! \        R4      ) 4      4      V n
        \        P                  ! \        P                  ! \        R4      4      4      V n        \        P                  ! \        P                  ! \        R4      ) 4      4      V n        \        P                  ! \        P                  ! \        R4      4      4      V n        R# R# )F©ÚbiasÚinfN)ÚsuperÚ__init__Úuse_clipped_linearsr   ÚLinearÚlinearÚBufferr;   ÚtensorÚfloatÚ	input_minÚ	input_maxÚ
output_minÚ
output_max)Úselfrg   rh   ri   Ú	__class__s   &&&&€rC   rr   ÚGemma4ClippableLinear.__init__©   s¹   ø€ ô 	‰ÑÔØ#)×#=Ñ#=ˆÔ Ü—i’i ÀÔFˆŒà×#×#Ð#ÜŸYšY¤u§|¢|´U¸5³\°MÓ'BÓCˆDŒNÜŸYšY¤u§|¢|´E¸%³LÓ'AÓBˆDŒNÜ Ÿiši¬¯ª´e¸E³l°]Ó(CÓDˆDŒOÜ Ÿiši¬¯ª´U¸5³\Ó(BÓCˆDŽOñ	 $rF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r7   rW   rj   ©r;   r@   )rA   rB   s   "€rC   rD   rl   ¹   s#   ø€ ÷ 	ñ 	¡U§\¡\ð 	±e·l±lñ 	rF   c                ó  € V P                   '       d,   \        P                  ! WP                  V P                  4      pV P                  V4      pV P                   '       d,   \        P                  ! WP                  V P                  4      pV# ©N)rs   r;   Úclampry   rz   ru   r{   r|   )r}   rW   s   &&rC   ÚforwardÚGemma4ClippableLinear.forward¹   s^   € Ø×#×#Ð#Ü!ŸKšK¨·~±~ÀtÇ~Á~ÓVˆMàŸ™ MÓ2ˆà×#×#Ð#Ü!ŸKšK¨·±ÈÏÉÓXˆMàÐrF   )rz   ry   ru   r|   r{   rs   ©	rH   rI   rJ   rK   rr   r†   rN   rO   Ú__classcell__©r~   rB   s   @@rC   re   re   ¨   s    ù‡ € ÷Dó D÷ 	÷ 	ð 	rF   re   c                   óf   a a€ ] tR t^Åt oRV3R lV 3R llltV3R lR ltV3R lR ltRtVtV ;t	# )	ÚGemma4RMSNormc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r7   ÚdimÚepsÚ
with_scale)rk   rx   Úbool)rA   rB   s   "€rC   rD   ÚGemma4RMSNorm.__annotate__Æ   s'   ø€ ÷ Lñ L™Cð L¡eð LÁñ LrF   c                óÆ   <€ \         SV `  4        W n        W0n        V P                  '       d4   \        P
                  ! \        P                  ! V4      R R7      V n        R# R# )T)Úrequires_gradN)	rq   rr   r�   r�   r   Ú	Parameterr;   ÚonesÚweight)r}   rŽ   r�   r�   r~   s   &&&&€rC   rr   ÚGemma4RMSNorm.__init__Æ   s@   ø€ Ü‰ÑÔØŒØ$Œà�?�?ˆ?ÜŸ,š,¤u§z¢z°#£ÀdÔKˆDŽKñ rF   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r7   rW   r‚   )rA   rB   s   "€rC   rD   r’   Î   s   ø€ ÷ =ñ =¡5§<¡<ñ =rF   c                ó¤   € VP                  ^4      P                  RRR7      V P                  ,           pV\        P                   ! VR4      ,          # )r7   T)Úkeepdiméÿÿÿÿç      à¿)ÚpowÚmeanr�   r;   )r}   rW   Úmean_squareds   && rC   Ú_normÚGemma4RMSNorm._normÎ   sA   € Ø$×(Ñ(¨Ó+×0Ñ0°¸TÐ0ÓBÀTÇXÁXÕMˆàœuŸyšy¨°tÓ<Õ<Ð<rF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r�   r‚   )rA   rB   s   "€rC   rD   r’   Ó   s#   ø€ ÷ 4ñ 4¡U§\¡\ð 4±e·l±lñ 4rF   c                óÆ   € V P                  VP                  4       4      pV P                  '       d!   W P                  P                  4       ,          pVP	                  V4      # r„   )r¡   rx   r�   r—   Útype_as)r}   rW   Únormed_outputs   && rC   r†   ÚGemma4RMSNorm.forwardÓ   sH   € ØŸ
™
 =×#6Ñ#6Ó#8Ó9ˆØ�?�?ˆ?Ø)¯K©K×,=Ñ,=Ó,?Õ?ˆMØ×$Ñ$ ]Ó3Ð3rF   )r�   r—   r�   )g�íµ ÷Æ°>T)
rH   rI   rJ   rK   rr   r¡   r†   rN   rO   r‰   rŠ   s   @@rC   rŒ   rŒ   Å   s*   ù‡ € ÷Lõ L÷=ð =÷
4÷ 4ð 4rF   rŒ   c                   óˆ   a a€ ] tR t^Út oRtV3R lV 3R llt]P                  ! 4       V3R lR l4       tV3R lt	Rt
VtV ;t# )Ú Gemma4AudioRelPositionalEncodingzáSinusoidal relative positional encoding for the audio encoder.

Produces position embeddings of shape [1, context_size // 2 + 1, hidden_size] with
concatenated [sin..., cos...] layout matching the original Gemma4 convention.
c                ó    <€ V ^8„  d   QhRS[ /# ©r7   rg   ©r-   )rA   rB   s   "€rC   rD   Ú-Gemma4AudioRelPositionalEncoding.__annotate__ã   s   ø€ ÷ dñ dÑ0ñ drF   c                ó8  <€ \         SV `  4        VP                  V n        VP                  VP                  ,           ^,
          VP
                  ,           V n        RpRpV P                  ^,          p\        P                  ! W2,          4      \        V^,
          ^4      ,          pV\        P                  ! \        P                  ! V4      V) ,          4      ,          p\        P                  ! VP                  ^ 4      P                  ^ 4      RR7      V n        R# )é   ç      ð?ç     ˆÃ@F©Ú
persistentN)rq   rr   Úhidden_sizeÚattention_chunk_sizeÚattention_context_leftÚattention_context_rightÚcontext_sizeÚmathÚlogÚmaxr;   ÚexpÚaranger   rv   Ú	unsqueezeÚinv_timescales)r}   rg   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementr¿   r~   s   &&     €rC   rr   Ú)Gemma4AudioRelPositionalEncoding.__init__ã   sÙ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔà×'Ñ'¨&×*GÑ*GÕGÈ!ÕKÈf×NlÑNlÕlð 	Ôð ˆØˆØ×)Ñ)¨QÕ.ˆÜ"&§(¢(¨=Õ+HÓ"IÌCÐP^ÐabÕPbÐdeÓLfÕ"fÐØ&¬¯ª´5·<²<ÀÓ3OÐSjÐRjÕ3jÓ)kÕkˆÜ Ÿiši¨×(@Ñ(@ÀÓ(C×(MÑ(MÈaÓ(PÐ]bÔcˆÖrF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r�   r‚   )rA   rB   s   "€rC   rD   r­   ñ   s#   ø€ ÷ 7ñ 7¡U§\¡\ð 7±e·l±lñ 7rF   c                ó’  € \         P                  ! V P                  ^,          RRVP                  R7      pVR,          pW P                  P                  VP                  R7      ,          p\         P                  ! \         P                  ! V4      \         P                  ! V4      .RR7      pVP                  VP                  R7      # )r7   ©Údevice©rŽ   ©Údtyperœ   ©.N)
r;   r½   r¸   rÈ   r¿   ÚtoÚcatÚsinÚcosrË   )r}   rW   Úposition_idsÚscaled_timeÚ	pos_embeds   &&   rC   r†   Ú(Gemma4AudioRelPositionalEncoding.forwardð   s’   € ä—|’| D×$5Ñ$5¸Õ$:¸BÀÈ=×K_ÑK_Ô`ˆØ# IÕ.ˆØ"×%8Ñ%8×%;Ñ%;À=×CWÑCWÐ%;Ó%XÕXˆÜ—I’IœuŸyšy¨Ó5´u·y²yÀÓ7MÐNÐTVÔWˆ	Ø�|‰| -×"5Ñ"5ˆ|Ó6Ð6rF   c                ó:   <€ V ^8„  d   Qh/ S[ P                  ;R&   # )r7   r¿   r‚   )rA   rB   s   "€rC   rD   r­   Ú   s   ø‡ ‚ ñ —L‘LÑ ò rF   )r¸   r´   r¿   )rH   rI   rJ   rK   rL   rr   r;   Úno_gradr†   rM   rN   rO   r‰   rŠ   s   @@rC   r©   r©   Ú   s7   ù‡ € ñ÷dó dð ‡]‚]ƒ_÷7ó ð7÷/ … rF   r©   c                   óŽ   a a€ ] tR t^ùt oRtV3R lV 3R lltV3R lR ltV3R lR ltV3R lR	 ltRV3R
 lR llt	Rt
VtV ;t# )ÚGemma4AudioAttentionz3Chunked local attention with relative position biasc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r7   rg   Ú	layer_idx©r-   rk   )rA   rB   s   "€rC   rD   Ú!Gemma4AudioAttention.__annotate__ü   s    ø€ ÷ añ aÑ0ð a¹Sñ arF   c                ó–  <€ \         SV `  4        Wn        W n        VP                  V n        VP                  VP                  ,          V n        VP                  V n	        V P                  R,          \        P                  ! ^4      ,          V n        \        P                  ! ^\        P                  ,           4      \        P                  ! ^4      ,          V n        VP                  V n        VP"                  ^,
          V n        VP&                  V n        V P                   V P$                  ,           V P(                  ,           V n        \-        WP                  V P                  V P                  ,          4      V n        \-        WP                  V P                  V P                  ,          4      V n        \-        WP                  V P                  V P                  ,          4      V n        \-        WP                  VP                  4      V n        \6        P8                  ! VP                  V P                  V P                  ,          RR7      V n        \6        P<                  ! \>        P@                  ! V P                  4      4      V n!        \6        PD                  ! \>        PF                  ! V P
                  4      RR7      V n$        R# )ç      à?Frn   r²   Nr�   )%rq   rr   rg   rÛ   Úattention_logit_capÚattention_logits_soft_capr´   Únum_attention_headsÚhead_dimÚ	num_headsr¹   rº   Úq_scaleÚeÚk_scalerµ   Ú
chunk_sizer¶   Úmax_past_horizonr·   Úmax_future_horizonr¸   re   Úq_projÚk_projÚv_projÚpostr   rt   Úrelative_k_projr•   r;   ÚzerosÚper_dim_scalerv   rw   Úsoftcap©r}   rg   rÛ   r~   s   &&&€rC   rr   ÚGemma4AudioAttention.__init__ü   sÌ  ø€ Ü‰ÑÔØŒØ"ŒØ)/×)CÑ)CˆÔ&Ø×*Ñ*¨f×.HÑ.HÕHˆŒØ×3Ñ3ˆŒàŸ™ tÕ+¬t¯xªx¸«{Õ:ˆŒÜ—x’x ¤D§F¡F¥
Ó+¬d¯hªh°q«kÕ9ˆŒà ×5Ñ5ˆŒØ &× =Ñ =ÀÕ AˆÔØ"(×"@Ñ"@ˆÔØ ŸO™O¨d×.CÑ.CÕCÀd×F]ÑF]Õ]ˆÔä+¨F×4FÑ4FÈÏÉÐY]×YfÑYfÕHfÓgˆŒÜ+¨F×4FÑ4FÈÏÉÐY]×YfÑYfÕHfÓgˆŒÜ+¨F×4FÑ4FÈÏÉÐY]×YfÑYfÕHfÓgˆŒÜ)¨&×2DÑ2DÀf×FXÑFXÓYˆŒ	ä!Ÿyšy¨×);Ñ);¸T¿^¹^ÈdÏmÉmÕ=[ÐbgÔhˆÔÜŸ\š\¬%¯+ª+°d·m±mÓ*DÓEˆÔä—y’y¤§¢¨d×.LÑ.LÓ!MÐZ_Ô`ˆŽrF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r�   r‚   )rA   rB   s   "€rC   rD   rÝ     s'   ø€ ÷ pñ p©u¯|©|ð pÁÇÁñ prF   c           	     ó6  € VP                   w  r#rEW0P                  ,           ^,
          V P                  ,          pW`P                  ,          V,
          p\        P                  ! V^ ^ ^ ^ ^ V34      pVP	                  W&V P                  WE4      P                  4       # )z€Splits a `(batch_size, seq_len, num_heads, head_dim)` tensor into non-overlapping blocks of `chunk_size` along the sequence dim.)Úshaperè   ÚFÚpadÚreshapeÚ
contiguous)r}   rW   Ú
batch_sizeÚseq_lenrä   rã   Ú
num_blocksrù   s   &&      rC   Ú_convert_to_blockÚ&Gemma4AudioAttention._convert_to_block  s|   € à3@×3FÑ3FÑ0ˆ
˜YØ§¡Õ/°!Õ3¸¿¹ÕGˆ
ØŸ?™?Õ*¨WÕ4ˆÜŸš˜m¨a°°A°q¸!¸SÐ-AÓBˆØ×$Ñ$ Z¸T¿_¹_ÈiÓb×mÑmÓoÐorF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r�   r‚   )rA   rB   s   "€rC   rD   rÝ     s#   ø€ ÷ *ñ *±E·L±Lð *ÁUÇ\Á\ñ *rF   c           
     óP  € VP                   w  r#rE\        P                  ! V^ ^ ^ ^ V P                  V P                  V P
                  ,           ^,
          34      pVP                  ^V P                  V P
                  4      p\        P                  ! VR^4      pVP                  4       # )z`Extracts overlapping context windows of `context_size` for every block, strided by `chunk_size`.rœ   )r÷   rø   rù   ré   rê   rè   Úunfoldr¸   r;   Úmovedimrû   )r}   rW   rü   rý   rä   rã   s   &&    rC   Ú_extract_block_contextÚ+Gemma4AudioAttention._extract_block_context  s‘   € à3@×3FÑ3FÑ0ˆ
˜YÜŸšØ˜A˜q ! Q¨×(=Ñ(=¸t×?VÑ?VÐY]×YhÑYhÕ?hÐklÕ?lÐmó
ˆð &×,Ñ,¨Q°×0AÑ0AÀ4Ç?Á?ÓSˆÜŸš m°R¸Ó;ˆØ×'Ñ'Ó)Ð)rF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r7   Úxrj   r‚   )rA   rB   s   "€rC   rD   rÝ   (  s'   ø€ ÷ Sñ S™EŸL™Lð S©U¯\©\ñ SrF   c                ó  € VP                   w  r#rEpV P                  p\        P                  ! V^ V^,           V,
          34      pVP	                  W#WEV^,           ,          4      pVRRWW,          13,          pVP	                  W#WEV4      # )zjRelative position shift for blocked attention. See appendix B of https://huggingface.co/papers/1901.02860..N)r÷   r¸   rø   rù   Úview)r}   r	  rü   rä   rþ   Ú
block_sizeÚposition_lengthr¸   s   &&      rC   Ú
_rel_shiftÚGemma4AudioAttention._rel_shift(  s‚   € àIJÏÉÑFˆ
˜z°Ø×(Ñ(ˆÜ�EŠE�!�a˜¨Õ)¨OÕ;Ð<Ó=ˆØ�F‰F�:¨*ÀLÐSTÕDTÕ6UÓVˆØˆcÐ.�ZÕ.Ð.Ð.Õ/ˆØ�v‰v�j¨ZÀ\ÓRÐRrF   c          
      ó¤   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[S[ P                  R3,          /# )r7   rW   Úposition_embeddingsra   Nrj   )r;   r@   rb   r?   )rA   rB   s   "€rC   rD   rÝ   1  sT   ø€ ÷ 1)ñ 1)á—|‘|ð1)ñ #Ÿ\™\ð1)ñ ×(Ñ(¨4Õ/ð	1)ñ
 
‰u�|‰|˜TÐ!Õ	"ñ1)rF   c                ó,  € VP                   w  rEpWEV P                  V P                  3pV P                  V4      P	                  4       P                  V4      pV P                  V4      P	                  4       P                  V4      p	V P                  V4      P	                  4       P                  V4      p
W€P                  ,          \        P                  ! V P                  4      ,          pW�P                  ,          p	V P                  V4      pV P                  V	4      p	V P                  V
4      p
VP                   ^,          pV P                  V4      pVP                  RV P                  V P                  4      pVP!                  VP"                  R7      pVP%                  ^ ^^^^4      pWÙP%                  ^ ^^^^4      ,          pVP'                  W@P                  RV P                  4      pWüP%                  ^^^ 4      ,          pVP'                  W@P                  W°P(                  R4      pV P+                  V4      pVV,           pVV P,                  ,          p\.        P0                  ! V4      pVV P,                  ,          pVe5   VP3                  VP5                  4       V P6                  P8                  4      p\        P:                  ! VR\.        P<                  R7      P!                  V
P"                  4      pVV
P%                  ^ ^^^^4      ,          pVP%                  ^ ^^^^4      P'                  WKV P(                  ,          R4      pVRRV13,          P?                  4       pV PA                  VP!                  VP"                  4      4      pVV3# )r¯   rÊ   N©rŽ   rË   ºNNNrœ   )!r÷   rä   rã   rë   rx   r  rì   rí   rå   rø   Úsoftplusrñ   rç   rÿ   r  rï   rÍ   rË   Úpermuterú   rè   r  rò   r;   ÚtanhÚmasked_fillÚlogical_notrg   Úattention_invalid_logits_valueÚsoftmaxÚfloat32rû   rî   )r}   rW   r  ra   rü   Ú
seq_lengthÚ_Úhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrþ   Úrelative_key_statesÚqueriesÚ	matrix_acÚqueries_flatÚ	matrix_bdÚattn_weightsÚattn_outputs   &&&&               rC   r†   ÚGemma4AudioAttention.forward1  sï  € ð %2×$7Ñ$7Ñ!ˆ
 Ø"°·±ÀÇÁÐNˆà—{‘{ =Ó1×7Ñ7Ó9×>Ñ>¸|ÓLˆØ—[‘[ Ó/×5Ñ5Ó7×<Ñ<¸\ÓJˆ
Ø—{‘{ =Ó1×7Ñ7Ó9×>Ñ>¸|ÓLˆà#§l¡lÕ2´Q·Z²ZÀ×@RÑ@RÓ5SÕSˆØ§,¡,Õ.ˆ
à×-Ñ-¨lÓ;ˆØ×0Ñ0°Ó<ˆ
Ø×2Ñ2°<Ó@ˆØ!×'Ñ'¨Õ*ˆ
à"×2Ñ2Ð3FÓGÐØ1×6Ñ6°r¸4¿>¹>È4Ï=É=ÓYÐØ1×4Ñ4¸<×;MÑ;MÐ4ÓNÐà×&Ñ& q¨!¨Q°°1Ó5ˆØ×0Ñ0°°A°q¸!¸QÓ?Õ?ˆ	à—‘ z·>±>À2ÀtÇ}Á}ÓUˆØ ×#>Ñ#>¸qÀ!ÀQÓ#GÕGˆ	Ø×%Ñ% j·.±.À*ÏoÉoÐ_aÓbˆ	Ø—O‘O IÓ.ˆ	à  9Õ,ˆØ# d§l¡lÕ2ˆÜ—z’z ,Ó/ˆØ# d§l¡lÕ2ˆàÒ%Ø'×3Ñ3Ø×*Ñ*Ó,¨d¯k©k×.XÑ.XóˆLô —y’y °2¼U¿]¹]ÔK×NÑNÈ|×OaÑOaÓbˆØ" \×%9Ñ%9¸!¸QÀÀ1ÀaÓ%HÕHˆØ!×)Ñ)¨!¨Q°°1°aÓ8×@Ñ@ÀÐZ^×ZiÑZiÕMiÐkmÓnˆØ! ! [ j [ .Õ1×<Ñ<Ó>ˆØ—i‘i §¡¨}×/BÑ/BÓ CÓDˆà˜LÐ(Ð(rF   )rá   rè   rg   r¸   rã   rì   rç   rÛ   rê   ré   rä   rñ   rî   rë   rå   rï   rò   rí   r„   )rH   rI   rJ   rK   rL   rr   rÿ   r  r  r†   rN   rO   r‰   rŠ   s   @@rC   rØ   rØ   ù   sE   ù‡ € Ù=÷aó a÷4pð p÷*ð *÷Sð S÷1)÷ 1)ò 1)rF   rØ   c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )Ú'Gemma4AudioSubSampleConvProjectionLayerie  c           	     óÖ   <€ \         SV `  4        \        P                  ! VVRR^RR7      V n        \        P
                  ! W#RRR7      V n        \        P                  ! 4       V n        R# )é   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingro   T)r�   Úelementwise_affinero   N)r.  r.  )r7   r7   )	rq   rr   r   ÚConv2dÚconvÚ	LayerNormÚnormÚReLUÚact)r}   r/  r0  Únorm_epsr~   s   &&&&€rC   rr   Ú0Gemma4AudioSubSampleConvProjectionLayer.__init__f  sU   ø€ Ü‰ÑÔÜ—I’IØ#Ø%ØØØØô
ˆŒ	ô —L’L ÐPTÐ[`ÔaˆŒ	Ü—7’7“9ˆŽrF   c                ó\   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          /# )r7   rW   ÚmaskNr‚   )rA   rB   s   "€rC   rD   Ú4Gemma4AudioSubSampleConvProjectionLayer.__annotate__s  s(   ø€ ÷ #ñ #¡U§\¡\ð #¹¿¹ÈÕ9Lñ #rF   c           
     ó   € Ve,   VP                  VP                  R7      pWR,          ,          pV P                  VP                  V P                  P                  P                  4      4      pV P                  V P                  VP                  ^ ^^^4      4      P                  ^ ^^^4      P                  4       4      pVe
   VR,          pW3# )NrÇ   )r  Nr  N)r  :NNr7   )	rÍ   rÈ   r6  r—   rË   r:  r8  r  rû   )r}   rW   r>  s   &&&rC   r†   Ú/Gemma4AudioSubSampleConvProjectionLayer.forwards  s³   € ØÒØ—7‘7 -×"6Ñ"6�7Ó7ˆDØ)Ð1AÕ,BÕBˆMàŸ	™	 -×"2Ñ"2°4·9±9×3CÑ3C×3IÑ3IÓ"JÓKˆØŸ™ §¡¨=×+@Ñ+@ÀÀAÀqÈ!Ó+LÓ!M×!UÑ!UÐVWÐYZÐ\]Ð_`Ó!a×!lÑ!lÓ!nÓoˆàÒØ˜•<ˆDàÐ"Ð"rF   )r:  r6  r8  r„   rˆ   rŠ   s   @@rC   r,  r,  e  s   ù‡ € õ÷#÷ #ò #rF   r,  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú"Gemma4AudioSubSampleConvProjectioni�  c                ó    <€ V ^8„  d   QhRS[ /# r«   r¬   )rA   rB   s   "€rC   rD   Ú/Gemma4AudioSubSampleConvProjection.__annotate__‚  s   ø€ ÷ [ñ [Ñ0ñ [rF   c                ó°  <€ \         SV `  4        \        ^VP                  ^ ,          VP                  R7      V n        \        VP                  ^ ,          VP                  ^,          VP                  R7      V n        VP                  ^ ,          ^,          VP                  ^,          ,          p\        P                  ! W!P                  RR7      V n
        R# )r¯   )r/  r0  r;  Frn   N)rq   rr   r,  Úsubsampling_conv_channelsÚrms_norm_epsÚlayer0Úlayer1r   rt   r´   Úinput_proj_linear)r}   rg   Úproj_input_dimr~   s   && €rC   rr   Ú+Gemma4AudioSubSampleConvProjection.__init__‚  s©   ø€ Ü‰ÑÔÜ=ØØ×9Ñ9¸!Õ<Ø×(Ñ(ô
ˆŒô
 >Ø×8Ñ8¸Õ;Ø×9Ñ9¸!Õ<Ø×(Ñ(ô
ˆŒð
 !×:Ñ:¸1Õ=ÀÕBÀf×FfÑFfÐghÕFiÕiˆÜ!#§¢¨>×;MÑ;MÐTYÔ!ZˆÖrF   c                ó    <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  3,          /# )r7   Úinput_featuresÚinput_features_maskNrj   ©r;   r@   r?   )rA   rB   s   "€rC   rD   rE  ‘  sH   ø€ ÷ ;ñ ;áŸ™ð;ñ #Ÿ\™\¨DÕ0ð;ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ	;rF   c                ó  € VP                  ^4      pV P                  W24      w  r4V P                  W44      w  r4VP                  w  rVrvVP	                  ^ ^^^4      P                  4       P                  WWR4      pV P                  V4      V3# )r¯   rœ   )r¾   rI  rJ  r÷   r  rû   rú   rK  )r}   rO  rP  rW   r>  rü   r  rý   s   &&&     rC   r†   Ú*Gemma4AudioSubSampleConvProjection.forward‘  s‰   € ð
 '×0Ñ0°Ó3ˆØ"Ÿk™k¨-ÓMÑˆØ"Ÿk™k¨-Ó>Ñˆà$1×$7Ñ$7Ñ!ˆ
�wØ%×-Ñ-¨a°°A°qÓ9×DÑDÓF×NÑNÈzÐdfÓgˆØ×%Ñ% mÓ4°dÐ:Ð:rF   )rK  rI  rJ  r„   rˆ   rŠ   s   @@rC   rC  rC  �  s    ù‡ € ÷[ó [÷;÷ ;ò ;rF   rC  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma4AudioFeedForwardiŸ  c                ó    <€ V ^8„  d   QhRS[ /# r«   r¬   )rA   rB   s   "€rC   rD   Ú#Gemma4AudioFeedForward.__annotate__   s   ø€ ÷ 7ñ 7Ñ0ñ 7rF   c                óÄ  <€ \         SV `  4        Wn        \        WP                  VP                  ^,          4      V n        \        WP                  ^,          VP                  4      V n        \        VP                  4      V n        \        VP                  4      V n	        \        VP                  ,          V n        VP                  V n        VP                  V n        R# )é   N)rq   rr   rg   re   r´   Úffw_layer_1Úffw_layer_2rŒ   Úpre_layer_normÚpost_layer_normr
   Ú
hidden_actÚact_fnÚgradient_clippingÚresidual_weightÚpost_layer_scale©r}   rg   r~   s   &&€rC   rr   ÚGemma4AudioFeedForward.__init__   s©   ø€ Ü‰ÑÔØŒä0°×9KÑ9KÈV×M_ÑM_ÐbcÕMcÓdˆÔÜ0°×9KÑ9KÈaÕ9OÐQW×QcÑQcÓdˆÔä+¨F×,>Ñ,>Ó?ˆÔÜ,¨V×-?Ñ-?Ó@ˆÔÜ˜V×.Ñ.Õ/ˆŒà!'×!9Ñ!9ˆÔØ &× 6Ñ 6ˆÖrF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r�   r‚   )rA   rB   s   "€rC   rD   rW  ®  ó#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rF   c                óÄ  € \        V P                  \        P                  ! VP                  4      P
                  4      pTp\        P                  ! W) V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      p\        P                  ! W) V4      pV P                  V4      pWP                  ,          pW,          pV# r„   )Úminr`  r;   ÚfinforË   r»   r…   r\  rZ  r_  r[  r]  rb  )r}   rW   r`  Úresiduals   &&  rC   r†   ÚGemma4AudioFeedForward.forward®  sÁ   € ä × 6Ñ 6¼¿ºÀM×DWÑDWÓ8X×8\Ñ8\Ó]Ðà ˆÜŸš MÐ3EÐGXÓYˆØ×+Ñ+¨MÓ:ˆà×(Ñ(¨Ó7ˆØŸ™ MÓ2ˆØ×(Ñ(¨Ó7ˆäŸš MÐ3EÐGXÓYˆØ×,Ñ,¨]Ó;ˆØ×.Ñ.Õ.ˆØÕ!ˆàÐrF   )r_  rg   rZ  r[  r`  r]  rb  r\  rˆ   rŠ   s   @@rC   rU  rU  Ÿ  s   ù‡ € ÷7ó 7÷÷ ð rF   rU  c                   óN   a a€ ] tR tRt o]R 4       tV3R lV 3R lltRtVtV ;t	# )ÚGemma4AudioCausalConv1diÃ  c                ó¦   € V P                   ^ ,          ^,
          V P                  ^ ,          ,          ^,           pWP                  ^ ,          ,
          # ©r   )r1  Údilationr2  )r}   Úeffective_kernel_sizes   & rC   Úleft_padÚ Gemma4AudioCausalConv1d.left_padÑ  s<   € à!%×!1Ñ!1°!Õ!4°qÕ!8¸D¿M¹MÈ!Õ<LÕ LÈqÕ PÐØ$§{¡{°1¥~Õ5Ð5rF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  r‚   )rA   rB   s   "€rC   rD   Ú$Gemma4AudioCausalConv1d.__annotate__Ö  s'   ø€ ÷ "ñ "á�<‰<ð"ñ 
�‰ñ	"rF   c                óx   <€ \         P                  P                  WP                  ^ 34      p\        SV `  V4      # ro  )r   r   rù   rr  rq   r†   )r}   r	  r~   s   &&€rC   r†   ÚGemma4AudioCausalConv1d.forwardÖ  s1   ø€ ô �M‰M×Ñ˜a§-¡-°Ð!3Ó4ˆä‰w‰˜qÓ!Ð!rF   rG   )
rH   rI   rJ   rK   r   rr  r†   rN   rO   r‰   rŠ   s   @@rC   rm  rm  Ã  s(   ù‡ € ð ñ6ó ð6÷"÷ "ó "rF   rm  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma4AudioLightConv1diä  c                ó    <€ V ^8„  d   QhRS[ /# r«   r¬   )rA   rB   s   "€rC   rD   Ú#Gemma4AudioLightConv1d.__annotate__å  s   ø€ ÷ :ñ :Ñ0ñ :rF   c                óB  <€ \         SV `  4        Wn        \        WP                  VP                  ^,          4      V n        \        WP                  VP                  4      V n        \        VP                  VP                  VP                  VP                  RR7      V n	        \        VP                  VP                  RR7      V n        \        VP                  VP                  RR7      V n        \        VP                  ,          V n        VP"                  V n        R# )r7   F)r/  r0  r1  Úgroupsro   T©r�   r�   N)rq   rr   rg   re   r´   Úlinear_startÚ
linear_endrm  Úconv_kernel_sizeÚdepthwise_conv1drŒ   rH  r\  Ú	conv_normr
   r^  r_  r`  rc  s   &&€rC   rr   ÚGemma4AudioLightConv1d.__init__å  sß   ø€ Ü‰ÑÔØŒä1°&×:LÑ:LÈf×N`ÑN`ÐcdÕNdÓeˆÔÜ/°×8JÑ8JÈF×L^ÑL^Ó_ˆŒÜ 7Ø×*Ñ*Ø×+Ñ+Ø×/Ñ/Ø×%Ñ%Øô!
ˆÔô ,¨F×,>Ñ,>ÀF×DWÑDWÐdhÔiˆÔÜ& v×'9Ñ'9¸v×?RÑ?RÐ_cÔdˆŒÜ˜V×.Ñ.Õ/ˆŒà!'×!9Ñ!9ˆÖrF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r�   r‚   )rA   rB   s   "€rC   rD   r{  ù  rf  rF   c                ó  € TpV P                  V4      pV P                  V4      p\        P                  P	                  VRR7      pV P                  VP                  ^^4      4      P                  ^^4      p\        V P                  \        P                  ! VP                  4      P                  4      p\        P                  ! W) V4      pV P                  V4      pV P                  V4      pV P!                  V4      pW,          pV# )r¯   rÉ   rœ   )r\  r  r   r   Úglur‚  Ú	transposerh  r`  r;   ri  rË   r»   r…   rƒ  r_  r€  )r}   rW   rj  r`  s   &&  rC   r†   ÚGemma4AudioLightConv1d.forwardù  sâ   € Ø ˆà×+Ñ+¨MÓ:ˆØ×)Ñ)¨-Ó8ˆÜŸ™×)Ñ)¨-¸RÐ)Ó@ˆà×-Ñ-¨m×.EÑ.EÀaÈÓ.KÓL×VÑVÐWXÐZ[Ó\ˆô   × 6Ñ 6¼¿ºÀM×DWÑDWÓ8X×8\Ñ8\Ó]ÐÜŸš MÐ3EÐGXÓYˆØŸ™ }Ó5ˆàŸ™ MÓ2ˆØŸ™¨Ó6ˆØÕ!ˆØÐrF   )r_  rg   rƒ  r‚  r`  r€  r  r\  rˆ   rŠ   s   @@rC   ry  ry  ä  s   ù‡ € ÷:ó :÷(÷ ð rF   ry  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma4AudioLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÚ   rÜ   )rA   rB   s   "€rC   rD   ÚGemma4AudioLayer.__annotate__  s   ø€ ÷ :ñ :Ñ0ð :¹Sñ :rF   c                ón  <€ \         SV `  4        Wn        \        V4      V n        \        V4      V n        \        W4      V n        \        V4      V n	        \        VP                  4      V n        \        VP                  4      V n        \        VP                  4      V n        VP                  V n        R # r„   )rq   rr   rg   rU  Úfeed_forward1Úfeed_forward2rØ   Ú	self_attnry  Úlconv1drŒ   r´   Únorm_pre_attnÚnorm_post_attnÚnorm_outr`  ró   s   &&&€rC   rr   ÚGemma4AudioLayer.__init__  s‹   ø€ Ü‰ÑÔØŒä3°FÓ;ˆÔÜ3°FÓ;ˆÔÜ-¨fÓ@ˆŒÜ-¨fÓ5ˆŒä*¨6×+=Ñ+=Ó>ˆÔÜ+¨F×,>Ñ,>Ó?ˆÔÜ% f×&8Ñ&8Ó9ˆŒà!'×!9Ñ!9ˆÖrF   c          
      ó¦   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  RS[S[,          RS[ P                  /# )r7   rW   ra   Nr  Úkwargsrj   )r;   r@   rb   r    r"   )rA   rB   s   "€rC   rD   r�    sZ   ø€ ÷  ñ  á—|‘|ð ñ ×(Ñ(¨4Õ/ð ñ #Ÿ\™\ð	 ñ
 Ñ+Õ,ð ñ 
�‰ñ rF   c                óF  € \        V P                  \        P                  ! V P                  P
                  P                  4      P                  4      pV P                  V4      pTp\        P                  ! W) V4      pV P	                  V4      pV P                  VVVR 7      w  r\        P                  ! W) V4      pV P                  V4      pW,          pV P                  V4      pV P                  V4      p\        P                  ! W) V4      pV P                  V4      pV# ))rW   r  ra   )rh  r`  r;   ri  r“  r—   rË   r»   r�  r…   r‘  r”  r’  r�  r•  )r}   rW   ra   r  r˜  r`  rj  r  s   &&&&,   rC   r†   ÚGemma4AudioLayer.forward  s  € ô   × 6Ñ 6¼¿ºÀD×DVÑDV×D]ÑD]×DcÑDcÓ8d×8hÑ8hÓiÐà×*Ñ*¨=Ó9ˆØ ˆäŸš MÐ3EÐGXÓYˆØ×*Ñ*¨=Ó9ˆàŸ>™>Ø'Ø 3Ø)ð *ó 
Ñˆô Ÿš MÐ3EÐGXÓYˆØ×+Ñ+¨MÓ:ˆØÕ!ˆàŸ™ ]Ó3ˆØ×*Ñ*¨=Ó9ˆäŸš MÐ3EÐGXÓYˆØŸ™ mÓ4ˆàÐrF   )	rg   r�  r�  r`  r’  r•  r”  r“  r‘  rˆ   rŠ   s   @@rC   r‹  r‹    s   ù‡ € ÷:ó :÷ ÷  ð  rF   r‹  c                   ób   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR ltRtVtV ;t	# )	ÚGemma4VisionPatchEmbedderiC  c                ó    <€ V ^8„  d   QhRS[ /# r«   ©r0   )rA   rB   s   "€rC   rD   Ú&Gemma4VisionPatchEmbedder.__annotate__D  s   ø€ ÷ tñ tÑ1ñ trF   c                ó   <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP
                  V n        \        P                  ! ^V P                  ^,          ,          V P                  RR7      V n        \        P                  ! \        P                  ! ^V P
                  V P                  4      4      V n        R# )r.  Frn   N)rq   rr   rg   r´   Ú
patch_sizeÚposition_embedding_sizer   rt   Ú
input_projr•   r;   r–   Úposition_embedding_tablerc  s   &&€rC   rr   Ú"Gemma4VisionPatchEmbedder.__init__D  s�   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ ×+Ñ+ˆŒØ'-×'EÑ'EˆÔ$äŸ)š) A¨¯©¸Õ(:Õ$:¸D×<LÑ<LÐSXÔYˆŒÜ(*¯ª´U·Z²ZÀÀ4×C_ÑC_Ðae×aqÑaqÓ5rÓ(sˆÖ%rF   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r7   Úpixel_position_idsÚpadding_positionsrj   r‚   )rA   rB   s   "€rC   rD   rŸ  N  s4   ø€ ÷ #ñ #±u·|±|ð #ÑX]×XdÑXdð #Ñin×iuÑiuñ #rF   c                óD  € VP                  ^ R7      p\        P                  ! VR,          V P                  ^ ,          4      p\        P                  ! VR,          V P                  ^,          4      pWE,           p\        P
                  ! VP                  R4      RV4      pV# )aC  Compute 2-D patch position embeddings via embedding lookup.

``pixel_position_ids`` has shape ``(batch, num_patches, 2)`` where the
last dimension holds (x, y) indices into ``position_embedding_table``
(shape ``(2, position_embedding_size, hidden_size)``).  The result is the
sum of the x- and y-embeddings for each patch.
©rh  ç        ©.r   ©.r¯   rœ   )r…   rø   Ú	embeddingr¤  r;   Úwherer¾   )r}   r§  r¨  Úclamped_positionsÚx_embÚy_embr  s   &&&    rC   Ú_position_embeddingsÚ.Gemma4VisionPatchEmbedder._position_embeddingsN  sŒ   € ð /×4Ñ4¸Ð4Ó;Ðô —’Ð-¨fÕ5°t×7TÑ7TÐUVÕ7WÓXˆÜ—’Ð-¨fÕ5°t×7TÑ7TÐUVÕ7WÓXˆØ#�mÐÜ#ŸkškÐ*;×*EÑ*EÀbÓ*IÈ3ÐPcÓdÐØ"Ð"rF   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r7   Úpixel_valuesr§  r¨  rj   r‚   )rA   rB   s   "€rC   rD   rŸ  d  s?   ø€ ÷ 	3ñ 	3Ù!ŸL™Lð	3Ù>C¿l¹lð	3Ù_d×_kÑ_kð	3á	�‰ñ	3rF   c                óü   € ^VR,
          ,          pV P                   P                  P                  ;pP                  '       d   VP	                  V4      pV P                  V4      pV P                  W#4      pWV,           # )r7   rß   )r£  r—   rË   Úis_floating_pointrÍ   r³  )r}   r¶  r§  r¨  Útarget_dtyperW   r  s   &&&&   rC   r†   Ú!Gemma4VisionPatchEmbedder.forwardd  sk   € ð ˜L¨3Õ.Õ/ˆØ ŸO™O×2Ñ2×8Ñ8Ð8ˆL×K×KÐKØ'Ÿ?™?¨<Ó8ˆLØŸ™¨Ó5ˆØ"×7Ñ7Ð8JÓ^ÐØÕ2Ð2rF   )rg   r´   r£  r¡  r¢  r¤  )
rH   rI   rJ   rK   rr   r³  r†   rN   rO   r‰   rŠ   s   @@rC   rœ  rœ  C  s*   ù‡ € ÷tó t÷#ð #÷,	3÷ 	3ð 	3rF   rœ  c                   ój   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltR
V3R lR lltR	tVt	V ;t
# )ÚGemma4VisionPoolerip  aK  Spatial pooling and ``sqrt(hidden_size)`` scaling for vision encodings.

The scaling expands the activation magnitude, which can exceed the float16 range, so it is
computed in float32 and the pooled features are returned in float32. The caller
(``Gemma4VisionModel.forward``) standardizes them and casts back to the working dtype.
c                ó    <€ V ^8„  d   QhRS[ /# r«   rž  )rA   rB   s   "€rC   rD   ÚGemma4VisionPooler.__annotate__x  s   ø€ ÷ 6ñ 6Ñ1ñ 6rF   c                óv   <€ \         SV `  4        VP                  V n        V P                  R ,          V n        R# )rß   N)rq   rr   r´   Úroot_hidden_sizerc  s   &&€rC   rr   ÚGemma4VisionPooler.__init__x  s/   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ $× 0Ñ 0°#Õ 5ˆÖrF   c          
      ó˜   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[RS[S[ P                  S[ P                  3,          /# )r7   rW   r§  Úlengthrj   ©r;   r@   rk   r?   )rA   rB   s   "€rC   rD   r¾  }  sG   ø€ ÷ 4ñ 4Ù"Ÿ\™\ð4Ù?D¿|¹|ð4ÙUXð4á	‰u�|‰|™UŸ\™\Ð)Õ	*ñ4rF   c                ó  € VP                   ^,          p\        WC,          R,          4      pV^,          pWc,          V8w  d(   \        RVP                    RV RV: RV: RV R24      hVP                  ^ R7      pVR,          P	                  RR	R
7      ^ ,          ^,           p\
        P                  ! WuRR7      p	V	R,          W…,          V	R,          ,          ,           p	\        P                  ! V	P                  4       V4      P                  4       V,          p
V
P                  ^^4      VP                  4       ,          p\
        P                  ! V
^ 8H  P                  ^R7      4      pVP                  VP                  4      V3# )z¹
2D spatial pooling according to patch positions.
Pools the input tokens by averaging patches within a `k^2` grid, where `k` is determined by the ratio between
input and output lengths
rß   zCannot pool z to z: k=z^2 times length=z	 must be Ú.rª  T©rŽ   r›   Úfloor)Úrounding_moderÉ   r¬  rœ   r­  )r÷   rk   Ú
ValueErrorr…   r»   r;   Údivrø   Úone_hotÚlongrx   rˆ  r  ÚallrÍ   rË   )r}   rW   r§  rÃ  Úinput_seq_lenÚkÚ	k_squaredr°  Úmax_xÚkernel_idxsÚweightsÚoutputr>  s   &&&&         rC   Ú_avg_pool_by_positionsÚ)Gemma4VisionPooler._avg_pool_by_positions}  s`  € ð &×+Ñ+¨AÕ.ˆÜ�Õ(¨SÕ0Ó1ˆØ�q•Dˆ	ØÕ Ô.ÜØ˜}×2Ñ2Ð3°4¸°x¸uÀ!ÁÐEVÈvÉiÐW`ÐanÐ`oÐopÐqóð ð /×4Ñ4¸Ð4Ó;ÐØ! &Õ)×-Ñ-°"¸dÐ-ÓCÀAÕFÈÕJˆÜ—i’iÐ 1ÀGÔLˆØ! &Õ)¨U­Z¸;ÀvÕ;NÕ,NÕNˆÜ—)’)˜K×,Ñ,Ó.°Ó7×=Ñ=Ó?À)ÕKˆØ×"Ñ" 1 aÓ(¨=×+>Ñ+>Ó+@Õ@ˆÜ× Ò  '¨Q¡,×!3Ñ!3¸Ð!3Ó!:Ó;ˆØ�y‰y˜×,Ñ,Ó-¨tÐ3Ð3rF   c                óÀ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[R,          RS[S[ P                  S[ P                  3,          /# )r7   rW   r§  r¨  Úoutput_lengthNrj   rÄ  )rA   rB   s   "€rC   rD   r¾  ˜  s_   ø€ ÷ 0ñ 0á—|‘|ð0ñ "ŸL™Lð0ñ !Ÿ<™<ð	0ñ
 ˜T•zð0ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ0rF   c                óX  € WAP                   ^,          8”  d$   \        RV RVP                   ^,           R24      hVP                  VP                  R4      R4      pVP                   ^,          V8w  d   V P	                  WV4      w  rVP                  4       V P                  ,          pW3# )r¯   z*Cannot output more soft tokens (requested z) than there are patches (z9). Change the value of `num_soft_tokens` when processing.r«  rœ   )r÷   rÊ  r  r¾   rÖ  rx   rÀ  )r}   rW   r§  r¨  rÙ  s   &&&&&rC   r†   ÚGemma4VisionPooler.forward˜  s¹   € ð ×.Ñ.¨qÕ1Ô1ÜØ<¸]¸Oð LØ"×(Ñ(¨Õ+Ð,Ð,eðgóð ð
 &×1Ñ1Ð2C×2MÑ2MÈbÓ2QÐSVÓWˆà×Ñ˜qÕ! ]Ô2Ø/3×/JÑ/JØ°=ó0Ñ,ˆMð &×+Ñ+Ó-°×0EÑ0EÕEˆØÐ/Ð/rF   )r´   rÀ  r„   )rH   rI   rJ   rK   rL   rr   rÖ  r†   rN   rO   r‰   rŠ   s   @@rC   r¼  r¼  p  s-   ù‡ € ñ÷6ó 6÷
4ð 4÷60÷ 0ò 0rF   r¼  c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚGemma4VisionMLPi³  c                ó    <€ V ^8„  d   QhRS[ /# r«   rž  )rA   rB   s   "€rC   rD   ÚGemma4VisionMLP.__annotate__´  s   ø€ ÷ 7ñ 7Ñ1ñ 7rF   c                óŠ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \        WP                  V P                  4      V n        \        WP                  V P                  4      V n        \        WP                  V P                  4      V n        \        VP                  ,          V n        R # r„   )rq   rr   rg   r´   Úintermediate_sizere   Ú	gate_projÚup_projÚ	down_projr
   Úhidden_activationr_  rc  s   &&€rC   rr   ÚGemma4VisionMLP.__init__´  s�   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜ.¨v×7GÑ7GÈ×I_ÑI_Ó`ˆŒÜ,¨V×5EÑ5EÀt×G]ÑG]Ó^ˆŒÜ.¨v×7MÑ7MÈt×O_ÑO_Ó`ˆŒÜ˜V×5Ñ5Õ6ˆŽrF   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r„   ©rä  r_  râ  rã  ©r}   r	  rä  s   && rC   r†   ÚGemma4VisionMLP.forward¾  ó6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐrF   ©r_  rg   rä  râ  r´   rá  rã  rˆ   rŠ   s   @@rC   rÝ  rÝ  ³  s   ù‡ € ÷7ó 7÷ò rF   rÝ  c                   óÎ   a a€ ] tR tRt oRt]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR	 ll4       4       t]	P                  ! 4       R
 4       tR tRtVtV ;t# )ÚGemma4VisionRotaryEmbeddingiÃ  zí
Simple axial 2D rope with same freqs used for H and W grids. The freqs are
pre-computed using `head-dim//4` which is later used to concat H and W positions.
The final angles rotate over the whole head dim, no partial rotation involved.
rÈ   ú5.18©Úversionc                ó    <€ V ^8„  d   QhRS[ /# r«   rž  )rA   rB   s   "€rC   rD   Ú(Gemma4VisionRotaryEmbedding.__annotate__Ë  s   ø€ ÷ Oñ OÑ1ñ OrF   c                óÎ  <€ \         SV `  4        Wn        V P                  P                  R ,          V n        V P
                  pV P                  R8w  d/   \        V P                  P                   RV P                   24      hV! V P                  V4      w  q@n	        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚaxialz) supports only axial rope, but requested Fr²   N)rq   rr   rg   Úrope_parametersrõ  Úcompute_axial_rope_parametersrÊ  r~   rH   Úattention_scalingr   rv   Úinv_freqÚcloneÚoriginal_inv_freq)r}   rg   rÈ   Úrope_init_fnrú  r~   s   &&&  €rC   rr   Ú$Gemma4VisionRotaryEmbedding.__init__Ê  s¬   ø€ ä‰ÑÔØŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!CÑ!CˆØ�>‰>˜WÔ$Ü §¡× 7Ñ 7Ð8Ð8aÐbf×bpÑbpÐaqÐrÓsÐsÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖrF   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r7   rg   rj   )r0   r?   r;   r@   rx   )rA   rB   s   "€rC   rD   ró  Ú  s/   ø€ ÷ 5ñ 5Ñ.@ð 5Ñ\aÑbg×bnÑbnÑpuÐbuÕ\vñ 5rF   c           	     óB  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pV^,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetarã   Nr°   rÊ   ©r÷  Úgetattrr´   râ   r;   r½   rx   rÍ   )rg   rÈ   r˜  ÚbaserŽ   Úspatial_dimÚattention_factorrú  s   &&,     rC   rø  Ú9Gemma4VisionRotaryEmbedding.compute_axial_rope_parametersØ  s‡   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆØ˜Q•hˆàÐØ˜$¤5§<¢<°°;ÀÌÏÉÔ#UÐXcÕ#cÕdÕeˆØ�{‰{˜6Ó"Ð$4Ð4Ð4rF   c                óÒ  € V P                   R,          P                  4       pVR,          P                  4       p\        VP                  P                  \
        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        WC,          pVP                  4       V P                  ,          pVP                  4       V P                  ,          pR R R 4       V P                  X4      pV P                  X4      pVP                  VP                  4      VP                  VP                  4      3#   + '       g   i     Lh; i)NÚmpsÚcpuF©Údevice_typeÚenabled)N.rÌ   )rú  rx   Ú
isinstancerÈ   Útyper>   r(   rÐ   rù  rÏ   Úrecomposition_frequenciesrÍ   rË   )	r}   r	  rÑ   Úinv_freq_expandedÚposition_ids_expandedr  ÚfreqsrÐ   rÏ   s	   &&&      rC   r†   Ú#Gemma4VisionRotaryEmbedding.forwardì  sø   € à ŸM™M¨)Õ4×:Ñ:Ó<ÐØ ,¨YÕ 7× =Ñ =Ó ?Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ)Õ=ˆEØ—)‘)“+ × 6Ñ 6Õ6ˆCØ—)‘)“+ × 6Ñ 6Õ6ˆC÷ Dð
 ×,Ñ,¨SÓ1ˆØ×,Ñ,¨SÓ1ˆØ�v‰v�a—g‘g‹ §¡ q§w¡w£Ð/Ð/÷ D×Cús   Â+AEÅE&	c                óX   € VR,          VR,          r2\         P                  ! W"W3.RR7      # )zM
Recompose the frequencies into the final spatial layout used per each grid.
rÉ   )r  r  r   )r  r  r¯   rœ   )r;   rÎ   )r}   ÚfreqÚfreq_hÚfreq_ws   &&  rC   r  Ú5Gemma4VisionRotaryEmbedding.recomposition_frequenciesü  s)   € ð
 ˜g�¨¨W­�Ü�yŠy˜&¨&Ð9¸rÔBÐBrF   )rù  rg   rú  rü  rõ  r„   )rH   rI   rJ   rK   rL   r'   rr   Ústaticmethodrø  r;   rÖ   r†   r  rN   rO   r‰   rŠ   s   @@rC   rî  rî  Ã  s{   ù‡ € ññ �X vÔ.÷Oô Oó /ðOð Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_ñ0ó ð0÷Cò CrF   rî  c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..NrÉ   rœ   )r÷   r;   rÎ   )r	  Úx1Úx2s   &  rC   Úrotate_halfr    sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'rF   c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        /# )r7   r	  rÐ   rÏ   Úunsqueeze_dim©r;   r@   rk   )rA   s   "rC   rD   rD     s5   € ÷ .ñ .œEŸL™Lð .¬u¯|©|ð .Ä%Ç,Á,ð .Ô_bñ .rF   c                ó„   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           # )a$  Applies Rotary Position Embedding to the query and key tensors.

Args:
    x (`torch.Tensor`): The tensor to embed.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)r¾   r  ©r	  rÐ   rÏ   r   s   &&&&rC   Úapply_rotary_pos_embr$    s6   € ð" �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�Gœ A›¨Õ,Õ-Ð-rF   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r7   rW   Ún_reprj   r!  )rA   s   "rC   rD   rD   "  s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	UrF   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r  r  Nr  r  )r÷   Úexpandrú   )rW   r&  ÚbatchÚnum_key_value_headsÚslenrã   s   &&    rC   Ú	repeat_kvr,  "  sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTrF   c                ór  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        \
        ,          R\        R,          R	\        R,          R
\        \        P                  \        P                  3,          /	# )r7   ÚmoduleÚqueryÚkeyÚvaluera   NÚdropoutÚscalingrò   rj   )r   ÚModuler;   r@   rx   rk   r?   )rA   s   "rC   rD   rD   .  s™   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ”S�[ð%ô �T�\ð%ô �T�\ð%ô Œ5�<‰<œŸ™Ð%Õ&ñ%rF   c                 ó¤  € Vf   V P                   R,          p\        W P                  4      p	\        W0P                  4      p
\        P                  ! WP                  ^^4      4      V,          pVe'   W·,          p\        P                  ! V4      pW·,          pVe	   W´,           p\        P                  P                  VR\        P                  R7      P                  VP                  4      p\        P                  P                  WµV P                  R7      p\        P                  ! Wº4      pVP                  ^^4      P                  4       pWË3# )Nr  )ÚpÚtrainingr�   rœ   )rã   r,  Únum_key_value_groupsr;   Úmatmulrˆ  r  r   r   r  r  rÍ   rË   r2  r7  rû   )r.  r/  r0  r1  ra   r2  r3  rò   r˜  r!  r"  r(  r)  s   &&&&&&&&,    rC   Úeager_attention_forwardr:  .  s  € ð ‚Ø—/‘/ 4Õ'ˆä˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LàÒØ#Õ-ˆÜ—z’z ,Ó/ˆØ#Õ-ˆØÒ!Ø#Õ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$rF   c                óÄ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  /# )r7   r	  rÐ   rÏ   rÑ   r   rj   r!  )rA   s   "rC   rD   rD   P  sX   € ÷ 5&ñ 5&Ü‡|�|ð5&ä	�‰ð5&ô 
�‰ð5&ô —,‘,ð	5&ô
 ð5&ô ‡\�\ñ5&rF   c           
     ó  € VP                   R,          pV P                   R,          p^V^V,          ,          ,          pV^ 8:  d   \        RV RV RV R24      hV.V,          p\        P                  ! WRR7      p	\        P                  ! WRR7      p
\        P                  ! W(RR7      p\	        V4       Uu. uF$  p\        Wœ,          W¬,          W¼,          VR7      NK&  	  pp\        P                  ! VRR7      # u upi )a#  Applies multidimensional RoPE to inputs.

Args:
    x (`torch.Tensor`): The tensor to embed.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        If position_ids.ndim + 2 == x.ndim, then this function passes through to `apply_rotary_pos_emb()`.
        Otherwise, position_ids is used to split the inputs, x, into multiple pieces, where each piece is fed to
        `apply_rotary_pos_emb()`, and then concatenated back together.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.

Returns:
  Tensor of shape [B, L, N, H] with RoPE applied.
zEInvalid configuration: num_rotated_channels_per_dim must be > 0, got z (num_input_channels=z, ndim=Ú)rÉ   r#  rœ   )r÷   rÊ  r;   ÚsplitÚranger$  rÎ   )r	  rÐ   rÏ   rÑ   r   ÚndimÚnum_input_channelsÚnum_rotated_channels_per_dimÚsplit_sizesÚx_partsÚ	cos_partsÚ	sin_partsrÐ  Úy_partss   &&&&&         rC   Úapply_multidimensional_roperH  P  s
  € ð8 ×Ñ˜bÕ!€DØŸ™ �ÐØ#$Ð(:¸qÀ4½xÕ(HÕ#IÐ à# qÔ(ÜðØ,Ð-Ð-BÐCUÐBVð WØ�F˜!ðó
ð 	
ð 0Ð0°4Õ7€KÜ�kŠk˜!¨bÔ1€GÜ—’˜C°"Ô5€IÜ—’˜C°"Ô5€Iô �t”óñ ˆAô 	Ø�jØ•Ø•Ø'÷		
ñ ð ð ô �9Š9�W "Ô%Ð%ùòs   Â9*C=c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚGemma4VisionAttentioniˆ  ú=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# rÚ   ©r0   rk   )rA   rB   s   "€rC   rD   Ú"Gemma4VisionAttention.__annotate__‹  s    ø€ ÷ ^ñ ^Ñ1ð ^¹cñ ^rF   c                ó  <€ \         SV `  4        \        VR 4      '       d   VP                  V,          MRV n        Wn        W n        \        VRVP                  VP                  ,          4      V n
        VP                  VP                  ,          V n        RV n        V P
                  P                  V n        RV n        \!        WP                  VP                  V P                  ,          4      V n        \!        WP                  VP                  V P                  ,          4      V n        \!        WP                  VP                  V P                  ,          4      V n        \!        WP                  V P                  ,          VP                  4      V n        \+        VP                  VP,                  R7      V n        \+        VP                  VP,                  R7      V n        \+        V P                  VP,                  RR7      V n        R# )Úlayer_typesNrã   r°   F©rŽ   r�   r~  )rq   rr   ÚhasattrrP  Ú
layer_typerg   rÛ   r  r´   râ   rã   r*  r8  r3  Úattention_dropoutÚ	is_causalre   rë   rì   rí   Úo_projrŒ   rH  Úq_normÚk_normÚv_normró   s   &&&€rC   rr   ÚGemma4VisionAttention.__init__‹  sw  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!ØˆŒØ!%§¡×!>Ñ!>ˆÔØˆŒÜ+¨F×4FÑ4FÈ×HbÑHbÐei×erÑerÕHrÓsˆŒÜ+¨F×4FÑ4FÈ×HbÑHbÐei×erÑerÕHrÓsˆŒÜ+¨F×4FÑ4FÈ×HbÑHbÐei×erÑerÕHrÓsˆŒÜ+¨F×4NÑ4NÐQU×Q^ÑQ^Õ4^Ð`f×`rÑ`rÓsˆŒä#¨¯©¸V×=PÑ=PÔQˆŒÜ#¨¯©¸V×=PÑ=PÔQˆŒÜ# D§M¡M°v×7JÑ7JÐW\Ô]ˆŽrF   c                ó<  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# ©r7   rW   r  ra   NrÑ   r˜  rj   )r;   r@   Ú
LongTensorr    r"   r?   )rA   rB   s   "€rC   rD   rN  ž  s�   ø€ ÷ ,)ñ ,)á—|‘|ð,)ñ #Ÿ\™\ð,)ñ Ÿ™ tÕ+ð	,)ñ
 ×&Ñ&¨Õ-ð,)ñ Ñ+Õ,ð,)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ,)rF   c                óR  € VP                   R R p. VORNV P                  N5pVw  r‰V P                  V4      P                  V4      p
V P	                  V
4      p
\        W¨W”4      p
V
P                  ^^4      p
V P                  V4      P                  V4      pV P                  V4      p\        W¸W”4      pVP                  ^^4      pV P                  V4      P                  V4      pV P                  V4      pVP                  ^^4      p\        P                  ! V P                  P                  \        4      pT! V V
VVV3RV P                   '       d   V P"                  MRRV P$                  /VB w  rïVP&                  ! . VORN5!  P)                  4       pV P+                  V4      pWï3# )Nr2  r«  r3  rœ   )r÷   rã   rë   r  rW  rH  rˆ  rì   rX  rí   rY  r   Úget_interfacerg   Ú_attn_implementationr:  r7  rT  r3  rú   rû   rV  )r}   rW   r  ra   rÑ   r˜  Úinput_shaper  rÐ   rÏ   r   r!  r"  Úattention_interfacer)  r(  s   &&&&&,          rC   r†   ÚGemma4VisionAttention.forwardž  s£  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà&‰ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆÜ2°<ÀcÓXˆØ#×-Ñ-¨a°Ó3ˆà—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—[‘[ Ó,ˆ
Ü0°À#ÓTˆ
Ø×)Ñ)¨!¨QÓ/ˆ
à—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆØ#×-Ñ-¨a°Ó3ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð /3¯m¯m¨m�D×*Ò*Àð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(rF   )rT  rg   rã   rU  rX  rì   rÛ   rS  r8  rV  rW  rë   r3  rY  rí   ©NNN©
rH   rI   rJ   rK   rL   rr   r†   rN   rO   r‰   rŠ   s   @@rC   rJ  rJ  ˆ  s#   ù‡ € ÙG÷^ó ^÷&,)÷ ,)ò ,)rF   rJ  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma4VisionEncoderLayeriÍ  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÚ   rM  )rA   rB   s   "€rC   rD   Ú%Gemma4VisionEncoderLayer.__annotate__Î  s    ø€ ÷ 
cñ 
cÑ1ð 
c¹cñ 
crF   c                óÐ  <€ \         SV `  4        Wn        VP                  V n        W n        \        WR 7      V n        \        V4      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        R# )©rg   rÛ   ©r�   N)rq   rr   rg   r´   rÛ   rJ  r‘  rÝ  ÚmlprŒ   rH  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormró   s   &&&€rC   rr   Ú!Gemma4VisionEncoderLayer.__init__Î  sª   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ"ŒÜ.°fÔRˆŒÜ" 6Ó*ˆŒÜ,¨T×-=Ñ-=À6×CVÑCVÔWˆÔÜ(5°d×6FÑ6FÈF×L_ÑL_Ô(`ˆÔ%Ü)6°t×7GÑ7GÈV×M`ÑM`Ô)aˆÔ&Ü*7¸×8HÑ8HÈf×NaÑNaÔ*bˆÖ'rF   c                ó0  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# r\  )r;   r@   r]  r    r"   r?   r<   )rA   rB   s   "€rC   rD   ri  Ú  s•   ø€ ÷ ñ á—|‘|ðñ #Ÿ\™\ðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ Ñ+Õ,ðñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	UñrF   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRV/VB w  rV P                  V4      pWa,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pWa,           pV# )rW   r  ra   rÑ   rG   )rn  r‘  ro  rp  rm  rq  )r}   rW   r  ra   rÑ   r˜  rj  r  s   &&&&&,  rC   r†   Ú Gemma4VisionEncoderLayer.forwardÚ  s³   € ð !ˆà×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Õ0ˆàÐrF   )	rg   r´   rn  rÛ   rm  ro  rq  rp  r‘  rd  rˆ   rŠ   s   @@rC   rg  rg  Í  s    ù‡ € ÷
có 
c÷÷ ò rF   rg  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma4VisionEncoderiù  c                ó    <€ V ^8„  d   QhRS[ /# r«   rž  )rA   rB   s   "€rC   rD   Ú Gemma4VisionEncoder.__annotate__ú  s   ø€ ÷ 
ñ 
Ñ1ñ 
rF   c           
     ó  <€ \         SV `  4        Wn        VP                  V n        \        V4      V n        \        P                  ! \        V P                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        R# u upi )rk  N)rq   rr   rg   Únum_hidden_layersÚ
num_layersrî  Ú
rotary_embr   Ú
ModuleListr?  rg  Úlayers)r}   rg   Úir~   s   && €rC   rr   ÚGemma4VisionEncoder.__init__ú  se   ø€ Ü‰ÑÔØŒØ ×2Ñ2ˆŒÜ5°fÓ=ˆŒÜ—m’mÜKPÐQU×Q`ÑQ`ÔKaÓbÑKaÀaÔ%¨V×AÑKaÑbó
ˆŽùÚbs   ÁBc          
      ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[S[,          RS[/# )r7   Úinputs_embedsra   r§  Nr˜  rj   )r;   r@   r]  r    r"   r   )rA   rB   s   "€rC   rD   ry    s]   ø€ ÷ Hñ Há—|‘|ðHñ Ÿ™ðHñ "×,Ñ,¨tÕ3ð	Hñ
 Ñ+Õ,ðHñ 
!ñHrF   c           
     óâ   € \        V P                  VVR7      pTpV P                  WS4      pV P                  RV P                  P                    F  pV! V3RVRVRV/VB pK  	  \        VR7      # )zw
pixel_position_ids (torch.Tensor):
    Patch positions as (x, y) coordinates in the image as [batch, num_patches, 2].
)rg   rƒ  ra   Nra   r  rÑ   ©Úlast_hidden_state)r   rg   r}  r  r{  r   )r}   rƒ  ra   r§  r˜  rW   r  Údecoder_layers   &&&&,   rC   r†   ÚGemma4VisionEncoder.forward  s”   € ô 3Ø—;‘;Ø'Ø)ô
ˆð &ˆØ"Ÿo™o¨mÓPÐð "Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà-ðð %8ðð 0ð	ð
 ñŠMñ Jô '¸ÔGÐGrF   )rg   r  r|  r}  r„   rˆ   rŠ   s   @@rC   rw  rw  ù  s!   ù‡ € ÷
ó 
÷H÷ Hò HrF   rw  c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚGemma4TextMLPi%  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÚ   ©r/   rk   )rA   rB   s   "€rC   rD   ÚGemma4TextMLP.__annotate__&  s   ø€ ÷ 7ñ 7Ñ/ð 7¹Cñ 7rF   c                óŠ  <€ \         SV `  4        VP                  VP                  ,
          pY#u;8¬  ;'       d    ^ 8„  Mu pVP                  ;'       d    TpWn        VP                  V n        VP                  V'       d   ^M^,          V n        \        P                  ! V P                  V P                  RR7      V n
        \        P                  ! V P                  V P                  RR7      V n        \        P                  ! V P                  V P                  RR7      V n        \        VP                  ,          V n        R# )r   Frn   N)rq   rr   r{  Únum_kv_shared_layersÚuse_double_wide_mlprg   r´   rá  r   rt   râ  rã  rä  r
   rå  r_  )r}   rg   rÛ   Úfirst_kv_shared_layer_idxÚis_kv_shared_layerr�  r~   s   &&&   €rC   rr   ÚGemma4TextMLP.__init__&  sì   ø€ Ü‰ÑÔØ$*×$<Ñ$<¸v×?ZÑ?ZÕ$ZÐ!Ø&×GÔGÀaÔGÐØ$×8Ñ8×OÐOÐ=OÐØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9×BU¹QÐ[\Õ!]ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Õ6ˆŽrF   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r„   rè  ré  s   && rC   r†   ÚGemma4TextMLP.forward3  rë  rF   rì  rˆ   rŠ   s   @@rC   rŠ  rŠ  %  s   ù‡ € ÷7ó 7÷ò rF   rŠ  c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚGemma4TextRotaryEmbeddingi8  rÈ   rï  rð  c                ó    <€ V ^8„  d   QhRS[ /# r«   ©r/   )rA   rB   s   "€rC   rD   Ú&Gemma4TextRotaryEmbedding.__annotate__:  s   ø€ ÷ Uñ UÑ/ñ UrF   c           
     óÚ  <€ \         S
V `  4        VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n        / V n	        / V n
        V P                   Fò  pV P
                  P                  V,          pVf   K&  VR,          ;pR8w  d   \        V,          pMV P                  pW`P                  V&   WPP                  V&   VP                  V,          pV! WrVR7      w  r‰\        W R2\         P"                  ! VRR7      4       \        W R2\         P"                  ! VP%                  4       RR7      4       \        W R2V	4       Kô  	  R # )	Nrõ  Údefault©rS  Ú	_inv_freqFr²   Ú_original_inv_freqÚ_attention_scaling)rq   rr   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrg   ÚsortedÚsetrP  Úrope_init_fnsrõ  r÷  r   Úcompute_default_rope_parametersÚper_layer_configÚsetattrr   rv   rû  )r}   rg   rÈ   rS  Úrope_paramsrõ  rý  Úrope_configÚcurr_inv_freqÚcurr_attention_scalingr~   s   &&&       €rC   rr   Ú"Gemma4TextRotaryEmbedding.__init__9  s=  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØSUˆÔØ)+ˆŒà×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà(¨Õ5Ð5�	¸)ÔCÜ2°9Õ=‘à#×CÑC�à-9×Ñ˜zÑ*Ø)2�N‰N˜:Ñ&ð !×1Ñ1°*Õ=ˆKÙ4@ÀÐakÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó' +rF   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r7   rg   rS  Nrj   )r/   r>   r?   r;   r@   rx   )rA   rB   s   "€rC   rD   rš  [  s7   ø€ ÷ 5ñ 5Ù ð5Ù;>À½:ð5á	‰u�|‰|™UÐ"Õ	#ñ5rF   c           	     ó>  € V P                   V,          R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
r  rã   Nr°   rÊ   r  )rg   rÈ   rS  r˜  r  rŽ   r  rú  s   &&&,    rC   r§  Ú9Gemma4TextRotaryEmbedding.compute_default_rope_parametersY  sƒ   € ð& ×%Ñ% jÕ1°,Õ?ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4rF   c                óZ  € \        W R 24      p\        W R24      pVR
,          P                  VP                  ^ ,          R^4      P                  \        P
                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	\        P                  ! W™3RR7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP                  VP                  R	7      XP                  VP                  R	7      3#   + '       g   i     LH; i)rž  r   N©rË   rÈ   r	  r
  Fr  rÉ   rÊ   )Nr  Nrœ   )r  Nr  )r  r(  r÷   rÍ   r;   rx   rÈ   r  r  r>   r(   rˆ  rÎ   rÐ   rÏ   rË   )r}   r	  rÑ   rS  rú  rù  r  r  r  r  ÚembrÐ   rÏ   s   &&&&         rC   r†   Ú!Gemma4TextRotaryEmbedding.forwardt  sJ  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã:A FÆF*	)rg   rP  r¢  r£  r¦  rõ  r„   ©NN)rH   rI   rJ   rK   r'   rr   r  r§  r;   rÖ   r   r†   rN   rO   r‰   rŠ   s   @@rC   r—  r—  8  su   ù‡ € Ù�X vÔ.÷Uô Uó /ðUð> Ù�X vÔ.÷5ñ 5ó /ó ð5ð2 ‡]‚]ƒ_Øñ<ó ó ÷<ð <rF   r—  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚGemma4TextAttentioniŠ  rK  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÚ   rŒ  )rA   rB   s   "€rC   rD   Ú Gemma4TextAttention.__annotate__�  s   ø€ ÷ /
ñ /
Ñ/ð /
¹Cñ /
rF   c                ój  <€ \         SV `  4        \        VR 4      '       d   VP                  V,          MRV n        Wn        W n        V P                  R8H  V n        V P                  '       d   VP                  MRV n        VP                  V,          pVP                  V n
        VP                  ;'       d    V P                  '       * V n        VP                  VP                  ,          V n        RV n        V P
                  P"                  V n        VP$                  R8g  V n        V P
                  P(                  \+        V P
                  R^ 4      ,
          pY$u;8¬  ;'       d    ^ 8¬  Mu V n        VP                  RV pV P,                  '       * ;'       dG    V\/        V4      ^,
          VRRR
1,          P1                  VP                  V,          4      ,
          8H  V n        \4        P6                  ! VP8                  VP                  V P                  ,          VP:                  R7      V n        \?        V P                  VP@                  R7      V n!        V P,                  '       gü   \?        V P                  VP@                  R7      V n"        \?        V P                  VP@                  RR	7      V n#        \4        P6                  ! VP8                  VP                  V P                  ,          VP:                  R7      V n$        V P                  '       gI   \4        P6                  ! VP8                  VP                  V P                  ,          VP:                  R7      MRV n%        \4        P6                  ! VP                  V P                  ,          VP8                  VP:                  R7      V n&        R# )rP  NÚsliding_attentionr°   rÎ  r�  rn   rQ  Fr~  rœ   )'rq   rr   rR  rP  rS  rg   rÛ   Ú
is_slidingÚsliding_windowr¨  rã   Úattention_k_eq_vÚuse_alternative_attentionrâ   r*  r8  r3  rT  Úuse_bidirectional_attentionrU  r{  r  r’  ÚlenÚindexÚstore_full_length_kvr   rt   r´   Úattention_biasrë   rŒ   rH  rW  rX  rY  rì   rí   rV  )r}   rg   rÛ   Úlayer_configr‘  Úprev_layersr~   s   &&&   €rC   rr   ÚGemma4TextAttention.__init__�  sÍ  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŒØ"ŒØŸ/™/Ð-@Ñ@ˆŒØ7;··°˜f×3Ò3ÈDˆÔà×.Ñ.¨yÕ9ˆØ$×-Ñ-ˆŒØ)/×)@Ñ)@×)XÐ)XÈÏÉÔEXˆÔ&Ø$*×$>Ñ$>À,×BbÑBbÕ$bˆÔ!ØˆŒØ!%§¡×!>Ñ!>ˆÔØ×;Ñ;¸uÑDˆŒð %)§K¡K×$AÑ$AÄGÈDÏKÉKÐYoÐqrÓDsÕ$sÐ!Ø"+×"MÔ"MÈAÔ"MˆÔØ×(Ñ(Ð)CÐ*CÐDˆØ(,×(?Ñ(?Ô$?÷ %/ð %/ÀIÔQTÐU`ÓQaÐdeÕQeÐhsÙˆbˆDõi
ç
‰%�×"Ñ" 9Õ-Ó
.õR/ñ E/ˆÔ!ô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô $¨¯©¸6×;NÑ;NÔOˆŒð ×&×&Ð&Ü'¨D¯M©M¸v×?RÑ?RÔSˆDŒKÜ'¨¯©¸6×;NÑ;NÐ[`ÔaˆDŒKäŸ)š)Ø×"Ñ" L×$DÑ$DÀtÇ}Á}Õ$TÐ[a×[pÑ[pôˆDŒKð ×5×5Ð5ô —	’	Ø×&Ñ&¨×(HÑ(HÈ4Ï=É=Õ(XÐ_e×_tÑ_tõð ð ŒKô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽrF   c                óL  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[S[S[S[ P                  S[ P                  3,          3,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )	r7   rW   r  ra   Nr:   rV   r˜  rj   )r;   r@   r=   r>   r?   r   r    r   )rA   rB   s   "€rC   rD   rº  ¾  sŸ   ø€ ÷ =)ñ =)á—|‘|ð=)ñ #Ÿ\™\ð=)ñ Ÿ™ tÕ+ð	=)ñ
 ™s¡E©%¯,©,¹¿¹Ð*DÕ$EÐEÕFð=)ñ  �ð=)ñ Ñ-Õ.ð=)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ=)rF   c           	     óú  € VP                   R R p. VORNV P                  N5pVw  ršV P                  V4      P                  V4      pV P	                  V4      p\        W¹V
^R7      pVP                  ^^4      pV P                  '       dL   W@P                  ,          w  rÍVP                  VP                  4      pVP                  VP                  4      pM¤V P                  V4      P                  V4      pV P                  e!   V P                  V4      P                  V4      MTpV P                  V4      p\        WÉV
^R7      pVP                  ^^4      pV P                  V4      pVP                  ^^4      pVe1   V P                  '       g   VP                  WÍV P                   4      w  rÍV P"                  '       d   WÍ3W@P                  &   \$        P&                  ! V P(                  P*                  \,        4      pT! V VVVV3RV P.                  '       d   V P0                  MRRV P2                  RV P4                  /VB w  ppVP6                  ! . VORN5!  P9                  4       pV P;                  V4      pVV3# )N)r   r2  r«  r3  r¾  rœ   )r÷   rã   rë   r  rW  r$  rˆ  r’  rS  rÍ   rÈ   rì   rí   rX  rY  ÚupdaterÛ   rÄ  r   r_  rg   r`  r:  r7  rT  r3  r¾  rú   rû   rV  )r}   rW   r  ra   r:   rV   r˜  ra  r  rÐ   rÏ   r   r!  r"  rb  r)  r(  s   &&&&&&,          rC   r†   ÚGemma4TextAttention.forward¾  sW  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà&‰ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆÜ+¨L¸sÐRSÔTˆØ#×-Ñ-¨a°Ó3ˆð
 ×"×"Ð"Ø'7¿¹Õ'HÑ$ˆJà#Ÿ™ |×':Ñ':Ó;ˆJØ'Ÿ?™?¨<×+>Ñ+>Ó?‰LàŸ™ ]Ó3×8Ñ8¸ÓFˆJØLPÏKÉKÒLc˜4Ÿ;™; }Ó5×:Ñ:¸<ÔHÐisˆLàŸ™ ZÓ0ˆJÜ-¨j¸sÐRSÔTˆJØ#×-Ñ-¨a°Ó3ˆJàŸ;™; |Ó4ˆLØ'×1Ñ1°!°QÓ7ˆLàÒ&¨t×/F×/FÐ/FØ'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJØ×$×$Ð$Ø0:Ð0HÐŸ_™_Ñ-ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð /3¯m¯m¨m�D×*Ò*Àð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rF   )rT  rg   rã   rU  r’  r½  rX  rì   rÛ   rS  r8  rV  rW  rë   r3  r¾  rÄ  rÀ  rY  rí   r„   re  rŠ   s   @@rC   r¸  r¸  Š  s"   ù‡ € ÙG÷/
ó /
÷b=)÷ =)ò =)rF   r¸  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚGemma4TextExpertsiþ  z2Collection of expert weights stored as 3D tensors.c                ó    <€ V ^8„  d   QhRS[ /# r«   r™  )rA   rB   s   "€rC   rD   ÚGemma4TextExperts.__annotate__  s   ø€ ÷ 7ñ 7Ñ/ñ 7rF   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P                  4      4      V n        \        P                  ! \        P                  ! V P                  V P                  V P                  4      4      V n        \        VP                  ,          V n        R# )r7   N)rq   rr   Únum_expertsr´   Ú
hidden_dimÚmoe_intermediate_sizeÚintermediate_dimr   r•   r;   ÚemptyÚgate_up_projrä  r
   rå  r_  rc  s   &&€rC   rr   ÚGemma4TextExperts.__init__  s­   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×5Ñ5Õ6ˆŽrF   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r7   rW   Útop_k_indexÚtop_k_weightsrj   r‚   )rA   rB   s   "€rC   rD   rÐ    sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#rF   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesrÉ   Nrœ   )rœ   éþÿÿÿ)r;   Ú
zeros_likerÖ   r   r   rÌ  rÒ  r  ÚgreaterÚsumÚnonzeror¯  ru   r×  Úchunkr_  rä  Ú
index_add_rÍ   rË   )r}   rW   rÚ  rÛ  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          rC   r†   ÚGemma4TextExperts.forward  ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)r_  rä  r×  rÓ  rÕ  rÒ  re  rŠ   s   @@rC   rÎ  rÎ  þ  s!   ù‡ € á<÷7ó 7÷#÷ #ð #rF   rÎ  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGemma4TextRouteri&  c                ó    <€ V ^8„  d   QhRS[ /# r«   r™  )rA   rB   s   "€rC   rD   ÚGemma4TextRouter.__annotate__'  s   ø€ ÷ 
Mñ 
MÑ/ñ 
MrF   c                ó:  <€ \         SV `  4        Wn        VP                  V n        V P                  R,          V n        VP
                  V n        \        V P                  V P                  RR7      V n        \        P                  ! VP                  VP                  RR7      V n        \        P                  ! \        P                  ! V P                  4      4      V n        \        P                  ! \        P                  ! VP                  4      4      V n        R# )rß   Fr~  rn   Nr�   )rq   rr   rg   r´   Úscalar_root_sizerH  r�   rŒ   r8  r   rt   rÒ  Úprojr•   r;   r–   ÚscaleÚper_expert_scalerc  s   &&€rC   rr   ÚGemma4TextRouter.__init__'  sº   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ $× 0Ñ 0°$Õ 6ˆÔØ×&Ñ&ˆŒä! $×"2Ñ"2¸¿¹ÈUÔSˆŒ	Ü—I’I˜f×0Ñ0°&×2DÑ2DÈ5ÔQˆŒ	Ü—\’\¤%§*¢*¨T×-=Ñ-=Ó">Ó?ˆŒ
Ü "§¢¬U¯ZªZ¸×8JÑ8JÓ-KÓ LˆÖrF   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          /# r�   rQ  )rA   rB   s   "€rC   rD   ró  3  s5   ø€ ÷ @ñ @¡U§\¡\ð @±e¹E¿L¹LÉ%Ï,É,Ð<VÕ6Wñ @rF   c                ó¶  € V P                  V4      pWP                  ,          V P                  ,          pV P                  V4      p\        P
                  P                  VR\        P                  R7      p\        P                  ! VV P                  P                  RR7      w  rEWDP                  RRR7      ,          pW@P                  V,          ,          pW4V3# )r¯   r  )rÐ  rŽ   TrÇ  rœ   )r8  r÷  rõ  rö  r   r   r  r;   r  Útopkrg   Útop_k_expertsrá  rø  )r}   rW   Úexpert_scoresÚrouter_probabilitiesrÛ  rÚ  s   &&    rC   r†   ÚGemma4TextRouter.forward3  s¸   € ØŸ	™	 -Ó0ˆØ%¯
©
Õ2°T×5JÑ5JÕJˆàŸ	™	 -Ó0ˆä!Ÿ}™}×4Ñ4°]ÈÔRW×R_ÑR_Ð4Ó`Ðô &+§Z¢ZØ Ø�k‰k×'Ñ'Øô&
Ñ"ˆð 	×*Ñ*¨r¸4Ð*Ó@Õ@ˆð &×(=Ñ(=¸kÕ(JÕJˆà#°KÐ?Ð?rF   )rg   r�   r´   r8  rø  rö  rõ  r÷  rˆ   rŠ   s   @@rC   rñ  rñ  &  s#   ù‡ € ÷
Mó 
M÷@÷ @ð @rF   rñ  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma4TextDecoderLayeriK  c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# rÚ   )r/   r0   rk   )rA   rB   s   "€rC   rD   Ú#Gemma4TextDecoderLayer.__annotate__L  s'   ø€ ÷ hñ hÑ/Ñ2DÕDð hÑQTñ hrF   c                ó.  <€ \         SV `  4        Wn        VP                  V n        W n        \        WR 7      V n        \        W4      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n        \        P                   ! \"        P$                  ! ^4      4      V n        VP(                  V n        V P(                  '       d§   \*        VP,                  ,          V n        \        P0                  ! V P                  V P(                  RR7      V n        \        P0                  ! V P(                  V P                  RR7      V n        \        V P                  VP                  R7      V n        VP8                  V n        V P8                  '       d•   \;        V4      V n        \?        V4      V n         \        V P                  VP                  R7      V n!        \        V P                  VP                  R7      V n"        \        V P                  VP                  R7      V n#        R# R# )rk  rl  Frn   N)$rq   rr   rg   r´   rÛ   r¸  r‘  rŠ  rm  rŒ   rH  rn  ro  rp  rq  r   rv   r;   r–   Úlayer_scalarÚhidden_size_per_layer_inputr
   rå  r_  rt   Úper_layer_input_gateÚper_layer_projectionÚpost_per_layer_input_normÚenable_moe_blockrñ  ÚrouterrÎ  ÚexpertsÚpost_feedforward_layernorm_1Úpost_feedforward_layernorm_2Úpre_feedforward_layernorm_2ró   s   &&&€rC   rr   ÚGemma4TextDecoderLayer.__init__L  sÛ  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ"ŒÜ,°FÔPˆŒÜ  Ó3ˆŒÜ,¨T×-=Ñ-=À6×CVÑCVÔWˆÔÜ(5°d×6FÑ6FÈF×L_ÑL_Ô(`ˆÔ%Ü)6°t×7GÑ7GÈV×M`ÑM`Ô)aˆÔ&Ü*7¸×8HÑ8HÈf×NaÑNaÔ*bˆÔ'ÜŸIšI¤e§j¢j°£mÓ4ˆÔà+1×+MÑ+MˆÔ(Ø×+×+Ð+Ü  ×!9Ñ!9Õ:ˆDŒKÜ(*¯	ª	°$×2BÑ2BÀD×DdÑDdÐkpÔ(qˆDÔ%Ü(*¯	ª	°$×2RÑ2RÐTX×TdÑTdÐkpÔ(qˆDÔ%Ü-:¸4×;KÑ;KÐQW×QdÑQdÔ-eˆDÔ*à &× 7Ñ 7ˆÔØ× × Ð Ü*¨6Ó2ˆDŒKÜ,¨VÓ4ˆDŒLÜ0=¸d×>NÑ>NÐTZ×TgÑTgÔ0hˆDÔ-Ü0=¸d×>NÑ>NÐTZ×TgÑTgÔ0hˆDÔ-Ü/<¸T×=MÑ=MÐSY×SfÑSfÔ/gˆDÖ,ñ !rF   c                óN  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[S[S[ P                  S[ P                  3,          3,          R,          RS[ P                  RS[ P                  R,          RS[ P
                  R,          RS[R,          R	S[ P                  /# )
r7   rW   Úper_layer_inputr:   Nr  ra   rÑ   rV   rj   )r;   r@   r=   r>   r?   r]  r   )rA   rB   s   "€rC   rD   r  h  s¤   ø€ ÷ 9ñ 9á—|‘|ð9ñ Ÿ™ð9ñ ™s¡E©%¯,©,¹¿¹Ð*DÕ$EÐEÕFÈÕMð	9ñ
 #Ÿ\™\ð9ñ Ÿ™ tÕ+ð9ñ ×&Ñ&¨Õ-ð9ñ  �ð9ñ 
�‰ñ9rF   c                ón  € Tp	V P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rV P                  V4      pW‘,           pTp	V P                  V4      pV P	                  V4      pV P
                  '       d    V P                  V4      pV	P                  RV	P                  R,          4      pV P                  V4      w  r­pV P                  V4      pV P                  WþV4      pVP                  V	P                  4      pV P                  V4      pW¿,           pV P                  V4      pW‘,           pV P                  '       dW   Tp	V P                  V4      pV P!                  V4      pW,          pV P#                  V4      pV P%                  V4      pW‘,           pWP&                  ,          pV# )rW   r  ra   r:   rÑ   rV   rG   rœ   )rn  r‘  ro  rp  rm  r  r  rú   r÷   r  r  r  r  rq  r  r  r_  r	  r
  r  )r}   rW   r  r:   r  ra   rÑ   rV   r˜  rj  r  Úhidden_states_1Úhidden_states_flatrÛ  rÚ  Úhidden_states_2s   &&&&&&&&,       rC   r†   ÚGemma4TextDecoderLayer.forwardh  sÑ  € ð !ˆà×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð .ð	
ð
 &ð
ð ,ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆà× × Ð Ø"×?Ñ?ÀÓNˆOð "*×!1Ñ!1°"°h·n±nÀRÕ6HÓ!IÐØ,0¯K©KÐ8JÓ,KÑ)ˆA˜kØ"×>Ñ>Ð?QÓRˆOØ"Ÿl™l¨?ÈÓWˆOØ-×5Ñ5°h·n±nÓEˆOØ"×?Ñ?ÀÓPˆOð ,Õ=ˆMà×7Ñ7¸ÓFˆØ Õ0ˆà×+×+Ð+Ø$ˆHØ ×5Ñ5°mÓDˆMØ ŸK™K¨Ó6ˆMØ)Õ;ˆMØ ×5Ñ5°mÓDˆMØ ×:Ñ:¸=ÓIˆMØ$Õ4ˆMà×*Ñ*Õ*ˆØÐrF   )r_  rg   r  r  r´   r  rn  rÛ   r  rm  r  r	  ro  rq  r  r  r
  rp  r  r  r‘  )NNNNNNrˆ   rŠ   s   @@rC   r  r  K  s    ù‡ € ÷hó h÷89÷ 9ò 9rF   r  c                   ó^   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lV 3R lltRtVtV ;t	# )	ÚGemma4TextScaledWordEmbeddingi¤  zT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r7   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scale)rk   rx   )rA   rB   s   "€rC   rD   Ú*Gemma4TextScaledWordEmbedding.__annotate__©  s0   ø€ ÷ Rñ R¡sð R¹3ð RÉSð RÑ_dñ RrF   c                ó–   <€ \         SV `  WV4       W@n        \        P                  ! \
        P                  ! V4      R R7      V n        R# )Fr²   N)rq   rr   Úscalar_embed_scaler   rv   r;   rw   r  )r}   r  r  r  r  r~   s   &&&&&€rC   rr   Ú&Gemma4TextScaledWordEmbedding.__init__©  s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÔQˆÖrF   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r7   Ú	input_idsr‚   )rA   rB   s   "€rC   rD   r   ®  s   ø€ ÷ Sñ S¡§¡ñ SrF   c                óŠ   <€ \         SV `  V4      V P                  P                  V P                  P
                  4      ,          # r„   )rq   r†   r  rÍ   r—   rË   )r}   r%  r~   s   &&€rC   r†   Ú%Gemma4TextScaledWordEmbedding.forward®  s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÕRÐRrF   )r  r"  )r°   re  rŠ   s   @@rC   r  r  ¤  s(   ù‡ € ñ÷Rõ R÷
S÷ Só SrF   r  c                   óØ   a a€ ] tR tRt oRtRt. ROtRR.tRtRt	Rt
RtRtRtRt]P                   ! 4       V 3R l4       tR tR	 tRV3R
 lV 3R llltRV3R lR lltV3R ltRtVtV ;t# )ÚGemma4PreTrainedModeli²  ÚmodelTrV   r:   Nc                ó†	  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       dÆ   R pRpVP                  ^,          p\        P                  ! W2,          4      \        V^,
          ^4      ,          pV\        P                  ! \        P                  ! V4      V) ,          4      ,          p\        P                  ! VP                   VP#                  ^ 4      P#                  ^ 4      4       R# \        V\$        4      '       dN   \        P&                  ! VP(                  VP*                  4       \        P,                  ! VP.                  4       R# \        V\0        4      '       d”   VP2                  P5                  4        Fs  w  rxVP6                  P8                  V,          p	V! W—R7      w  r«\        P                  ! \;        W R24      V
4       \        P                  ! \;        W R24      V
4       Ku  	  R# \        V\<        4      '       d.   \        P&                  ! VP>                  VP@                  4       R# \        V\B        4      '       dC   \        P
                  ! VPD                  4       \        P
                  ! VPF                  4       R# \        V\H        4      '       d_   V P6                  PJ                  p\        PL                  ! VPN                  RVR7       \        PL                  ! VPP                  RVR7       R# \        V\R        4      '       d#   \        P
                  ! VPT                  4       R# \        V\V        4      '       d¿   VPX                  '       d­   \        P&                  ! VPZ                  \]        R4      ) 4       \        P&                  ! VP^                  \]        R4      4       \        P&                  ! VP`                  \]        R4      ) 4       \        P&                  ! VPb                  \]        R4      4       R# \        V\d        4      '       da   VP6                  Pf                  '       dC   \        P,                  ! VPh                  4       \        P
                  ! VPj                  4       R# R# R# )	r°   r±   r�  rž  rŸ  r«  )rŸ   Ústdrp   N)6rq   Ú_init_weightsr  rœ  ÚinitÚones_r¤  r©   r´   r¹   rº   r»   r;   r¼   r½   Úcopy_r¿   r¾   rØ   Ú	constant_rò   rá   Úzeros_rñ   r—  r¦  Úitemsrg   r¨  r  r  r  r"  rñ  r÷  rø  rÎ  Úinitializer_rangeÚnormal_r×  rä  r  r  re   rs   ry   rx   rz   r{   r|   ÚGemma4VisionModelÚstandardizeÚstd_biasÚ	std_scale)r}   r.  rÀ   rÁ   rÂ   rÃ   r¿   rS  rý  r«  r¬  r  r,  r~   s   &&           €rC   r-  Ú#Gemma4PreTrainedModel._init_weightsÇ  s  ø€ ä‰Ñ˜fÔ%Ü�fÔ7×8Ò8Ü�JŠJ�v×6Ñ6Ö7Ü˜Ô @×AÒAØˆMØ#ˆMØ#×/Ñ/°1Õ4ˆNÜ&*§h¢h¨}Õ/LÓ&MÔPSÐTbÐefÕTfÐhiÓPjÕ&jÐ#Ø*¬U¯YªY´u·|²|ÀNÓ7SÐWnÐVnÕ7nÓ-oÕoˆNÜ�JŠJ�v×,Ñ,¨n×.FÑ.FÀqÓ.I×.SÑ.SÐTUÓ.VÖWÜ˜Ô 4×5Ò5Ü�NŠN˜6Ÿ>™>¨6×+KÑ+KÔLÜ�KŠK˜×,Ñ,Ö-Ü˜Ô 9×:Ò:Ø,2×,@Ñ,@×,FÑ,FÖ,HÑ(�
Ø$Ÿm™m×<Ñ<¸ZÕH�Ù#/°Ô#SÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó	 -Iô
 ˜Ô =×>Ò>Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÖIÜ˜Ô 0×1Ò1Ü�JŠJ�v—|‘|Ô$Ü�JŠJ�v×.Ñ.Ö/Ü˜Ô 1×2Ò2Ø—+‘+×/Ñ/ˆCÜ�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô 6×7Ò7Ü�JŠJ�v×*Ñ*Ö+Ü˜Ô 5×6Ò6¸6×;U×;UÐ;UÜ�NŠN˜6×+Ñ+¬e°E«l¨]Ô;Ü�NŠN˜6×+Ñ+¬U°5«\Ô:Ü�NŠN˜6×,Ñ,¬u°U«|¨mÔ<Ü�NŠN˜6×,Ñ,¬e°E«lÖ;Ü˜Ô 1×2Ò2°v·}±}×7P×7PÐ7PÜ�KŠK˜Ÿ™Ô(Ü�JŠJ�v×'Ñ'Ö(ñ 8QÑ2rF   c                ó.   € V P                   P                  # r„   ©Ú
base_modelÚembed_tokens_per_layer©r}   s   &rC   Úget_per_layer_input_embeddingsÚ4Gemma4PreTrainedModel.get_per_layer_input_embeddingsð  s   € Ø�‰×5Ñ5Ð5rF   c                ó&   € WP                   n        R # r„   r<  ©r}   r1  s   &&rC   Úset_per_layer_input_embeddingsÚ4Gemma4PreTrainedModel.set_per_layer_input_embeddingsó  s   € Ø16�‰Ö.rF   c                ób   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[RS[P                  /# )r7   Únew_num_tokensNÚpad_to_multiple_ofÚmean_resizingrj   )rk   r‘   r   Ú	Embedding)rA   rB   s   "€rC   rD   Ú"Gemma4PreTrainedModel.__annotate__ö  s?   ø€ ÷ ñ á˜d�
ðñ   $�Jðñ ð	ñ
 
�‰ñrF   c                óP   <€ \         SV `  VVVR 7      pV P                  WV4       V# ))rG  rH  rI  )rq   Úresize_token_embeddingsÚ_resize_per_layer_embeddings)r}   rG  rH  rI  rƒ  r~   s   &&&& €rC   rM  Ú-Gemma4PreTrainedModel.resize_token_embeddingsö  s:   ø€ ô ™Ñ7Ø)Ø1Ø'ð 8ó 
ˆð
 	×)Ñ)¨.ÈmÔ\ØÐrF   c                óH   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[/# )r7   rG  NrH  rI  ©rk   r‘   )rA   rB   s   "€rC   rD   rK    s6   ø€ ÷ Jñ Já˜d�
ðJñ   $�JðJñ ñ	JrF   c                ó²  € V P                   V P                  P                  4       n        V P                  P                  4       P                  '       d„   V P                  4       pV P                  WAW#4      p\        VR 4      '       d   VP                  p\        WV4       VP                  VP                  P                  4       V P                  V4       R# R# )Ú_hf_hookN)Ú
vocab_sizerg   Úget_text_configÚvocab_size_per_layer_inputr  r@  Ú_get_resized_embeddingsrR  rS  r1   Úrequires_grad_r—   r”   rD  )r}   rG  rH  rI  r>  Únew_embeddings_per_layerÚhooks   &&&&   rC   rN  Ú2Gemma4PreTrainedModel._resize_per_layer_embeddings  s²   € ð DHÇ?Á?ˆ�‰×#Ñ#Ó%Ô@Ø�;‰;×&Ñ&Ó(×D×DÐDØ%)×%HÑ%HÓ%JÐ"Ø'+×'CÑ'CØ&Ð8Jó(Ð$ô Ð-¨z×:Ò:Ø-×6Ñ6�Ü"Ð#;ÔBØ$×3Ñ3Ð4J×4QÑ4Q×4_Ñ4_Ô`Ø×/Ñ/Ð0HÖIñ ErF   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r«   ©r.   )rA   rB   s   "€rC   rD   rK  ²  s   ø‡ ‚ áÑò rF   rG   )r  rg  rœ  r‹  )ÚimageÚtextÚvideoÚaudio)NNT)rH   rI   rJ   rK   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ_can_record_outputsÚinput_modalitiesr;   rÖ   r-  r@  rD  rM  rN  rM   rN   rO   r‰   rŠ   s   @@rC   r)  r)  ²  s“   ù‡ € ð  ÐØ&*Ð#òÐð $5Ð6HÐ"IÐØÐØ€NØÐà!ÐØ"&ÐØÐØ:Ðà
‡]‚]ƒ_ô&)ó ð&)òP6ò7÷õ ÷Jò J÷e … rF   r)  zAThe base Gemma 4 language model without a language modeling head.c                   óÊ   a a€ ] tR tRt oRtR]! ]^ R7      R]R]/t	V3R lV 3R llt
]]]RV3R lR	 ll4       4       4       tV3R
 lR ltRV3R lR lltV3R ltRtVtV ;t# )ÚGemma4TextModeli  Úrouter_logits)rÃ  rW   rX   c                ó    <€ V ^8„  d   QhRS[ /# r«   r™  )rA   rB   s   "€rC   rD   ÚGemma4TextModel.__annotate__!  s   ø€ ÷ .ñ .Ñ/ñ .rF   c           
     ón  <€ \         SV `  V4       VP                  V n        VP                  V n        \        VP                  VP                  V P                  V P                  P                  R ,          R7      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                   R7      V n        \%        V4      V n        RV n        \+        V P                  P,                  4      V n        VP0                  V n        V P0                  '       dÝ   \        VP2                  VP                  VP0                  ,          V P                  VP0                  R ,          R7      V n        R	V n        \        P8                  ! VP                  VP                  VP0                  ,          RR7      V n        VP                  R,          V n        \        VP0                  VP                   R7      V n        . V n         \C        V P                  4       FT  w  r4VPD                  PF                  '       g   K#  V P@                  PI                  R
 Uu. uF  pRV RV 2NK  	  up4       KV  	  V PK                  4        R# u upi u upi )rß   )r  rl  Frn   zlayers.z.self_attn.Nr�   gÍ;fž æ?)rì   rí   rX  rY  )&rq   rr   Úpad_token_idr  rT  r  r´   rg   Úembed_tokensr   r~  r?  r{  r  r  rŒ   rH  r8  r—  r}  Úgradient_checkpointingr¥  rP  Úunique_layer_typesr  rV  r>  Úper_layer_input_scalert   Úper_layer_model_projectionÚ per_layer_model_projection_scaleÚper_layer_projection_normÚ"_keys_to_ignore_on_load_unexpectedÚ	enumerater‘  r’  ÚextendÚ	post_init)r}   rg   rÛ   r€  ÚlayerÚnamer~   s   &&    €rC   rr   ÚGemma4TextModel.__init__!  s-  ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒô :Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÕQmô
ˆÔô —m’mÜHMÈf×NfÑNfÔHgÓhÑHg¸9Ô# FÖ6ÑHgÑhó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü3°FÓ;ˆŒØ&+ˆÔ#Ü"% d§k¡k×&=Ñ&=Ó">ˆÔð
 ,2×+MÑ+MˆÔ(Ø×+×+Ð+Ü*GØ×1Ñ1Ø×(Ñ(¨6×+MÑ+MÕMØ× Ñ Ø"×>Ñ>ÀÕCô	+ˆDÔ'ð *3ˆDÔ&Ü.0¯iªiØ×"Ñ"Ø×(Ñ(¨6×+MÑ+MÕMØô/ˆDÔ+ð
 5;×4FÑ4FÈÕ4LˆDÔ1Ü-:¸6×;]Ñ;]Ðci×cvÑcvÔ-wˆDÔ*ð 35ˆÔ/Ü! $§+¡+Ö.‰HˆAØ�‰×1×1Ò1Ø×7Ñ7×>Ñ>Ù@hÓiÑ@h¸�w˜q˜c ¨T¨FÓ3Ñ@hÑiöñ /ð 	�‰ÖùòI iùò@ js   Â'J-É?J2
c                ó&  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[,          R
S[/	# )r7   r%  Nra   rÑ   rV   rƒ  Úper_layer_inputsÚ	use_cacher˜  rj   )	r;   r]  r@   r   r<   r‘   r    r"   r[   )rA   rB   s   "€rC   rD   rq  T  s±   ø€ ÷ Y
ñ Y
á×#Ñ# dÕ*ðY
ñ Ÿ™ tÕ+ðY
ñ ×&Ñ&¨Õ-ð	Y
ñ
  �ðY
ñ ×(Ñ(¨4Õ/ðY
ñ  Ÿ,™,¨Õ-ðY
ñ ˜$•;ðY
ñ Ñ+Õ,ðY
ñ 
'ñY
rF   c                óÜ  € VRJ VRJ,          '       d   \        R4      hVe   Ve   \        R4      hVe   V P                  V4      pV P                  '       d'   Vf   V P                  W4      pV P	                  WV4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP                  4       M^ p	\        P                  ! VP                  ^,          VP                  R7      V	,           pVP                  ^ 4      p\        T;p
\        4      '       g/   RV P                  RVRVR	VR
V/pR\        R/ VB R\!        R/ VB /p
Tp/ pV P"                   F  pV P%                  WÃV4      WÞ&   K  	  VP'                  R\)        4       4      p\+        V P,                  RV P                  P.                   4       Fo  w  ppVe   VRRVR3,          MRpV! VV3RVRWÐP                  P0                  V,          ,          RW P                  P0                  V,          ,          R
VR	V/VB pKq  	  V P3                  V4      p\5        TTVP7                  RR4      '       d   VR7      # RR7      # )a3  
per_layer_inputs (`torch.Tensor`, *optional*):
    Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
    hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
    via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
    you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
    to reverse the main embedding, which is expensive.
Nú:You must specify exactly one of input_ids or inputs_embedsú<You cannot specify per_layer_inputs if input_ids is provided©rg   rÇ   rg   rƒ  ra   rV   rÑ   Úfull_attentionr¼  r:   r  r  Úreturn_shared_kv_statesF)r†  rV   r:   rG   )rÊ  rt  r  Úget_per_layer_inputsÚproject_per_layer_inputsr   rg   Úget_seq_lengthr;   r½   r÷   rÈ   r¾   r  r=   r   r   rv  r}  Úpopr   r|  r  r{  rP  r8  r[   Úget)r}   r%  ra   rÑ   rV   rƒ  rƒ  r„  r˜  Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrW   r  rS  r:   r€  r‡  r  s   &&&&&&&&,          rC   r†   ÚGemma4TextModel.forwardQ  s�  € ð, ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ð%5Ò%AÜÐ[Ó\Ð\àÒ Ø ×-Ñ-¨iÓ8ˆMà×+×+Ð+ØÒ'Ø#'×#<Ñ#<¸YÓ#VÐ Ø#×<Ñ<¸]Ó]Ðç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð#Ðð &ˆØ ÐØ×1Ô1ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 2ð "Ÿ:™:Ð&8¼(»*ÓEÐô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAˆ}Ø>NÒ>ZÐ.¨q°!°Q¸¨zÖ:Ð`dˆOá)ØØñ	ð "2ð	ð %8¿¹×8OÑ8OÐPQÕ8RÕ$Sð		ð
  3·;±;×3JÑ3JÈ1Õ3MÕNð	ð *ð	ð !0ð	ð ñ	ŠMñ !Xð Ÿ	™	 -Ó0ˆä,Ø+Ø+Ø17·±Ð<UÐW\×1]Ò1]Ð-ô
ð 	
ð dhô
ð 	
rF   c                ó„   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  /# ©r7   r%  Nrƒ  rj   r‚   )rA   rB   s   "€rC   rD   rq  ¯  s?   ø€ ÷ *
ñ *
©e¯l©l¸TÕ.Að *
ÑRW×R^ÑR^ÐaeÕReð *
Ñjo×jvÑjvñ *
rF   c                óÄ  € V P                   '       g   \        RV P                   24      hVf¹   \        P                  ! 4       ;_uu_ 4        VR,          V P
                  P                  R,          V P                  P                  R,          ,          8H  P                  ^R7      P                  4       R	,          p VP                  VP                  R,          4      p RRR4       V P                  V4      P                  ! . VP                  OV P                  P                  NV P                   N5!  #   \         d    \        R4      hi ; i  + '       g   i     Lx; i)
aç  Compute the token-identity component of Per-Layer Embeddings (PLE).

Looks up `input_ids` in `embed_tokens_per_layer` (a scaled embedding that multiplies
by `sqrt(hidden_size_per_layer_input)`) and reshapes the packed output from
`[batch, seq, num_hidden_layers * hidden_size_per_layer_input]` to
`[batch, seq, num_hidden_layers, hidden_size_per_layer_input]`.

If only `inputs_embeds` is provided (no `input_ids`), reverses the main embedding
to recover `input_ids` for the PLE lookup.
z}Attempting to call get_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nrß   rÉ   :Nr7   Na)  It seems like you tried to call `forward` from `inputs_embeds` without providing `input_ids`, and that the `inputs_embeds` you provided do not exactly match the embedding weights. Since Gemma4 needs to reverse the embedding to compute another embedding, make sure you provide exact `inputs_embeds`)r  r  Nr  )NNr  r  )r  r7   )r  ÚRuntimeErrorrg   r;   rÖ   rt  r—   r´   rÎ  râ  r  r÷   r>  rú   r{  )r}   r%  rƒ  s   &&&rC   r‹  Ú$Gemma4TextModel.get_per_layer_inputs¯  s:  € ð ×/×/Ð/Üð*Ø*.¯+©+¨ð8óð ð ÒÜ—’—•ð & mÕ4Ø×,Ñ,×3Ñ3Ð4DÕEÈÏÉ×H_ÑH_ÐadÕHdÕdñe÷ ‘S˜Q�S“Zß‘W“Y˜tõ%ð ðØ )§¡¨}×/BÑ/BÀ2Õ/FÓ G‘I÷ !ð$ ×*Ñ*¨9Ó5×=Ò=ð 
Ø�_‰_ð
à�K‰K×)Ñ)ð
ð ×,Ñ,ó
ð 	
øô $ô Ü&ðróð ðú÷ !—ús   ÁA/EÂ<"D5Ä5EÅEÅE	c                óv   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  /# )r7   rƒ  rƒ  Nrj   r‚   )rA   rB   s   "€rC   rD   rq  Û  s?   ø€ ÷ !Vñ !Vá—|‘|ð!Vñ  Ÿ,™,¨Õ-ð!Vñ 
�‰ñ	!VrF   c                ó~  € V P                   '       g   \        RV P                   24      hV P                  V4      V P                  ,          pVP
                  ! . VP                  RR OV P                  P                  NV P                   N5!  pV P                  V4      pVf   V# W2,           V P                  ,          # )aL  Compute the context-aware component of PLE and combine with token-identity.

Projects `inputs_embeds` through `per_layer_model_projection` (Linear), scales by
`1/sqrt(hidden_size)`, reshapes to `[batch, seq, num_layers, ple_dim]`, and normalizes
with `per_layer_projection_norm` (RMSNorm).

If `per_layer_inputs` (the token-identity component from `get_per_layer_inputs()`)
is provided, combines both: `(context_projection + token_identity) * (1/sqrt(2))`.
If `per_layer_inputs` is None (e.g. for multimodal inputs where input_ids are not
available), returns just the context projection.
z�Attempting to call project_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nrœ   )
r  r—  rg   rx  ry  rú   r÷   r{  rz  rw  )r}   rƒ  rƒ  r	  s   &&& rC   rŒ  Ú(Gemma4TextModel.project_per_layer_inputsÛ  sÌ   € ð  ×/×/Ð/Üð2Ø26·+±+°ð@óð ð
  $×>Ñ>¸}ÓMÐPT×PuÑPuÕuÐØ3×;Ò;ð  
Ø× Ñ   "Ð%ð 
à�K‰K×)Ñ)ð 
ð ×,Ñ,ó 
Ðð
  $×=Ñ=Ð>RÓSÐàÒ#Ø'Ð'à$Õ7¸4×;UÑ;UÕUÐUrF   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r«   r™  )rA   rB   s   "€rC   rD   rq    s   ø‡ ‚ áÑò rF   )r{  rt  r>  ru  r  r  r8  r  rw  rx  ry  rz  r}  rv  rT  )r_  )NNNNNNNr„   )rH   rI   rJ   rK   rl  r*   rñ  r  r¸  rk  rr   r)   r+   r#   r†   r‹  rŒ  rM   rN   rO   r‰   rŠ   s   @@rC   rn  rn    s…   ù‡ € ð !Ðà™Ð(8ÀÔBØÐ/ØÐ)ðÐ÷.ó .ð`  ØØ÷Y
ñ Y
ó ó ó  ðY
÷v*
ð *
÷X!Vò !V÷I … rF   rn  z>The base Gemma 4 language model with a language modeling head.c                   ó    a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tR	tV3R
 lV 3R llt	]
]RV3R lR ll4       4       tV3R ltRtVtV ;t# )ÚGemma4ForCausalLMiÿ  úlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrW   rU   Úkeep_full_weightr*  c                ó    <€ V ^8„  d   QhRS[ /# r«   r™  )rA   rB   s   "€rC   rD   ÚGemma4ForCausalLM.__annotate__  s   ø€ ÷ ñ Ñ/ñ rF   c                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# ©Frn   N)
rq   rr   rn  r*  rT  r   rt   r´   r   r~  rc  s   &&€rC   rr   ÚGemma4ForCausalLM.__init__  sU   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰ÖrF   c                óx  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[ P                  R,          RS[S[,          RS[	/# )r7   r%  Nra   rÑ   rV   rƒ  Úlabelsr„  Úlogits_to_keeprƒ  r˜  rj   )
r;   r]  r@   r   r<   r‘   rk   r    r"   rR   )rA   rB   s   "€rC   rD   r¤    sÛ   ø€ ÷ E
ñ E
á×#Ñ# dÕ*ðE
ñ Ÿ™ tÕ+ðE
ñ ×&Ñ&¨Õ-ð	E
ñ
  �ðE
ñ ×(Ñ(¨4Õ/ðE
ñ × Ñ  4Õ'ðE
ñ ˜$•;ðE
ñ ™eŸl™lÕ*ðE
ñ  Ÿ,™,¨Õ-ðE
ñ Ñ+Õ,ðE
ñ 
&ñE
rF   c
                ó`  € V P                   ! RRVRVRVRVRVRV	RV/V
B pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VR	VR	3,          4      pV P                  P                  eO   WàP                  P                  ,          p\        P                  ! V4      pWàP                  P                  ,          pRpVe   V P                  ! WæV P                  3/ V
B p\        VVVP                  VP                  VP                  VP                   R
7      # )a]  
per_layer_inputs (`torch.Tensor`, *optional*):
    Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
    hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
    via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
    you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
    to reverse the main embedding, which is expensive.

Example:

```python
>>> from transformers import AutoTokenizer, Gemma4ForCausalLM

>>> model = Gemma4ForCausalLM.from_pretrained("google/gemma-4-E2B-it")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-E2B-it")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```r%  ra   rÑ   rV   rƒ  rƒ  r„  Nr  )rT   rU   rV   rW   rX   r:   rG   )r*  r†  r  rk   Úslicer   rg   Úfinal_logit_softcappingr;   r  Úloss_functionrT  rR   rV   rW   rX   r:   )r}   r%  ra   rÑ   rV   rƒ  r©  r„  rª  rƒ  r˜  ÚoutputsrW   Úslice_indicesrU   rT   s   &&&&&&&&&&,     rC   r†   ÚGemma4ForCausalLM.forward  sC  € ðP 26·²ñ 	2
Øð	2
à)ð	2
ð &ð	2
ð ,ð		2
ð
 (ð	2
ð .ð	2
ð  ð	2
ð ñ	2
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ�;‰;×.Ñ.Ò:ØŸk™k×AÑAÕAˆFÜ—Z’Z Ó'ˆFØŸk™k×AÑAÕAˆFàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rF   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r«   r™  )rA   rB   s   "€rC   rD   r¤  ÿ  s   ø‡ ‚ ñ Ñò rF   )r   r*  rT  )	NNNNNNNr   N)rH   rI   rJ   rK   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planrb  rr   r$   r#   r†   rM   rN   rO   r‰   rŠ   s   @@rC   rž  rž  ÿ  sv   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€JàÐ÷ó ð Ø÷E
ñ E
ó ó ðE
÷) … rF   rž  c                óR   € V ^8„  d   QhR\         \        \        3,          R\        /# )r7   r¾  rj   )r?   rk   r   )rA   s   "rC   rD   rD   [  s"   € ÷ ñ ´´s¼C°xµð ÄXñ rF   c                ó   a € R V 3R llpV# )zD
This creates uni/bidirectional attention mask with sliding window.
c          
      óT   € V ^8„  d   QhR\         R\         R\         R\         R\        /# )r7   Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrj   rQ  )rA   s   "rC   rD   Ú2sliding_window_mask_function.<locals>.__annotate__`  s/   € ÷ &ñ &œcð &¬Sð &¼ð &Äcð &Ìdñ &rF   c                 óh   <€ S	w  rEW#,
          pV^ 8¬  Wd8  ,          pV^ 8  V) V8  ,          pWx,          # ro  rG   )
rº  r»  r¼  r½  Úleft_window_sizeÚright_window_sizeÚdistÚ	left_maskÚ
right_maskr¾  s
   &&&&     €rC   Ú
inner_maskÚ0sliding_window_mask_function.<locals>.inner_mask`  sC   ø€ Ø.<Ñ+Ðà�~ˆØ˜Q‘Y 4Ñ#:Õ;ˆ	Ø˜Q‘h D 5Ð+<Ñ#<Õ=ˆ
ØÕ%Ð%rF   rG   )r¾  rÅ  s   f rC   Úsliding_window_mask_functionrÇ  [  s   ø€ ÷
&ð &ð ÐrF   c                   ó¶   a a€ ] tR tRt oRtRtRtR]R]/t	V3R lV 3R llt
V3R	 lR
 lt]]]! RR7      RV3R lR ll4       4       4       tV3R ltRtVtV ;t# )ÚGemma4AudioModelik  znAn audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.rO  zmodel.audio_towerrW   rX   c                ó    <€ V ^8„  d   QhRS[ /# r«   r¬   )rA   rB   s   "€rC   rD   ÚGemma4AudioModel.__annotate__v  s   ø€ ÷ ñ Ñ0ñ rF   c           	     óŽ  <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        P                  ! VP                  VP                  R R7      V n        V P#                  4        R# u upi )Trn   N)rq   rr   rg   rC  Úsubsample_conv_projectionr©   Úrel_pos_encr   r~  r?  r{  r‹  r  rt   r´   Úoutput_proj_dimsÚoutput_projr~  ró   s   && €rC   rr   ÚGemma4AudioModel.__init__v  s˜   ø€ Ü‰Ñ˜Ô ØŒä)KÈFÓ)SˆÔ&Ü;¸FÓCˆÔÜ—m’mÜBGÈ×H`ÑH`ÔBaÓbÑBa°YÔ˜fÖ0ÑBaÑbó
ˆŒô Ÿ9š9 V×%7Ñ%7¸×9PÑ9PÐW[Ô\ˆÔà�‰Öùò	 cs   ÁCc                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r7   Úmask_4drj   r‚   )rA   rB   s   "€rC   rD   rË  ƒ  s#   ø€ ÷ .ñ .±e·l±lð .ÁuÇ|Á|ñ .rF   c                ó¼  € VP                   w  r#rCVP                  pV P                  P                  pV P                  P                  ^,
          pV P                  P
                  pWF,           ^,
          V,          p	W–,          p
W¤,
          p\        P                  ! V^ V^ V3RR7      pVP                  V^W–V
4      p\        P                  ! WÇV3RR7      p\        P                  ! W•R7      V,          p\        P                  ! Wg,           V,           VR7      pVR,          VR,          ,           pVR,          P                  V^RVR4      pVP                  RV4      # )zÄ
Convert a standard 4D attention mask `[batch_size, 1, seq_len, seq_len]` to the 5D blocked format
`[batch_size, 1, num_blocks, chunk_size, context_size]` expected by the chunked local attention,
F)r1  rÇ   )r  N)Nr  )NNr  Nr  rœ   )r÷   rÈ   rg   rµ   r¶   r·   rø   rù   rú   r;   r½   r(  Úgather)r}   rÓ  rü   r  rý   rÈ   rè   ré   rê   rþ   Úpadded_seq_lenÚ
pad_amountÚmask_5dÚblock_startsÚoffsetsÚ
kv_indicess   &&              rC   Ú_convert_4d_mask_to_blocked_5dÚ/Gemma4AudioModel._convert_4d_mask_to_blocked_5dƒ  s(  € ð
 %,§M¡MÑ!ˆ
�wØ—‘ˆà—[‘[×5Ñ5ˆ
ØŸ;™;×=Ñ=ÀÕAÐØ!Ÿ[™[×@Ñ@ÐàÕ*¨QÕ.°:Õ=ˆ
Ø#Õ0ˆØ#Õ-ˆ
ä—%’%˜ ! Z°°JÐ!?ÀuÔMˆØ—/‘/ *¨a°ÈÓXˆÜ—%’%˜Ð4FÐ!GÈuÔUˆä—|’| JÔ>ÀÕKˆÜ—,’,˜zÕ<Ð?QÕQÐZ`ÔaˆØ! 'Õ*¨W°WÕ-=Õ=ˆ
ØÐ 6Õ7×>Ñ>¸zÈ1ÈbÐR\Ð^`Óaˆ
à�~‰~˜b *Ó-Ð-rF   z&Encodes audio features to soft tokens.r2   c          
      ó¶   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P
                  3,          /# )r7   rO  ra   Nr˜  rj   )r;   r@   r    r"   r?   rb   )rA   rB   s   "€rC   rD   rË  ¡  s`   ø€ ÷ cñ cáŸ™ðcñ Ÿ™ tÕ+ðcñ Ñ+Õ,ð	cñ
 
‰u�|‰|™U×-Ñ-Ð-Õ	.ñcrF   c           	     óÂ  € V P                  W4      w  rEV P                  V4      p\        V P                  VV\	        V P                  P
                  ^,
          V P                  P                  34      R7      pVe   V P                  V4      pV P                  RV P                  P                    F  pV! V3RVRV/VB pK  	  V P                  V4      p\        WER7      # )r¯   )rg   rƒ  ra   Úand_mask_functionNra   r  )r†  ra   )rÍ  rÎ  r   rg   rÇ  r¶   r·   rÜ  r  r{  rÐ  r_   )r}   rO  ra   r˜  rW   Úoutput_maskr  Úencoder_layers   &&&,    rC   r†   ÚGemma4AudioModel.forwardž  sç   € ð &*×%CÑ%CÀNÓ%cÑ"ˆØ"×.Ñ.¨}Ó=Ðä2Ø—;‘;Ø'Ø&Ü:Ø—‘×3Ñ3°aÕ7¸¿¹×9\Ñ9\Ð]óô	
ˆð Ò%Ø!×@Ñ@ÀÓPˆNà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà-ðð %8ðð ñ	ŠMñ Jð ×(Ñ(¨Ó7ˆÜ%¸ÔbÐbrF   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r«   r¬   )rA   rB   s   "€rC   rD   rË  k  s   ø‡ ‚ ñ Ñò rF   )rg   r  rÐ  rÎ  rÍ  r„   )rH   rI   rJ   rK   rL   Úmain_input_namerb  r‹  rØ   rk  rr   rÜ  r)   r+   r#   r†   rM   rN   rO   r‰   rŠ   s   @@rC   rÉ  rÉ  k  sx   ù‡ € Ùxð '€OØ+ÐàÐ)ØÐ*ðÐ÷
ó ÷.ð .ð6  ØÙÐ!IÔJ÷cñ có Kó ó  ðc÷m … rF   rÉ  c                   ó˜   a a€ ] tR tRt oRtR]R]/tV3R lV 3R llt]	]
]! RR7      V3R	 lR
 l4       4       4       tV3R ltRtVtV ;t# )r6  iÁ  zThe Gemma 4 Vision Encoder.rW   rX   c                ó    <€ V ^8„  d   QhRS[ /# r«   rž  )rA   rB   s   "€rC   rD   ÚGemma4VisionModel.__annotate__Ê  s   ø€ ÷ 
ñ 
Ñ1ñ 
rF   c                óê  <€ \         SV `  V4       \        V4      V n        \	        V4      V n        \        V4      V n        V P                  P                  '       d‡   \        P                  ! \        P                  ! V P                  P                  4      4      V n        \        P                  ! \        P                  ! V P                  P                  4      4      V n        V P#                  4        R # r„   )rq   rr   rœ  Úpatch_embedderrw  Úencoderr¼  Úpoolerrg   r7  r   rv   r;   rÖ  r´   r8  r9  r~  rc  s   &&€rC   rr   ÚGemma4VisionModel.__init__Ê  s•   ø€ Ü‰Ñ˜Ô Ü7¸Ó?ˆÔÜ*¨6Ó2ˆŒÜ(¨Ó0ˆŒà�;‰;×"×"Ð"ÜŸIšI¤e§k¢k°$·+±+×2IÑ2IÓ&JÓKˆDŒMÜŸYšY¤u§{¢{°4·;±;×3JÑ3JÓ'KÓLˆDŒNà�‰ÖrF   z1Encodes image pixels to soft tokens from patches.r2   c                ój   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[/# )r7   r¶  r§  r˜  rj   )r;   r<   r]  r    r"   r   )rA   rB   s   "€rC   rD   rè  Ù  sK   ø€ ÷ )Hñ )Há×'Ñ'ð)Hñ "×,Ñ,ð)Hñ Ñ+Õ,ð	)Hñ
 
!ñ)HrF   c           	     óD  € V P                   P                  pVP                  R,          WD,          ,          pVR8H  P                  RR7      pV P	                  WV4      pV P
                  ! R	RVRV( RV/VB pV P                  VP                  VVVR7      w  ršWš,          p	V P                   P                  '       d@   W�P                  P                  4       ,
          V P                  P                  4       ,          p	V	P                  VP                  4      p	\        V	R7      # )
a°  
pixel_values (`torch.FloatTensor` or `list[torch.FloatTensor]`):
    The images to encode. Either a single `[batch, channels, height, width]` tensor
    (all images same size) or a list of `[1, channels, height, width]` tensors (different sizes).
pixel_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`):
    The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
rÉ   rƒ  ra   r§  )rW   r§  r¨  rÙ  r…  rÞ  rœ   rG   )rg   Úpooling_kernel_sizer÷   rÎ  rê  rë  rì  r†  r7  r8  rx   r9  rÍ   rË   r   )r}   r¶  r§  r˜  rð  rÙ  r¨  rƒ  rÕ  rW   Úpooler_masks   &&&,       rC   r†   ÚGemma4VisionModel.forwardÖ  s  € ð  #Ÿk™k×=Ñ=ÐØ$×*Ñ*¨2Õ.Ð3FÕ3\Õ]ˆà/°2Ñ5×:Ñ:¸rÐ:ÓBÐØ×+Ñ+¨LÐN_Ó`ˆØ—’ñ 
Ø'ð
à-Ð-ð
ð  2ð
ð ñ	
ˆð &*§[¡[Ø ×2Ñ2Ø1Ø/Ø'ð	 &1ó &
Ñ"ˆð &Õ2ˆð �;‰;×"×"Ð"Ø*¯]©]×-@Ñ-@Ó-BÕBÀdÇnÁn×FZÑFZÓF\Õ\ˆMØ%×(Ñ(¨×)<Ñ)<Ó=ˆä&¸ÔGÐGrF   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r«   rž  )rA   rB   s   "€rC   rD   rè  Á  s   ø‡ ‚ ñ Ñò rF   )rë  rê  rì  r8  r9  )rH   rI   rJ   rK   rL   rg  rJ  rk  rr   r)   r+   r#   r†   rM   rN   rO   r‰   rŠ   s   @@rC   r6  r6  Á  s\   ù‡ € Ù%ð 	Ð1ØÐ+ðÐ÷

ó 
ð  ØÙÐ!TÔU÷)Hó Vó ó  ð)H÷1 … rF   r6  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚGemma4MultimodalEmbedderi  zQEmbeds token ids or soft tokens for multimodal content into language model space.c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# )r7   Úmultimodal_configÚtext_config)r-   r0   r/   )rA   rB   s   "€rC   rD   Ú%Gemma4MultimodalEmbedder.__annotate__  s)   ø€ ÷ xñ xá,Ñ/AÕAðxñ &ñxrF   c                óR  <€ \         SV `  4        \        VR VP                  4      V n        VP
                  V n        VP                  V n        \        P                  ! V P                  V P                  RR7      V n
        \        V P                  V P                  RR7      V n        R# )rÏ  Frn   r~  N)rq   rr   r  r´   Úmultimodal_hidden_sizerH  r�   Útext_hidden_sizer   rt   Úembedding_projectionrŒ   Úembedding_pre_projection_norm)r}   r÷  rø  r~   s   &&&€rC   rr   Ú!Gemma4MultimodalEmbedder.__init__  s†   ø€ ô
 	‰ÑÔä&-Ð.?ÐASÐUf×UrÑUrÓ&sˆÔ#Ø$×1Ñ1ˆŒØ +× 7Ñ 7ˆÔÜ$&§I¢I¨d×.IÑ.IÈ4×K`ÑK`ÐglÔ$mˆÔ!Ü-:¸4×;VÑ;VÐ\`×\dÑ\dÐqvÔ-wˆÖ*rF   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r7   rƒ  rj   r‚   )rA   rB   s   "€rC   rD   rù    s#   ø€ ÷ 6ñ 6¡U§\¡\ð 6±e·l±lñ 6rF   c                óF   € V P                  V4      pV P                  V4      # )a  Embeds token ids or soft tokens for multimodal content into language model space.
Args:
    inputs_embeds: A torch.Tensor containing the soft tokens to embed.
Returns:
    A torch.Tensor of embeddings with shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
)rþ  rý  )r}   rƒ  Úembs_normeds   && rC   r†   Ú Gemma4MultimodalEmbedder.forward  s%   € ð ×8Ñ8¸ÓGˆØ×(Ñ(¨Ó5Ð5rF   )rþ  rý  r�   rû  rü  re  rŠ   s   @@rC   rõ  rõ    s#   ù‡ € Ù[÷xó x÷6÷ 6ð 6rF   rõ  c                óæ   € V ^8„  d   QhR\         R\        P                  R\        P                  R,          R\        R,          R\        P                  R,          R\        P                  R\        /# )	r7   rg   rƒ  ra   NrV   rÑ   Úblock_sequence_idsrj   )r   r;   r@   r   r=   )rA   s   "rC   rD   rD      sl   € ÷ <ñ <Üð<ä—<‘<ð<ô —L‘L 4Õ'ð<ô ˜T•\ð	<ô
 —,‘, Õ%ð<ô Ÿ™ð<ô 
ñ<rF   c           
     óv  € RV RVRVRVRV/p\        R/ VB p\        VRR4      '       d.   RVP                  9   d   VP                  P                  R4      pM^ p\	        R/ VBR	V/B w  p	  r«r¬V	'       d   TpM\        WRW¼4      p\        R/ VBR
\        V4      R\        V P                  4      R	V/B pRVRV/# )aw  Create full_attention and sliding_attention masks with correct composition.

For global (full attention) layers:  causal only (no bidirectional)
For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))

Unlike Gemma 3 (which applies bidirectional attention on all layers), Gemma 4
explicitly disables bidirectional attention on global attention layers.
rg   rƒ  ra   rV   rÑ   r½  NTrÛ   Úor_mask_functionrà  r‰  r¼  rG   )	r   r  r½  rÃ  r   r   r   r   r¾  )rg   rƒ  ra   rV   rÑ   r  r’  Ú	full_maskÚsliding_layer_idxÚ
early_exitr  Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks   &&&&&&         rC   Úcreate_masks_for_vision_modelr     s  € ð" 	�&Ø˜Ø˜.Ø˜?Ø˜ð€Kô #Ñ1 [Ñ1€Iô ˆ ¨d×3Ò3¸À×@ZÑ@ZÔ8ZØ+×6Ñ6×<Ñ<¸TÓBÑàÐô 4Nñ 4Ø
ñ4à#ò4Ñ0€J��1�a A÷ Ø$6Ñ!ä$@Ø°	ó%
Ð!ô &ñ Ø
ñä*Ð+DÓEðô 1°×1FÑ1FÓGðð $ò	€Lð 	˜)Ø˜\ðð rF   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r7   Úmm_token_type_idsrÈ   rj   )r;   r@   rÈ   )rA   s   "rC   rD   rD   _  s0   € ÷ 	ñ 	´u·|±|ð 	ÌUÏ\É\ð 	Ô^c×^jÑ^jñ 	rF   c                 ó  € V P                  V4      p V ^8H  V ^8H  ,          p\        P                  ! V^RR7      pRVR&   W#( ,          p\        P                  ! VP	                  4       ^R7      ^,
          p\        P
                  ! W%R4      pV# )r¯   )ÚshiftsÚdimsFrÉ   rœ   r¬  )rÍ   r;   ÚrollÚcumsumrk   r¯  )r  rÈ   Ú	is_visionÚis_prev_visionÚnew_vision_startsÚvision_group_idsr  s   &&     rC   Úget_block_sequence_ids_for_maskr  _  s†   € Ø)×,Ñ,¨VÓ4Ðà" aÑ'Ð,=ÀÑ,BÕC€IÜ—Z’Z 	°!¸"Ô=€NØ"€N�6ÑØ! OÕ3ÐÜ—|’|Ð$5×$9Ñ$9Ó$;ÀÔCÀaÕGÐÜŸš YÀ"ÓEÐØÐrF   zŒ
    The base Gemma 4 model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c                   ó<  a a€ ] tR tRt oRtV3R lV 3R llt]]! RR7      RV3R lR ll4       4       tRV3R	 lR
 llt	]
]]RV3R lR ll4       4       4       tR tR t]]! RR7      V3R lR l4       4       t]]! RR7      RV3R lR ll4       4       tRtVtV ;t# )ÚGemma4Modelik  Fc                ó    <€ V ^8„  d   QhRS[ /# r«   r]  )rA   rB   s   "€rC   rD   ÚGemma4Model.__annotate__u  s   ø€ ÷ ñ ™|ñ rF   c                ó¨  <€ \         SV `  V4       VP                  e!   \        P                  ! VP                  4      MR V n        VP                  P                  V n        \        P                  ! VP                  R7      pW n        VP                  P                  V n	        VP                  e!   \        P                  ! VP                  4      MR V n        VP                  e!   \        VP                  VP                  4      MR V n        VP                  e!   \        VP                  VP                  4      MR V n        V P                  4        R # )Nrˆ  )rq   rr   Úvision_configr,   Úfrom_configÚvision_towerrø  rT  Úlanguage_modelrV  Úaudio_configÚaudio_towerrõ  Úembed_visionÚembed_audior~  )r}   rg   r$  r~   s   && €rC   rr   ÚGemma4Model.__init__u  s  ø€ Ü‰Ñ˜Ô ØKQ×K_ÑK_ÒKkœI×1Ò1°&×2FÑ2FÔGÐquˆÔØ ×,Ñ,×7Ñ7ˆŒä"×.Ò.°f×6HÑ6HÔIˆØ,ÔØ*0×*<Ñ*<×*WÑ*WˆÔ'ØIO×I\ÑI\ÒIhœ9×0Ò0°×1DÑ1DÔEÐnrˆÔð ×#Ñ#Ò/ô % V×%9Ñ%9¸6×;MÑ;MÔNàð 	Ôð ×"Ñ"Ò.ô % V×%8Ñ%8¸&×:LÑ:LÔMàð 	Ôð
 	�‰ÖrF   zOProjects the last hidden state from the vision model into language model space.r2   c                óx   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[/# )r7   r¶  Úimage_position_idsNr˜  rj   ©r;   r<   r]  r    r"   r   )rA   rB   s   "€rC   rD   r  Œ  sJ   ø€ ÷ ñ á×'Ñ'ðñ "×,Ñ,¨tÕ3ðñ Ñ+Õ,ð	ñ
 
$ñrF   c                óü  € V P                   ! RRVRV/VB pVP                  pV P                  VR7      pVP                  R,          V P                  P
                  P                  ^,          ,          p\        VP                  ^,          V,          R,          4      ^,          pVR8g  P                  RR7      p	V	P                  RR7      V,          P                  4       p
\        P                  ! Wj4      Vn        V# )	zÅ
image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
    The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
r¶  r§  ©rƒ  rß   rÉ   rG   rÞ  rœ   )r#  r†  r'  r÷   rg   r!  rð  rk   rÎ  rá  Útolistr;   r>  Úpooler_output)r}   r¶  r+  r˜  Úvision_outputsr†  r0  rÙ  rÑ  Únon_pad_maskrC  s   &&&,       rC   Úget_image_featuresÚGemma4Model.get_image_featuresŠ  sñ   € ð ×*Ò*ñ 
Ø%ð
à1ð
ð ñ
ˆð
 +×<Ñ<ÐØ×)Ñ)Ð8IÐ)ÓJˆà$×*Ñ*¨2Õ.°4·;±;×3LÑ3L×3`Ñ3`ÐbcÕ3cÕdˆÜÐ+×1Ñ1°!Õ4¸ÕEÈ#ÕMÓNÐRSÕSˆ	Ø*¨bÑ0×5Ñ5¸"Ð5Ó=ˆØ#×'Ñ'¨BÐ'Ó/°9Õ<×DÑDÓFˆÜ',§{¢{°=Ó'NˆÔ$ØÐrF   c          	      óÆ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[S[ P                  S[ P                  S[ P                  3,          /# r•  )r;   r]  r<   r?   rb   )rA   rB   s   "€rC   rD   r  ¥  sb   ø€ ÷ +Jñ +Já×#Ñ# dÕ*ð+Jñ ×(Ñ(¨4Õ/ð+Jñ 
‰u×Ñ¡×!1Ñ!1±5×3CÑ3CÐCÕ	Dñ	+JrF   c           
     ó.  € VeK   WP                   P                  8H  pWP                   P                  8H  pWP                   P                  8H  pEMDVV P	                  4       ! \
        P                  ! RV P                   P                  \
        P                  VP                  R7      4      8H  P                  R4      pVV P	                  4       ! \
        P                  ! RV P                   P                  \
        P                  VP                  R7      4      8H  P                  R4      pVV P	                  4       ! \
        P                  ! RV P                   P                  \
        P                  VP                  R7      4      8H  P                  R4      pW4V3# )a;  
Obtains mask for multimodal placeholders (replaced by soft tokens) and hard text tokens.

Masks will be obtained from `mm_token_type_ids`, `input_ids`, or `inputs_embeds` as available and in that
precedence order. If passing `input_ids` or `inputs_embeds`, the image mask will be derived using
`config.image_token_id`. Same goes for audio and video masks

Args:
    input_ids: A tensor containing the hard token IDs from the text tokenizer.
    inputs_embeds: A tensor containing the embeddings for all hard text tokens.

Returns:
    image_mask, video_mask, audio_mask
r³  rG   rœ   )
rg   Úimage_token_idÚvideo_token_idÚaudio_token_idÚget_input_embeddingsr;   ÚfullrÍ  rÈ   rÎ  )r}   r%  rƒ  Úspecial_image_maskÚspecial_video_maskÚspecial_audio_masks   &&&   rC   Úget_placeholder_maskÚ Gemma4Model.get_placeholder_mask¥  sG  € ð& Ò Ø!*¯k©k×.HÑ.HÑ!HÐØ!*¯k©k×.HÑ.HÑ!HÐØ!*¯k©k×.HÑ.HÑ!HÒð Ø×,Ñ,Ô.Ü—J’J˜r 4§;¡;×#=Ñ#=ÄUÇZÁZÐXe×XlÑXlÔmóñ÷ ‰c�"‹gð ð Ø×,Ñ,Ô.Ü—J’J˜r 4§;¡;×#=Ñ#=ÄUÇZÁZÐXe×XlÑXlÔmóñ÷ ‰c�"‹gð ð Ø×,Ñ,Ô.Ü—J’J˜r 4§;¡;×#=Ñ#=ÄUÇZÁZÐXe×XlÑXlÔmóñ÷ ‰c�"‹gð ð "Ð7IÐIÐIrF   c                 ó>  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )r7   r%  Nr¶  Úpixel_values_videosrO  ra   rP  rÑ   rV   r  rƒ  r„  r+  Úvideo_position_idsrƒ  r˜  rj   )	r;   r]  r<   r@   r   r‘   r    r"   r5   )rA   rB   s   "€rC   rD   r  Õ  sI  ø€ ÷ d
ñ d
á×#Ñ# dÕ*ðd
ñ ×'Ñ'¨$Õ.ðd
ñ #×.Ñ.°Õ5ð	d
ñ
 ×)Ñ)¨DÕ0ðd
ñ Ÿ™ tÕ+ðd
ñ #Ÿ\™\¨DÕ0ðd
ñ ×&Ñ&¨Õ-ðd
ñ  �ðd
ñ !×+Ñ+¨dÕ2ðd
ñ ×(Ñ(¨4Õ/ðd
ñ ˜$•;ðd
ñ "×,Ñ,¨tÕ3ðd
ñ "×,Ñ,¨tÕ3ðd
ñ  Ÿ,™,¨Õ-ðd
ñ  Ñ+Õ,ð!d
ñ" 
#ñ#d
rF   c                ó„  € VRJ V
RJ,          '       d   \        R4      hVe   Ve   \        R4      hV P                  W4      w  pppVV,          V,          pRpV
f]   VP                  4       p\        P                  ! VV P
                  P                  P                  V4      pV P                  4       ! V4      p
VfÙ   V P
                  P                  4       P                  '       d¯   V P                  P                  P                  V P
                  P                  P                  R3,          pVP                  V
P                  4      p\        P                  ! VR,          VP!                  ^^R4      V
4      pV P                  P#                  VV4      pVEe0   V P%                  W,RR7      P&                  p\        P(                  ! V^ R7      P                  V
P                  V
P*                  4      pVP-                  4       pVP/                  R4      P1                  V
4      P                  V
P                  4      p\3        V
V,          P5                  4       VP5                  4       8H  RV R	VP6                  ^ ,           24       V
P9                  VP                  V
P                  4      VP                  V
P                  4      4      p
VEe0   V P;                  W=RR7      P&                  p\        P(                  ! V^ R7      P                  V
P                  V
P*                  4      pVP-                  4       pVP/                  R4      P1                  V
4      P                  V
P                  4      p\3        V
V,          P5                  4       VP5                  4       8H  R
V R	VP6                  ^ ,           24       V
P9                  VP                  V
P                  4      VP                  V
P                  4      4      p
VEeL   VEeG   V P=                  WFRR7      pVP&                  pVP>                  pVVP                  VP                  4      ,          pVP-                  4       pVP/                  R4      P1                  V
4      P                  V
P                  4      p\3        V
V,          P5                  4       VP5                  4       8H  RV R	VP6                  ^ ,          VP6                  ^,          ,           24       V
P9                  VP                  V
P                  4      VP                  V
P                  V
P*                  4      4      p
Vfb   Ve   VPA                  4       M^ p\        PB                  ! V
P6                  ^,          V
P                  R7      V,           pVP/                  ^ 4      p\E        T;p \F        4      '       g‰   RV P
                  P                  4       RV
RVRVRV/p!V P
                  P                  4       p"V"PH                  R8H  p#V#'       d)   V	e%   \K        WšP                  R7      p$\M        RRV$/V!B p M\O        R/ V!B p V P                  ! RRVRV RVRVRV
RVRR/VB p%\Q        V%PR                  V%PT                  V%PV                  V%PX                  Ve   XMRVe   XMRV%PZ                  R7      # )áÒ  
input_features_mask (`torch.FloatTensor` of shape `(num_images, seq_length)`):
    The attention mask for the input audio.
image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
    2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
    Passed through to the vision encoder for positional embedding computation.
video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
    2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
    Passed through to the vision encoder for positional embedding computation.
per_layer_inputs (`torch.Tensor`, *optional*):
    Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
    hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
    via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
    you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
    to reverse the main embedding, which is expensive.
Nr†  r‡  r  T)Úreturn_dictrÉ   z6Image features and image tokens do not match, tokens: z, features: z6Video features and video tokens do not match, tokens: z6Audio features and audio tokens do not match, tokens: rÇ   rg   rƒ  ra   rV   rÑ   Úvisionr  rƒ  r„  rF  )r†  rV   rW   rX   r8   r9   r:   rÌ   rœ   rG   ).rÊ  r?  rû  r;   r¯  rg   rø  rs  r:  rU  r  r$  rt  r—   rÍ   rÈ   r  r‹  r3  r0  rÎ   rË   rá  r¾   Ú	expand_asr&   Únumelr÷   Úmasked_scatterÚget_video_featuresÚget_audio_featuresra   r�  r½   r  r=   rÁ  r  r  r   r5   r†  rV   rW   rX   r:   )&r}   r%  r¶  rB  rO  ra   rP  rÑ   rV   r  rƒ  r„  r+  rC  rƒ  r˜  Ú
image_maskÚ
video_maskÚ
audio_maskÚmultimodal_maskÚllm_input_idsÚpad_embeddingÚllm_inputs_embedsÚimage_featuresÚn_image_tokensÚvideo_featuresÚn_video_tokensÚaudio_outputÚaudio_featuresÚaudio_mask_from_encoderÚn_audio_tokensr�  r‘  r’  rø  Ú	use_bidirr  r¯  s&   &&&&&&&&&&&&&&&,                      rC   r†   ÚGemma4Model.forwardÒ  sV  € ðJ ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ð%5Ò%AÜÐ[Ó\Ð\à-1×-FÑ-FÀyÓ-`Ñ*ˆ
�J 
Ø$ zÕ1°JÕ>ˆð ˆØÒ Ø%ŸO™OÓ-ˆMÜ!ŸKšK¨¸¿¹×9PÑ9P×9]Ñ9]Ð_lÓmˆMØ ×5Ñ5Ô7¸ÓFˆMàÒ#¨¯©×(CÑ(CÓ(E×(a×(aÐ(aØ ×/Ñ/×<Ñ<×CÑCÀDÇKÁK×D[ÑD[×DhÑDhÐjkÐDkÕlˆMØ-×0Ñ0°×1EÑ1EÓFˆOÜ %§¢¨O¸IÕ,FÈ×HZÑHZÐ[\Ð^_ÐacÓHdÐfsÓ tÐØ#×2Ñ2×GÑGÈÐWhÓiÐð Ó#Ø!×4Ñ4°\ÐcgÐ4Óh×vÑvˆNÜ"ŸYšY ~¸1Ô=×@Ñ@À×AUÑAUÐWd×WjÑWjÓkˆNð (Ÿ^™^Ó-ˆNØ#×-Ñ-¨bÓ1×;Ñ;¸MÓJ×MÑMÈm×NbÑNbÓcˆJÜ"Ø˜jÕ)×/Ñ/Ó1°^×5IÑ5IÓ5KÑKØHÈÐHXð YØ"×(Ñ(¨Õ+Ð,ð.ôð *×8Ñ8Ø—‘˜m×2Ñ2Ó3°^×5FÑ5FÀ}×G[ÑG[Ó5\óˆMð Ó*Ø!×4Ñ4Ø#ÀTð 5ó ç‰mð ô #ŸYšY ~¸1Ô=×@Ñ@À×AUÑAUÐWd×WjÑWjÓkˆNð (Ÿ^™^Ó-ˆNØ#×-Ñ-¨bÓ1×;Ñ;¸MÓJ×MÑMÈm×NbÑNbÓcˆJÜ"Ø˜jÕ)×/Ñ/Ó1°^×5IÑ5IÓ5KÑKØHÈÐHXð YØ"×(Ñ(¨Õ+Ð,ð.ôð *×8Ñ8Ø—‘˜m×2Ñ2Ó3°^×5FÑ5FÀ}×G[ÑG[Ó5\óˆMð
 Ó%Ð*=Ó*IØ×2Ñ2°>ÐdhÐ2ÓiˆLØ)×7Ñ7ˆNØ&2×&AÑ&AÐ#ð
 ,Ð,C×,FÑ,FÀ~×G\ÑG\Ó,]Õ^ˆNà'Ÿ^™^Ó-ˆNØ#×-Ñ-¨bÓ1×;Ñ;¸MÓJ×MÑMÈm×NbÑNbÓcˆJÜ"Ø˜jÕ)×/Ñ/Ó1°^×5IÑ5IÓ5KÑKØHÈÐHXð YØ"×(Ñ(¨Õ+¨n×.BÑ.BÀ1Õ.EÕEÐFðHôð *×8Ñ8Ø—‘˜m×2Ñ2Ó3°^×5FÑ5FÀ}×G[ÑG[Ð]j×]pÑ]pÓ5qóˆMð
 ÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFà˜$Ÿ+™+×5Ñ5Ó7Ø Ø  .Ø! ?Ø ðˆKð Ÿ+™+×5Ñ5Ó7ˆKØ#×?Ñ?À8ÑKˆIçÐ.Ò:Ü%DÐEV×_sÑ_sÔ%tÐ"Ü&Cñ 'Ø'9ð'à!ñ'Ñ#ô '@Ñ&NÀ+Ñ&NÐ#à×%Ò%ñ 	
Ø-ð	
à.ð	
ð &ð	
ð ,ð		
ð
 (ð	
ð  ð	
ð ð	
ð ñ	
ˆô )Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ò2J¡ÐPTØ2@Ò2L¡ÐRVØ$×5Ñ5ô
ð 	
rF   c                ó.   € V P                   P                  # r„   ©r$  r>  r?  s   &rC   r@  Ú*Gemma4Model.get_per_layer_input_embeddings{	  s   € Ø×"Ñ"×9Ñ9Ð9rF   c                ó&   € WP                   n        R # r„   r_  rC  s   &&rC   rD  Ú*Gemma4Model.set_per_layer_input_embeddings~	  s   € Ø5:×ÑÖ2rF   zPProjects the last hidden state from the audio encoder into language model space.c          	      óz   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[S[,          /# )r7   rO  rP  r˜  rj   )r;   r@   r    r"   r?   r_   )rA   rB   s   "€rC   rD   r  ƒ	  sG   ø€ ÷ ñ áŸ™ðñ #Ÿ\™\ðñ Ñ+Õ,ð	ñ
 
Ñ'Õ	'ñrF   c                ó¤   € V P                   f   \        R4      hV P                   ! W3RR/VB pV P                  VP                  R7      Vn        V# )a  
input_features (`torch.FloatTensor` of shape `(num_images, seq_length, num_features)`):
    The tensors corresponding to the input audio.
input_features_mask (`torch.FloatTensor` of shape `(num_images, seq_length)`):
    The attention mask for the input audio.
z•Audio features were requested, but the model was initialized without an audio_config. Cannot process audio without an audio tower and audio embedder.rF  Tr.  )r&  rÊ  r(  r†  r0  )r}   rO  rP  r˜  Úaudio_outputss   &&&, rC   rL  ÚGemma4Model.get_audio_features�	  sc   € ð ×ÑÒ#ÜðRóð ð
 ×(Ò(¨ÑiÐZ^ÐiÐbhÑiˆØ&*×&6Ñ&6À]×EdÑEdÐ&6Ó&eˆÔ#àÐrF   zQProjects the last hidden state from the vision encoder into language model space.c                óx   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[/# )r7   rB  rC  Nr˜  rj   r,  )rA   rB   s   "€rC   rD   r  œ	  sJ   ø€ ÷ ñ á"×.Ñ.ðñ "×,Ñ,¨tÕ3ðñ Ñ+Õ,ð	ñ
 
$ñrF   c           
     ó@  € VP                  ^ ^4      pV P                  ! RRVRVP                  ^ ^4      /VB pVP                  pV P                  VR7      pVP                  R,          V P
                  P                  P                  ^,          ,          p\        VP                  R,          V,          R,          4      ^,          pVR8g  P                  RR7      p	V	P                  R	R7      V,          P                  4       p
\        P                  ! Wj4      Vn        V# )
a  
video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
    2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
    Passed through to the vision encoder for positional embedding computation.
r¶  r§  r.  rß   rÉ   rG   rÞ  rœ   )rÞ  rœ   )Úflattenr#  r†  r'  r÷   rg   r!  rð  rk   rÎ  rá  r/  r;   r>  r0  )r}   rB  rC  r˜  r1  r†  r0  rÙ  rÑ  r2  rC  s   &&&,       rC   rK  ÚGemma4Model.get_video_featuresš	  s  € ð 2×9Ñ9¸!¸QÓ?ÐØ×*Ò*ñ 
Ø,ð
à1×9Ñ9¸!¸QÓ?ð
ð ñ
ˆð
 +×<Ñ<ÐØ×)Ñ)Ð8IÐ)ÓJˆà+×1Ñ1°"Õ5¸$¿+¹+×:SÑ:S×:gÑ:gÐijÕ:jÕkˆÜÐ,×2Ñ2°2Õ6¸-ÕGÈCÕOÓPÐTUÕUˆ	Ø*¨bÑ0×5Ñ5¸"Ð5Ó=ˆØ#×'Ñ'¨HÐ'Ó5¸ÕB×JÑJÓLˆä',§{¢{°=Ó'NˆÔ$ØÐrF   )r&  r(  r'  r$  r#  rT  rV  r„   r¶  )NNNNNNNNNNNNNN)rH   rI   rJ   rK   Úaccepts_loss_kwargsrr   r$   r#   r3  r?  r)   r†   r@  rD  rL  rK  rN   rO   r‰   rŠ   s   @@rC   r  r  k  sÓ   ù‡ € ð  Ð÷ó ð* ÙÐ!rÔs÷ñ ó tó ð÷2+Jò +JðZ  ØØ÷d
ñ d
ó ó ó  ðd
òL:ò;ð ÙÐ!sÔt÷ó uó ðð. ÙÐ!tÔu÷ñ ó vó ÷ð rF   r  z…
    The base Gemma 4 model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c                   óÔ   a a€ ] tR tRt oRR/tRtRtV3R lV 3R llt]RV3R lR	 ll4       t	]
]RV3R
 lR ll4       4       tR tR t]RV3R lR ll4       tRV 3R lltRtVtV ;t# )ÚGemma4ForConditionalGenerationi¹	  rŸ  z(model.language_model.embed_tokens.weightFr*  c                ó    <€ V ^8„  d   QhRS[ /# r«   r]  )rA   rB   s   "€rC   rD   Ú+Gemma4ForConditionalGeneration.__annotate__Ä	  s   ø€ ÷ ñ ™|ñ rF   c                óò   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  P                  VP                  P                  R R7      V n	        V P                  4        R# r¦  )rq   rr   r  r*  r   rt   rø  r´   rT  r   r~  rc  s   &&€rC   rr   Ú'Gemma4ForConditionalGeneration.__init__Ä	  sS   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ü—y’y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰ÖrF   c                ór   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          /# )r7   r¶  r+  Nr˜  )r;   r<   r]  r    r"   )rA   rB   s   "€rC   rD   ro  Ë	  sE   ø€ ÷ Yñ Yá×'Ñ'ðYñ "×,Ñ,¨tÕ3ðYñ Ñ+Õ,ñ	YrF   c                ó<   € V P                   P                  ! W3/ VB # )a  
image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
    2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
    Passed through to the vision encoder for positional embedding computation.
)r*  r3  )r}   r¶  r+  r˜  s   &&&,rC   r3  Ú1Gemma4ForConditionalGeneration.get_image_featuresÊ	  s   € ð �z‰z×,Ò,¨\ÑXÐQWÑXÐXrF   c          $      ó�  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  ,          RS[ P                  R,          RS[S[,          RS[	/# )r7   r%  Nr¶  rB  rO  ra   rP  rÑ   r+  rC  rV   r  rƒ  r©  r„  rª  rƒ  r˜  rj   )
r;   r]  r<   r@   r   r‘   rk   r    r"   rR   )rA   rB   s   "€rC   rD   ro  Ú	  ss  ø€ ÷ N
ñ N
á×#Ñ# dÕ*ðN
ñ ×'Ñ'¨$Õ.ðN
ñ #×.Ñ.°Õ5ð	N
ñ
 ×)Ñ)¨DÕ0ðN
ñ Ÿ™ tÕ+ðN
ñ #Ÿ\™\¨DÕ0ðN
ñ ×&Ñ&¨Õ-ðN
ñ "×,Ñ,¨tÕ3ðN
ñ "×,Ñ,¨tÕ3ðN
ñ  �ðN
ñ !×+Ñ+¨dÕ2ðN
ñ ×(Ñ(¨4Õ/ðN
ñ × Ñ  4Õ'ðN
ñ ˜$•;ðN
ñ  ™eŸl™lÕ*ð!N
ñ"  Ÿ,™,¨Õ-ð#N
ñ$ Ñ+Õ,ð%N
ñ& 
&ñ'N
rF   c                óÔ  € V P                   ! R/ RVbRVbRVbRVbRVbRVbRVbRV
bR	VbR
VbRVbRVbRVbRVbRV	bRRbVB pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  4       P                  ;pe)   VV,          p\        P                  ! V4      pVV,          pRpVe7   V P                  ! VWÐP                  P                  4       P                  3/ VB p\        VVVP                  VP                  VP                   VP"                  VP$                  VP&                  R7      # )rE  r%  r¶  rB  rO  ra   rP  rÑ   rV   r  rƒ  rƒ  r©  r„  r+  rC  rF  TNr  )rT   rU   rV   rW   rX   r8   r9   r:   rG   )r*  r†  r  rk   r¬  r   rg   rU  r­  r;   r  r®  rT  rR   rV   rW   rX   r8   r9   r:   )r}   r%  r¶  rB  rO  ra   rP  rÑ   r+  rC  rV   r  rƒ  r©  r„  rª  rƒ  r˜  r¯  rW   r°  rU   r­  rT   s   &&&&&&&&&&&&&&&&&,      rC   r†   Ú&Gemma4ForConditionalGeneration.forwardØ	  s¹  € ðL —*’*ò 
Ùð
á%ð
ñ !4ð
ñ *ð	
ñ
 *ð
ñ !4ð
ñ &ð
ñ ,ð
ñ 0ð
ñ (ð
ñ .ð
ñ ð
ñ  ð
ñ  2ð
ñ  2ð
ñ  Øñ#
ˆð(  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ'+§{¡{×'BÑ'BÓ'D×'\Ñ'\Ð\Ð#ÒiØÐ5Õ5ˆFÜ—Z’Z Ó'ˆFØÐ5Õ5ˆFàˆØÒØ×%Ò% f¨f·k±k×6QÑ6QÓ6S×6^Ñ6^ÑiÐbhÑiˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;Ø '× ;Ñ ;Ø$×5Ñ5ô	
ð 		
rF   c                ó6   € V P                   P                  4       # r„   )r*  r@  r?  s   &rC   r@  Ú=Gemma4ForConditionalGeneration.get_per_layer_input_embeddings*
  s   € Ø�z‰z×8Ñ8Ó:Ð:rF   c                ó<   € V P                   P                  V4       R # r„   )r*  rD  rC  s   &&rC   rD  Ú=Gemma4ForConditionalGeneration.set_per_layer_input_embeddings-
  s   € Ø�
‰
×1Ñ1°%Ö8rF   c                óà   <€ V ^8„  d   QhRS[ RS[P                  RS[P                  R,          RS[R,          RS[P                  R,          RS[P                  R,          RS[R,          R	S[/# )
r7   rg   rƒ  ra   NrV   rÑ   r  Úis_first_iterationrj   )r   r;   r@   r   r‘   r=   )rA   rB   s   "€rC   rD   ro  1
  s€   ø€ ÷ 8ñ 8Ù ð8á—|‘|ð8ñ Ÿ™ tÕ+ð8ñ  �ð	8ñ
 —l‘l TÕ)ð8ñ !Ÿ<™<¨$Õ.ð8ñ ! 4�Kð8ñ 
ñ8rF   c           
     óê   € R V P                  4       RVRVRVRV/pV P                  4       p	\        V	RR4      R8H  p
V
'       d(   Ve$   \        WQP                  R7      p\	        R
R	V/VB # \        R
/ VB # )rg   rƒ  ra   rV   rÑ   rÁ  NrG  rÇ   r  rG   )rU  r  r  rÈ   r  r   )rg   rƒ  ra   rV   rÑ   r  r}  r˜  r’  rø  r\  r  s   &&&&&&&,    rC   r   Ú8Gemma4ForConditionalGeneration.create_masks_for_generate0
  sš   € ð �f×,Ñ,Ó.Ø˜]Ø˜nØ˜Ø˜Lð
ˆð ×,Ñ,Ó.ˆÜ˜KÐ)FÈÓMÐQYÑYˆ	çÐ*Ò6Ü!@ÐAR×[oÑ[oÔ!pÐÜ0ñ Ø#5ðàñð ô
 )Ñ7¨;Ñ7Ð7rF   c                ó�   <€ \         SV `  ! V3R VRV/VB pV'       g   V'       d   RVR&   V'       g   VP                  RR4      pV# )r„  r}  Nr  rƒ  )rq   Úprepare_inputs_for_generationrŽ  )r}   r%  r„  r}  r˜  Úmodel_inputsr  r~   s   &&&&,  €rC   r�  Ú<Gemma4ForConditionalGeneration.prepare_inputs_for_generationO
  s[   ø€ Ü‘wÒ<Øñ
Ø!*ð
Ø?Qð
ØU[ñ
ˆ÷ #¯)à04ˆLÐ,Ñ-ç!Ø× Ñ Ð!3°TÓ:ˆAàÐrF   )r   r*  r„   )NNNNNNNNNNNNNNr   N)NF)TF)rH   rI   rJ   rK   r³  rk  rb  rr   r#   r3  r$   r†   r@  rD  r  r   r�  rN   rO   r‰   rŠ   s   @@rC   rm  rm  ¹	  s—   ù‡ € ð +Ð,VÐWÐØÐØÐ÷ó ð ÷Yñ Yó ðYð Ø÷N
ñ N
ó ó ðN
ò`;ò9ð ÷8ñ 8ó ð8÷<÷ rF   rm  )r¯   )r«  NN)r7   )rÉ  rž  rm  r  r)  rn  r6  )|r¹   Úcollectionsr   Úcollections.abcr   Údataclassesr   Ú	functoolsr   r;   r   Útorch.nnr   rø   Ú r	   r.  Úactivationsr
   Úcache_utilsr   r   Úconfiguration_utilsr   Ú
generationr   Úintegrationsr   Úmasking_utilsr   r   r   r   r   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr    Úutilsr!   r"   r#   r$   r%   r&   Úutils.deprecationr'   Úutils.genericr(   r)   Úutils.output_capturingr*   r+   Úauto.modeling_autor,   Úconfiguration_gemma4r-   r.   r/   r0   Úaccelerate.hooksr1   r5   rR   r[   r_   r4  re   rŒ   r©   rØ   r,  rC  rU  ÚConv1drm  ry  r‹  rœ  r¼  rÝ  rî  r  r$  r,  r:  rH  rJ  rg  rw  rŠ  r—  r¸  rÎ  rñ  r  rJ  r  r)  rn  rž  rÇ  rÉ  r6  rõ  r  r  r  rm  Ú__all__rG   rF   rC   Ú<module>rŸ     s	  ðó* Ý  Ý $Ý !Ý %ã Ý Ý $å &Ý !ß .Ý 3Ý )Ý 6÷	÷ 	ó 	õ CÝ 9ß Sß Kß FÝ &÷÷ õ 1ß Gß EÝ *ß gÓ gñ ×ÒÝ3ñ ðôð
 ôQÐ 7ó Qó óðQñ2 ðôð
 ôQ ;ó Qó óðQðD ô
QÐ$;ó 
Qó ð
Qð Ø
ô3Ð7ó 3ó ó ð3ô˜BŸI™Iô ô:4�B—I‘Iô 4ô*7 r§y¡yô 7ô>i)˜2Ÿ9™9ô i)ôX#¨b¯i©iô #ô8;¨¯©ô ;ô< ˜RŸY™Yô  ôH"˜bŸi™iô "ôB&˜RŸY™Yô &ôR0�r—y‘yô 0ôl*3 §	¡	ô *3ôZ@0˜Ÿ™ô @0ôF�b—i‘iô ô ?C "§)¡)ô ?CòD(÷.õ,	U÷%÷D5&ôpB)˜BŸI™Iô B)ôJ)Ð9ô )ôX)H˜"Ÿ)™)ô )HôX�B—I‘Iô ô&O< §	¡	ô O<ôdq)˜"Ÿ)™)ô q)ðh ô$#˜Ÿ	™	ó $#ó ð$#ôN"@�r—y‘yô "@ôJVÐ7ô VôrS B§L¡Lô Sð ôaJ˜Oó aJó ðaJñH Ð`ÔaôdVÐ+ó dVó bðdVñN Ð]Ô^ôX
Ð-¨ó X
ó _ðX
õvô ScÐ,ô ScôlAHÐ-ô AHôH6˜rŸy™yô 6õ6<õ~	ñ ðôôEÐ'ó EóðEñP
 ðôô[Ð%:¸Oó [óð[ò|‚rF   