+
    œ°³jù¥  ã                   ó  € ^ RI Ht ^ RIHt ^ RIt^ RIHt ^RIHt ^RI	H
t
HtHt ^RIHt ^RIHtHt ^RIHt ^R	IHt ^R
IHtHtHtHtHt ^RIHtHt ^RIH t H!t! ^RI"H#t# ^RI$H%t%H&t&H't' ^RI(H)t) ^RI*H+t+H,t, ^RI-H.t.H/t/ ^RI0H1t1 ]&! RR7      ] ! R R]4      4       4       t2 ! R R]Pf                  4      t4 ! R R]Pf                  4      t5 ! R R]Pf                  4      t6R R lt7R6R R  llt8R! t9R7R" lt: ! R# R$]Pf                  4      t; ! R% R&]4      t< ! R' R(]4      t=]& ! R) R*]!4      4       t> ! R+ R,]>4      t?]& ! R- R.]>4      4       t@]& ! R/ R0]>4      4       tAR1 R2 ltB]&! R3R7       ! R4 R5]>]4      4       tC. R8OtDR# )9é    )ÚCallable)Ú	dataclassN)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚMoonshineConfigz™
    Extends [~modeling_outputs.BaseModelOutput] to include the output attention mask since sequence length is not preserved in the model's forward.
    )Úcustom_introc                   ó4   a € ] tR t^2t o RtRtV 3R ltRtV tR# )ÚMoonshineEncoderModelOutputa~  
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to avoid performing attention on padding token indices after sequence compression. Returned because the
    sequence length may differ from the input sequence length. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.
Nc                óH   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   # )é   NÚattention_mask©ÚtorchÚTensor)ÚformatÚ__classdict__s   "€Úq/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/moonshine/modeling_moonshine.pyÚ__annotate__Ú(MoonshineEncoderModelOutput.__annotate__2   s   ø‡ ‚ ñ  —L‘L 4Õ'Ñ.ò! ó    © )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r'   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r,   s   @r-   r$   r$   2   s   ø‡ € ñð +/€N÷! ƒ r0   r$   c                   óD   a a€ ] tR t^Et oV 3R ltV3R lR ltRtVtV ;t# )ÚMoonshineEncoderMLPc                ó  <€ \         SV `  4        Wn        \        V,          V n        \
        P                  ! VP                  VP                  4      V n	        \
        P                  ! VP                  VP                  4      V n
        R # ©N©ÚsuperÚ__init__Úconfigr   Úactivation_fnÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©ÚselfrB   Ú
hidden_actÚ	__class__s   &&&€r-   rA   ÚMoonshineEncoderMLP.__init__F   s\   ø€ Ü‰ÑÔØŒÜ# JÕ/ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r&   Úhidden_statesÚreturnr(   )r+   r,   s   "€r-   r.   Ú MoonshineEncoderMLP.__annotate__M   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r0   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r>   )rH   rC   rI   )rK   rQ   s   &&r-   ÚforwardÚMoonshineEncoderMLP.forwardM   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr0   ©rC   rB   rH   rI   ©	r2   r3   r4   r5   rA   rU   r8   r9   Ú__classcell__©rM   r,   s   @@r-   r<   r<   E   s   ù‡ € õK÷÷ ð r0   r<   c                   óD   a a€ ] tR t^Tt oV 3R ltV3R lR ltRtVtV ;t# )ÚMoonshineDecoderMLPc                ó"  <€ \         SV `  4        Wn        \        V,          V n        \
        P                  ! VP                  VP                  ^,          4      V n	        \
        P                  ! VP                  VP                  4      V n
        R# )r&   Nr?   rJ   s   &&&€r-   rA   ÚMoonshineDecoderMLP.__init__U   sa   ø€ Ü‰ÑÔØŒÜ# JÕ/ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÈAÕ1MÓNˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rP   r(   )r+   r,   s   "€r-   r.   Ú MoonshineDecoderMLP.__annotate__\   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r0   c                ó¤   € V P                  V4      pVP                  ^RR7      w  rV P                  V4      V,          pV P                  V4      pV# )r&   ©Údiméÿÿÿÿ)rH   ÚchunkrC   rI   )rK   rQ   Úgates   && r-   rU   ÚMoonshineDecoderMLP.forward\   sQ   € ØŸ™ Ó/ˆØ+×1Ñ1°!¸Ð1Ó<ÑˆØ×*Ñ*¨4Ó0°=Õ@ˆØŸ™ Ó/ˆØÐr0   rW   rX   rZ   s   @@r-   r\   r\   T   s   ù‡ € õK÷÷ ð r0   r\   c                   óÎ   a a€ ] tR t^dt o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚMoonshineRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r&   rB   r    )r+   r,   s   "€r-   r.   Ú%MoonshineRotaryEmbedding.__annotate__f   s   ø€ ÷ Oñ O™ñ Or0   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r@   rA   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrB   Úrope_parametersrp   Úcompute_default_rope_parametersr   Úattention_scalingrD   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)rK   rB   rj   Úrope_init_fnrz   rM   s   &&&  €r-   rA   Ú!MoonshineRotaryEmbedding.__init__e   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr0   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r&   rB   rR   )r!   Útupler)   r*   Úfloat)r+   r,   s   "€r-   r.   rn   x   s.   ø€ ÷ 5ñ 5±ð 5Ñ[`Ñaf×amÑamÑotÐatÕ[uñ 5r0   c           	     óŠ  € V P                   R,          pV P                   P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        WT,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimN©Údtype)
rv   ÚgetÚgetattrrF   Únum_attention_headsÚintr)   Úaranger�   Úto)	rB   rj   ÚkwargsÚbaser„   r…   rc   Úattention_factorrz   s	   &&,      r-   rw   Ú8MoonshineRotaryEmbedding.compute_default_rope_parametersv   s¦   € ð ×%Ñ% lÕ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r0   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)r‡   rj   ÚmpsÚcpuF)Údevice_typeÚenabledrb   r†   )NºNNNNrd   )r—   Nr—   )rz   ÚexpandÚshaper�   r)   r�   rj   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrx   Úsinr‡   )
rK   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr•   ÚfreqsÚembrŸ   r    s
   &&&       r-   rU   Ú MoonshineRotaryEmbedding.forwardŒ   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rx   rB   rz   rt   r|   ru   rp   r>   )r2   r3   r4   r5   r   rA   Ústaticmethodrw   r)   Úno_gradr   rU   r8   r9   rY   rZ   s   @@r-   ri   ri   d   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ÷<ð <r0   ri   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r&   rQ   Ún_reprR   ©r)   r*   r‹   )r+   s   "r-   r.   r.   Ÿ   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur0   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r—   r—   Nr—   r—   )r™   r˜   Úreshape)rQ   r«   ÚbatchÚnum_key_value_headsÚslenr…   s   &&    r-   Ú	repeat_kvr²   Ÿ   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr0   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r&   ÚmoduleÚqueryÚkeyÚvaluer'   NÚscalingÚdropoutrŽ   )rD   ÚModuler)   r*   r�   r   r   )r+   s   "r-   r.   r.   «   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r0   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r&   )rc   r‡   )ÚpÚtrainingrd   )r²   Únum_key_value_groupsr)   Úmatmulr�   rD   Ú
functionalÚsoftmaxÚfloat32r�   r‡   r¹   r½   Ú
contiguous)r´   rµ   r¶   r·   r'   r¸   r¹   rŽ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r-   Úeager_attention_forwardrÈ   «   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r0   c                óz   € V R,          pV R,          p\         P                  ! V) V3RR7      P                  R4      # )z*Rotates half the hidden dims of the input.rb   ).:r   Nr&   ).:é   Nr&   rd   éþÿÿÿ)r)   ÚstackÚflatten)r¡   Úx1Úx2s   &  r-   Úrotate_halfrÐ   Ä   s6   € à	
ˆ9�€BØ	
ˆ9�€BÜ�;Š;˜˜˜R�y bÔ)×1Ñ1°"Ó5Ð5r0   c                óŽ  € VP                  V4      pVP                  V4      pVRRVP                  R,          ^,          13,          P                  ^RR7      pVRRVP                  R,          ^,          13,          P                  ^RR7      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜Wb,          \        V4      V,          ,           p
W‚,          \        V4      V,          ,           p\        P
                  ! W§.RR7      p
\        P
                  ! W¹.RR7      pW«3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nrb   rd   )Ú	unsqueezer™   Úrepeat_interleaverÐ   r)   rž   )ÚqÚkrŸ   r    Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds   &&&&&       r-   Úapply_rotary_pos_embrÞ   Ë   s6  € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
:Ñ
:¸1À"Ð
:Ó
E€CØ
ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
:Ñ
:¸1À"Ð
:Ó
E€Cð —‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ð �{œ{¨5Ó1°CÕ7Õ8€GØ�{œ{¨5Ó1°CÕ7Õ8€Gô �iŠi˜Ð)¨rÔ2€GÜ�iŠi˜Ð)¨rÔ2€GØÐÐr0   c                   óX   a a€ ] tR t^ót oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚMoonshineAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc          
      ó8   <€ V ^8„  d   QhRS[ RS[RS[RS[RS[/# )r&   rB   Ú	layer_idxÚ	is_causalrŠ   r°   )r!   r‹   Úbool)r+   r,   s   "€r-   r.   ÚMoonshineAttention.__annotate__ö   s=   ø€ ÷ #&ñ #&áð#&ñ ð#&ñ ð	#&ñ
 !ð#&ñ !ñ#&r0   c                ó   <€ \         SV `  4        VP                  R VRV/4       Wn        W n        \        VRVP                  VP                  ,          4      V n        VP                  VP                  ,          V n
        V P                  R,          V n        VP                  V n        W0n        \        P                  ! VP                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP                  V P                  ,          VP                  RR7      V n        V P                  P*                  eW   V P                  P*                  pW`P                  V,           ^,
          V,          ,          pWpP                  ,
          V n        R# ^ V n        R# )rŠ   r°   r…   ©ÚbiasFNg      à¿)r@   rA   ÚupdaterB   râ   r‰   rF   rŠ   r…   r°   r¾   r¸   Úattention_dropoutrã   rD   rE   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚpad_head_dim_to_multiple_ofÚhead_dim_padding)	rK   rB   râ   rã   rŠ   r°   Útarget_multipleÚtarget_head_dimrM   s	   &&&&&&  €r-   rA   ÚMoonshineAttention.__init__ö   s®  ø€ ô 	‰ÑÔØ�‰Ð,Ð.AÐCXÐZmÐnÔoØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØ"Œä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŒð �;‰;×2Ñ2Ò>Ø"Ÿk™k×EÑEˆOØ-·-±-À/Õ2QÐTUÕ2UÐZiÕ1iÕjˆOØ$3·m±mÕ$CˆDÖ!à$%ˆDÖ!r0   c                óˆ  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )	r&   rQ   Úposition_embeddingsNr'   Úpast_key_valuesÚkey_value_statesrŽ   rR   )r)   r*   r€   r   r   r   )r+   r,   s   "€r-   r.   rå     s¸   ø€ ÷ O)ñ O)á—|‘|ðO)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðO)ñ Ÿ™ tÕ+ð	O)ñ
  �ðO)ñ  Ÿ,™,¨Õ-ðO)ñ Ñ-Õ.ðO)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	MñO)r0   c           	     óÊ  € VP                   R R w  rxV P                  V4      P                  WxV P                  P                  V P
                  4      P                  ^^4      p	VR Jp
Ve`   VP                  P                  V P                  4      pV
'       d'   RVP                  V P                  &   VP                  pMVP                  pVe   TMTpV
'       d`   V'       dX   X'       dP   VP                  V P                  ,          P                  pVP                  V P                  ,          P                  pMÌV P                  V4      P                  VRV P                  P                  V P
                  4      P                  ^^4      pV P!                  V4      P                  VRV P                  P                  V P
                  4      P                  ^^4      pV
'       d#   Ve   VP#                  WÞV P                  4      w  rÞV
'       g7   Vw  pp\%        W�VV4      w  r�Ve   VP#                  WÞV P                  4      w  rÞ\&        P(                  ! V P                  P*                  \,        4      pV P.                  ;'       d    VR J ;'       d    V^8„  pV P0                  ^ 8”  d£   \2        P4                  P6                  P9                  V	^ V P0                  34      p	\2        P4                  P6                  P9                  V^ V P0                  34      p\2        P4                  P6                  P9                  V^ V P0                  34      pT! V V	VVV3RV P:                  '       g   RMV P<                  RV P>                  RV/VB w  ppV P0                  ^ 8”  d   VRR V P0                  ) 13,          pVPA                  WxR4      PC                  4       pV PE                  V4      pVV3# )NTr¹   ç        r¸   rã   .rd   )#r™   rì   ÚviewrB   r°   r…   r�   Ú
is_updatedrˆ   râ   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesrí   rî   ré   rÞ   r   Úget_interfaceÚ_attn_implementationrÈ   rã   rñ   r)   rD   rÀ   Úpadr½   rê   r¸   r®   rÃ   rï   )rK   rQ   rö   r'   r÷   rø   rŽ   ÚbszÚq_lenÚquery_statesÚis_cross_attentionrü   Úcurrent_statesrÄ   rÅ   rŸ   r    Úattention_interfacerã   rÇ   rÆ   s   &&&&&&,              r-   rU   ÚMoonshineAttention.forward  sa  € ð #×(Ñ(¨¨"Ð-‰
ˆð �K‰K˜Ó&×+Ñ+¨C¸¿¹×8WÑ8WÐY]×YfÑYfÓg×qÑqÐrsÐuvÓwð 	ð .°TÐ9ÐØÒ&Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJß!à=A�×*Ñ*¨4¯>©>Ñ:Ø"1×"GÑ"G‘à"1×"FÑ"F�ð .>Ò-IÑ)È}ˆß§/·jØ(×/Ñ/°·±Õ?×DÑDˆJØ*×1Ñ1°$·.±.ÕA×HÑH‰Lð —‘˜NÓ+ß‘�c˜2˜tŸ{™{×>Ñ>ÀÇÁÓNß‘˜1˜a“ð ð —‘˜NÓ+ß‘�c˜2˜tŸ{™{×>Ñ>ÀÇÁÓNß‘˜1˜a“ð ÷
 " oÒ&AØ+:×+AÑ+AÀ*Ð\`×\jÑ\jÓ+kÑ(�
ç!Ø*‰HˆC�Ü';¸LÐVYÐ[^Ó'_Ñ$ˆLàÒ*Ø+:×+AÑ+AÀ*Ð\`×\jÑ\jÓ+kÑ(�
ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðð —N‘N×KÐK ~¸Ð'=×KÐKÀ%È!Á)ˆ	à× Ñ  1Ô$Ü Ÿ8™8×.Ñ.×2Ñ2°<À!ÀT×EZÑEZÐA[Ó\ˆLÜŸ™×,Ñ,×0Ñ0°¸aÀ×AVÑAVÐ=WÓXˆJÜ Ÿ8™8×.Ñ.×2Ñ2°<À!ÀT×EZÑEZÐA[Ó\ˆLá$7ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð  ð
%
ð ñ
%
Ñ!ˆ�\ð × Ñ  1Ô$Ø% cÐ+C¨d×.CÑ.CÐ-CÐ+CÐ&CÕDˆKà!×)Ñ)¨#°bÓ9×DÑDÓFˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r0   )rê   rB   r…   rñ   rã   rí   râ   r¾   rï   rì   r¸   rî   )NNNN)
r2   r3   r4   r5   r6   rA   rU   r8   r9   rY   rZ   s   @@r-   rà   rà   ó   s%   ù‡ € ÙG÷#&ó #&÷JO)÷ O)ò O)r0   rà   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMoonshineEncoderLayerim  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r&   rB   râ   ©r!   r‹   )r+   r,   s   "€r-   r.   Ú"MoonshineEncoderLayer.__annotate__n  s   ø€ ÷ Uñ U™ð U¹3ñ Ur0   c                óh  <€ \         SV `  4        VP                  V n        \        VVR VP                  VP
                  R7      V n        \        WP                  4      V n	        \        P                  ! VP                  R R7      V n        \        P                  ! VP                  R R7      V n        R# )F©rB   râ   rã   rŠ   r°   rç   N)r@   rA   rF   rà   Úencoder_num_attention_headsÚencoder_num_key_value_headsÚ	self_attnr<   Úencoder_hidden_actÚmlprD   Ú	LayerNormÚinput_layernormÚpost_attention_layernorm©rK   rB   râ   rM   s   &&&€r-   rA   ÚMoonshineEncoderLayer.__init__n  s‰   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä+ØØØØ &× BÑ BØ &× BÑ Bô
ˆŒô ' v×/HÑ/HÓIˆŒÜ!Ÿ|š|¨F×,>Ñ,>ÀUÔKˆÔÜ(*¯ª°V×5GÑ5GÈeÔ(TˆÖ%r0   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r&   rQ   r'   Nr¢   r÷   Ú	use_cacherö   rŽ   rR   )r)   r*   Ú
LongTensorr   rä   r€   r   r   )r+   r,   s   "€r-   r.   r  ~  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr0   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )rQ   r'   r¢   r÷   r  rö   r1   )r  r  r  r  )
rK   rQ   r'   r¢   r÷   r  rö   rŽ   ÚresidualÚ_s
   &&&&&&&,  r-   rU   ÚMoonshineEncoderLayer.forward~  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr0   )rF   r  r  r  r  )NNNFNrX   rZ   s   @@r-   r  r  m  s    ù‡ € ÷Uó U÷ ÷ ò r0   r  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚMoonshineDecoderLayeriž  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r&   rB   râ   Nr  )r+   r,   s   "€r-   r.   Ú"MoonshineDecoderLayer.__annotate__Ÿ  s#   ø€ ÷ Lñ L™ð L¹3À½:ñ Lr0   c                ó  <€ \         SV `  4        VP                  V n        \        VVR VP                  VP
                  R7      V n        \        VVRVP                  VP
                  R7      V n        \        WP                  4      V n
        \        P                  ! VP                  RR7      V n        \        P                  ! VP                  RR7      V n        \        P                  ! VP                  RR7      V n        R# )Tr  Frç   N)r@   rA   rF   rà   rŠ   r°   r  Úencoder_attnr\   rL   r  rD   r  r  r  Úfinal_layernormr  s   &&&€r-   rA   ÚMoonshineDecoderLayer.__init__Ÿ  sÌ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä+ØØØØ &× :Ñ :Ø &× :Ñ :ô
ˆŒô /ØØØØ &× :Ñ :Ø &× :Ñ :ô
ˆÔô ' v×/@Ñ/@ÓAˆŒÜ!Ÿ|š|¨F×,>Ñ,>ÀUÔKˆÔÜ(*¯ª°V×5GÑ5GÈeÔ(TˆÔ%Ü!Ÿ|š|¨F×,>Ñ,>ÀUÔKˆÖr0   c                óZ  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  S[ P                  3,          R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )r&   rQ   r'   NÚencoder_hidden_statesÚencoder_attention_maskr¢   Úencoder_position_idsr÷   r  rö   Úencoder_position_embeddingsrŽ   rR   )	r)   r*   r  r   rä   r€   r   r   ÚFloatTensor)r+   r,   s   "€r-   r.   r'  ·  s   ø€ ÷ ,ñ ,á—|‘|ð,ñ Ÿ™ tÕ+ð,ñ  %Ÿ|™|¨dÕ2ð	,ñ
 !&§¡¨tÕ 3ð,ñ ×&Ñ&¨Õ-ð,ñ $×.Ñ.°Õ5ð,ñ  �ð,ñ ˜$•;ð,ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð,ñ &+©5¯<©<¹¿¹Ð+EÕ%FÈÕ%Mð,ñ Ñ+Õ,ð,ñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñ,r0   c                óB  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV	/VB w  rWÁ,           pVe4   TpV P                  V4      pV P                  VVVVVR7      w  rWÁ,           pTpV P	                  V4      pV P                  V4      pWÁ,           pV# )rQ   r'   r¢   r÷   r  rö   )rQ   rø   r'   r÷   r  r1   )r  r  r  r)  r*  r  )rK   rQ   r'   r-  r.  r¢   r/  r÷   r  rö   r0  rŽ   r!  r"  s   &&&&&&&&&&&,  r-   rU   ÚMoonshineDecoderLayer.forward·  só   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆà Ò,Ø$ˆHØ ×9Ñ9¸-ÓHˆMØ#×0Ñ0Ø+Ø!6Ø5Ø /Ø#ð  1ó  ÑˆMð %Õ4ˆMà ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØ Õ0ˆØÐr0   )r)  r*  rF   r  r  r  r  r>   )	NNNNNNFNNrX   rZ   s   @@r-   r%  r%  ž  s    ù‡ € ÷Lõ L÷0,÷ ,ò ,r0   r%  c                   ób   a € ] tR tRt o RtRtRtRtRR.tRt	Rt
RtV 3R lR	 ltV 3R
 ltRtV tR# )ÚMoonshinePreTrainedModeliæ  ÚmodelÚinput_valuesÚaudioTr  r%  c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r&   Úinput_lengths)r)   r  )r+   r,   s   "€r-   r.   Ú%MoonshinePreTrainedModel.__annotate__ô  s   ø€ ÷ #ñ #¹e×>NÑ>Nñ #r0   c                óÆ   € \        V^,
          ^@,          ^,           4      p\        V^,
          ^,          ^,           4      p\        V^,
          ^,          ^,           4      pV# )z8
Computes the output length of the convolutional layers
)r‹   )rK   r:  Úoutput_conv1_lengthÚoutput_conv2_lengthÚoutput_conv3_lengths   &&   r-   Ú _get_feat_extract_output_lengthsÚ9MoonshinePreTrainedModel._get_feat_extract_output_lengthsô  sZ   € ô " =°3Õ#6¸"Õ"<¸qÕ"@ÓAÐÜ!Ð#6¸Õ#:¸aÕ"?À!Õ"CÓDÐÜ!Ð#6¸Õ#:¸aÕ"?À!Õ"CÓDÐà"Ð"r0   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rm   r    )r+   r,   s   "€r-   r.   r;  æ  s   ø‡ ‚ áÑò r0   r1   N)r2   r3   r4   r5   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr@  r7   r8   r9   r:   s   @r-   r5  r5  æ  sL   ø‡ € ð  ÐØ$€OØÐØ&*Ð#Ø0Ð2IÐJÐØÐØ€Nà!Ð÷#ð #÷ ƒ r0   r5  c                   ó    a a€ ] tR tRt oRtRtR]R]/tV3R lV 3R llt	V3R lR	 lt
V3R
 lR lt]]RV3R lR ll4       4       tRtVtV ;t# )ÚMoonshineEncoderiÿ  z“
Transformer encoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineEncoderLayer`]

Args:
    config: MoonshineConfig
r7  Ú
attentionsrQ   c                ó    <€ V ^8„  d   QhRS[ /# rm   r    )r+   r,   s   "€r-   r.   ÚMoonshineEncoder.__annotate__  s   ø€ ÷ ñ ™ñ r0   c           	     ó€  <€ \         SV `  V4       Wn        VP                  p\        P
                  ! ^V^^@RR7      V n        \        P
                  ! V^V,          ^^R7      V n        \        P
                  ! ^V,          V^^R7      V n        \        P                  ! ^VRR7      V n
        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        P                   ! VRR7      V n        \%        VR7      V n        RV n        V P+                  4        R# u upi )	rÊ   F)Úkernel_sizeÚstriderè   )rQ  rR  gñhãˆµøä>)Ú
num_groupsÚnum_channelsÚepsrç   ©rB   N)r@   rA   rB   rF   rD   ÚConv1dÚconv1Úconv2Úconv3Ú	GroupNormÚ	groupnormÚ
ModuleListÚrangeÚencoder_num_hidden_layersr  rÿ   r  Ú
layer_normri   Ú
rotary_embÚgradient_checkpointingÚ	post_init)rK   rB   Ú	embed_dimÚidxrM   s   &&  €r-   rA   ÚMoonshineEncoder.__init__  sî   ø€ Ü‰Ñ˜Ô ØŒØ×&Ñ&ˆ	ä—Y’Y˜q )¸ÀRÈeÔTˆŒ
Ü—Y’Y˜y¨!¨i­-ÀQÈqÔQˆŒ
Ü—Y’Y˜q 9�}¨iÀQÈqÔQˆŒ
ÜŸš°ÀÐPTÔUˆŒä—m’mÜ;@À×AaÑAaÔ;bÓcÑ;b°CÔ" 6Ö/Ñ;bÑcó
ˆŒô Ÿ,š, y°uÔ=ˆŒÜ2¸&ÔAˆŒØ&+ˆÔ#Ø�‰Öùò ds   ÃD;c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r&   rR   ©rD   rº   )r+   r,   s   "€r-   r.   rO    s   ø€ ÷ ñ ¡b§i¡iñ r0   c                ó   € V P                   # r>   ©rX  ©rK   s   &r-   Úget_input_embeddingsÚ%MoonshineEncoder.get_input_embeddings  s   € Ø�z‰zÐr0   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r&   r·   ri  )r+   r,   s   "€r-   r.   rO  "  s   ø€ ÷ ñ ©"¯)©)ñ r0   c                ó   € Wn         R # r>   rk  ©rK   r·   s   &&r-   Úset_input_embeddingsÚ%MoonshineEncoder.set_input_embeddings"  s   € ØŽ
r0   c          	      óˆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[S[,          /# )r&   r7  r'   NrŽ   rR   )r)   r1  r*   r   r   r€   r   )r+   r,   s   "€r-   r.   rO  '  sN   ø€ ÷ <
ñ <
á×'Ñ'ð<
ñ Ÿ™ tÕ+ð<
ñ Ñ+Õ,ð	<
ñ
 
Ñ(Õ	(ñ<
r0   c           
     ó˜  € VP                  ^4      p\        P                  P                  V P	                  V4      4      pV P                  V4      p\        P                  P                  V P                  V4      4      p\        P                  P                  V P                  V4      4      pVP                  ^ ^^4      pRpVe@   V P                  VP                  R
,          4      pRpVRRRV13,          RRV13,          pTp\        V P                  VVVR7      p\        P                  ! ^ VP                  ^,          VP                   R7      P                  ^ 4      pV P#                  WHR7      p	V P$                   F  p
V
! V3RVRVRV	/VB pK  	  V P'                  V4      p\)        TVe   VP+                  4       R	7      # RR	7      # )a¾  
Args:
    input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
        Float values of the raw speech waveform. Raw speech waveform can be
        obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
        `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
        the soundfile library (`pip install soundfile`). To prepare the array into
        `input_values`, the [`AutoFeatureExtractor`] should be used for padding
        and conversion into a tensor of type `torch.FloatTensor`.
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`:
        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
        [What are attention masks?](../glossary#attention-mask)
N.©rB   Úinputs_embedsr'   r-  ©rj   ©r¢   r'   r¢   rö   )Úlast_hidden_stater'   rd   i€  )rÒ   rD   rÀ   ÚtanhrX  r\  ÚgelurY  rZ  Úpermuter@  r™   r
   rB   r)   rŒ   rj   ra  rÿ   r`  r$   r‹   )rK   r7  r'   rŽ   rQ   Úoutput_attention_maskÚmask_lenÚdownsample_strider¢   rö   Úencoder_layers   &&&,       r-   rU   ÚMoonshineEncoder.forward%  sÂ  € ð. $×-Ñ-¨aÓ0ˆÜŸ™×*Ñ*¨4¯:©:°lÓ+CÓDˆØŸ™ }Ó5ˆÜŸ™×*Ñ*¨4¯:©:°mÓ+DÓEˆÜŸ™×*Ñ*¨4¯:©:°mÓ+DÓEˆØ%×-Ñ-¨a°°AÓ6ˆð !%ÐØÒ%Ø×<Ñ<¸^×=QÑ=QÐRTÕ=UÓVˆHØ *ÐØ+¨CÑ1DÐ3DÐ1DÐ,DÕEÀcÈ9ÈHÈ9ÀnÕUˆNØ$2Ð!ä2Ø—;‘;Ø'Ø)Ø"/ô	
ˆô —|’| A }×':Ñ':¸1Õ'=Àm×FZÑFZÔ[×eÑeÐfgÓhˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[œ[ˆMÙ)Øñà-ðð *ðð %8ð	ð
 ñŠMñ )ð Ÿ™¨Ó6ˆä*Ø+Ø:OÒ:[Ð0×4Ñ4Ó6ô
ð 	
àaeô
ð 	
r0   )	rB   rX  rY  rZ  rb  r\  r`  rÿ   ra  r>   )r2   r3   r4   r5   r6   rD  rà   r  Ú_can_record_outputsrA   rm  rr  r   r   rU   r8   r9   rY   rZ   s   @@r-   rL  rL  ÿ  si   ù‡ € ñð %€OàÐ(ØÐ.ðÐ÷
ó ÷$ð ÷ð ð  Ø÷<
ñ <
ó ó  ÷<
ð <
r0   rL  c                   ó    a a€ ] tR tRt oRtR]! ]^RR7      R]R]! ]^RR7      /tV3R	 lV 3R
 llt	]
]RV3R lR ll4       4       tRtVtV ;t# )ÚMoonshineDecoderif  Ú	input_idsrM  r  )ÚindexÚ
layer_namerQ   Úcross_attentionsr)  c                ó    <€ V ^8„  d   QhRS[ /# rm   r    )r+   r,   s   "€r-   r.   ÚMoonshineDecoder.__annotate__o  s   ø€ ÷ ñ ™ñ r0   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \
        P                  ! VP                  R R7      V n        \!        VR7      V n        R V n        V P'                  4        R# u upi )Frç   rV  N)r@   rA   Úpad_token_idÚpadding_idxÚ
vocab_sizerD   Ú	EmbeddingrF   Úembed_tokensr]  r^  Únum_hidden_layersr%  rÿ   r  Únormri   ra  rb  rc  )rK   rB   re  rM   s   && €r-   rA   ÚMoonshineDecoder.__init__o  sÅ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÔSXÐY_×YqÑYqÔSrÓ$sÑSrÈCÔ%:¸6Ö%GÑSrÑ$sÓtˆŒÜ—L’L ×!3Ñ!3¸%Ô@ˆŒ	Ü2¸&ÔAˆŒØ&+ˆÔ#ð 	�‰Öùò %ts   ÂDc                ó^  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[,          RS[S[	,          /
# )r&   r†  Nr'   r¢   r÷   rw  r  r-  r.  rŽ   rR   )
r)   r  r*   r   r1  rä   r   r   r€   r   )r+   r,   s   "€r-   r.   r‹    sÍ   ø€ ÷ G
ñ G
á×#Ñ# dÕ*ðG
ñ Ÿ™ tÕ+ðG
ñ ×&Ñ&¨Õ-ð	G
ñ
  �ðG
ñ ×(Ñ(¨4Õ/ðG
ñ ˜$•;ðG
ñ  %×0Ñ0°4Õ7ðG
ñ !&§¡¨tÕ 3ðG
ñ Ñ+Õ,ðG
ñ 
Ñ(Õ	(ñG
r0   c	                óê  € VRJ VRJ,          '       d   \        R4      hVf   V P                  V4      pV'       d9   Vf5   \        \        V P                  R7      \        V P                  R7      4      pVfb   Ve   VP                  4       M^ p
\        P                  ! VP                  ^,          VP                  R7      V
,           pVP                  ^ 4      p\        V P                  VVVVR7      p\        V P                  VVVR7      pTpV P                  WÃR7      pV P                   F  pV! VVV3RVR	VR
VRVRV/V	B pK  	  V P                  V4      p\!        TV'       d   VR7      # RR7      # )a\  
encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
    Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
    of the decoder.
encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`:
    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.
    [What are attention masks?](../glossary#attention-mask)
Nz:You must specify exactly one of input_ids or inputs_embedsrV  rx  )rB   rw  r'   r÷   r¢   rv  ry  r.  r¢   r÷   r  rö   )rz  r÷   )Ú
ValueErrorr‘  r   r   rB   Úget_seq_lengthr)   rŒ   r™   rj   rÒ   r   r
   ra  rÿ   r“  r   )rK   r†  r'   r¢   r÷   rw  r  r-  r.  rŽ   Úpast_seen_tokensÚcausal_maskrQ   rö   Údecoder_layers   &&&&&&&&&,     r-   rU   ÚMoonshineDecoder.forward}  sš  € ð0 ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[œ[ˆMÙ)ØØØ%ñ
ð (>ð	
ð
 *ð
ð !0ð
ð $ð
ð %8ð
ð ñ
ŠMñ )ð Ÿ	™	 -Ó0ˆä8Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r0   )r‘  rb  rÿ   r“  rŽ  ra  r�  )NNNNNNNN)r2   r3   r4   r5   rD  r   rà   r%  rƒ  rA   r   r   rU   r8   r9   rY   rZ   s   @@r-   r…  r…  f  sk   ù‡ € à!€Oà‘nÐ%7¸qÈ[ÔYØÐ.Ø™NÐ+=ÀQÐSaÔbðÐ÷ó ð  Ø÷G
ñ G
ó ó  ÷G
ð G
r0   r…  c                   ó€   a a€ ] tR tRt oV3R lV 3R lltR tR tR tR t]	]
RV3R lR	 ll4       4       tR
tVtV ;t# )ÚMoonshineModeliÉ  c                ó    <€ V ^8„  d   QhRS[ /# rm   r    )r+   r,   s   "€r-   r.   ÚMoonshineModel.__annotate__Ë  ó   ø€ ÷ ñ ™ñ r0   c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # r>   )r@   rA   rL  Úencoderr…  Údecoderrc  ©rK   rB   rM   s   &&€r-   rA   ÚMoonshineModel.__init__Ë  s2   ø€ Ü‰Ñ˜Ô ä'¨Ó/ˆŒÜ'¨Ó/ˆŒà�‰Ör0   c                ó.   € V P                   P                  # r>   ©r¤  r‘  rl  s   &r-   rm  Ú#MoonshineModel.get_input_embeddingsÓ  s   € Ø�|‰|×(Ñ(Ð(r0   c                ó&   € WP                   n        R # r>   r¨  rq  s   &&r-   rr  Ú#MoonshineModel.set_input_embeddingsÖ  s   € Ø$)�‰Ö!r0   c                ó:   € V P                   P                  4        R# )z“
Calling this function will disable the gradient computation for the Moonshine encoder so that its parameters will
not be updated during training.
N)r£  Ú_freeze_parametersrl  s   &r-   Úfreeze_encoderÚMoonshineModel.freeze_encoderÙ  s   € ð
 	�‰×'Ñ'Ö)r0   c                ó   € \        R4      h)zŠ
Masks extracted features along time axis and/or along feature axis according to
[SpecAugment](https://huggingface.co/papers/1904.08779).
zNot needed for Moonshine)ÚAttributeErrorrl  s   &r-   Ú_mask_input_featuresÚ#MoonshineModel._mask_input_featuresà  s   € ô
 Ð7Ó8Ð8r0   c                ó¶  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[R,          RS[S[ P                  ,          R,          R	S[S[ P                  ,          R,          R
S[R,          RS[S[,          RS[/# )r&   r7  Nr'   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsr÷   Údecoder_inputs_embedsÚdecoder_position_idsr  rŽ   rR   )	r)   r1  r  r€   r   rä   r   r   r   )r+   r,   s   "€r-   r.   r   é  sö   ø€ ÷ C
ñ C
á×'Ñ'¨$Õ.ðC
ñ ×(Ñ(¨4Õ/ðC
ñ !×+Ñ+¨dÕ2ð	C
ñ
 !&× 0Ñ 0°4Õ 7ðC
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ðC
ñ -¨tÕ3ðC
ñ  %¡U×%6Ñ%6Õ7¸$Õ>ðC
ñ $¡E×$4Ñ$4Õ5¸Õ<ðC
ñ ˜$•;ðC
ñ Ñ+Õ,ðC
ñ 
ñC
r0   c
                óf  € Vf   V P                   ! V3RV/V
B pV P                  ! R
RVRVRVP                  RVP                  RVRVRVRV	/V
B p\	        VP                  VP
                  VP                  VP                  VP                  VP                  VP                  VP                  R	7      # )a:  
input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
    Float values of the raw speech waveform. Raw speech waveform can be
    obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
    `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
    the soundfile library (`pip install soundfile`). To prepare the array into
    `input_values`, the [`AutoFeatureExtractor`] should be used for padding
    and conversion into a tensor of type `torch.FloatTensor`.
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

Example:

```python
>>> import torch
>>> from transformers import AutoFeatureExtractor, MoonshineModel
>>> from datasets import load_dataset

>>> model = MoonshineModel.from_pretrained("UsefulSensors/moonshine-tiny")
>>> feature_extractor = AutoFeatureExtractor.from_pretrained("UsefulSensors/moonshine-tiny")
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
>>> input_values = inputs.input_values
>>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
>>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state
>>> list(last_hidden_state.shape)
[1, 2, 288]
```
r'   r†  r-  r.  r÷   rw  r¢   r  )rz  r÷   Údecoder_hidden_statesÚdecoder_attentionsr‰  Úencoder_last_hidden_stater-  Úencoder_attentionsr1   )	r£  r¤  rz  r'   r   r÷   rQ   rM  r‰  )rK   r7  r'   rµ  r¶  r·  r÷   r¸  r¹  r  rŽ   Údecoder_outputss   &&&&&&&&&&, r-   rU   ÚMoonshineModel.forwardç  sö   € ðZ Ò"Ø/3¯|ª|¸LÑ/rÐYgÐ/rÐkqÑ/rˆOàEIÇ\Â\ñ 
F
Ø'ð
F
à1ð
F
ð #2×"CÑ"Cð
F
ð $3×#AÑ#Að	
F
ð
 ,ð
F
ð 0ð
F
ð .ð
F
ð  ð
F
ð ñ
F
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r0   )r¤  r£  )	NNNNNNNNN)r2   r3   r4   r5   rA   rm  rr  r®  r²  r   r   rU   r8   r9   rY   rZ   s   @@r-   rž  rž  É  sM   ù‡ € ÷ó ò)ò*ò*ò9ð Ø÷C
ñ C
ó ó ÷C
ð C
r0   rž  c                óP   € V ^8„  d   QhR\         P                  R\        R\        /# )r&   r†  r�  Údecoder_start_token_idr¬   )r+   s   "r-   r.   r.   /  s&   € ÷ ñ ¤%§,¡,ð ¼cð Ô[^ñ r0   c                óÈ   € V P                  V P                  4      pV RRR13,          P                  4       VR&   W#R&   Vf   \        R4      hVP	                  VR8H  V4       V# )z)
Shift input ids one token to the right.
r—   Nz1self.model.config.pad_token_id has to be defined.rd   )r—   :rÊ   NN)r—   r   iœÿÿÿ)Ú	new_zerosr™   r{   r—  Úmasked_fill_)r†  r�  rÂ  Úshifted_input_idss   &&& r-   Úshift_tokens_rightrÇ  /  sp   € ð "×+Ñ+¨I¯O©OÓ<ÐØ(¨¨C¨R¨C¨Õ0×6Ñ6Ó8Ð�eÑØ4�dÑàÒÜÐLÓMÐMà×"Ñ"Ð#4¸Ñ#<¸lÔKàÐr0   zj
    The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.
    c                   óŽ   a a€ ] tR tRt oRR/tV3R lV 3R lltR tR tV3R lR	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )Ú!MoonshineForConditionalGenerationi?  zproj_out.weightz!model.decoder.embed_tokens.weightc                ó    <€ V ^8„  d   QhRS[ /# rm   r    )r+   r,   s   "€r-   r.   Ú.MoonshineForConditionalGeneration.__annotate__G  r¡  r0   c                óÊ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        V P                  4        R# )Frç   N)
r@   rA   rž  r6  rD   rE   rF   r�  Úproj_outrc  r¥  s   &&€r-   rA   Ú*MoonshineForConditionalGeneration.__init__G  sH   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆŒ
ÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÈeÔTˆŒð 	�‰Ör0   c                ó   € V P                   # r>   ©rÍ  rl  s   &r-   Úget_output_embeddingsÚ7MoonshineForConditionalGeneration.get_output_embeddingsO  s   € Ø�}‰}Ðr0   c                ó   € Wn         R # r>   rÐ  )rK   Únew_embeddingss   &&r-   Úset_output_embeddingsÚ7MoonshineForConditionalGeneration.set_output_embeddingsR  s   € Ø&Žr0   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rh  ri  )r+   r,   s   "€r-   r.   rË  U  s   ø€ ÷ 1ñ 1¡b§i¡iñ 1r0   c                ó6   € V P                   P                  4       # r>   )r6  rm  rl  s   &r-   rm  Ú6MoonshineForConditionalGeneration.get_input_embeddingsU  s   € Ø�z‰z×.Ñ.Ó0Ð0r0   c                óÞ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[R,          RS[S[ P                  ,          R,          R	S[S[ P                  ,          R,          R
S[R,          RS[ P                  R,          RS[S[,          RS[/# )r&   r7  Nr'   rµ  r¶  r·  r÷   r¸  r¹  r  ÚlabelsrŽ   rR   )	r)   r1  r  r€   r   rä   r   r   r   )r+   r,   s   "€r-   r.   rË  Z  s  ø€ ÷ `
ñ `
á×'Ñ'¨$Õ.ð`
ñ ×(Ñ(¨4Õ/ð`
ñ !×+Ñ+¨dÕ2ð	`
ñ
 !&× 0Ñ 0°4Õ 7ð`
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ð`
ñ -¨tÕ3ð`
ñ  %¡U×%6Ñ%6Õ7¸$Õ>ð`
ñ $¡E×$4Ñ$4Õ5¸Õ<ð`
ñ ˜$•;ð`
ñ × Ñ  4Õ'ð`
ñ Ñ+Õ,ð`
ñ 
ñ`
r0   c                óJ  € VP                  RR4      pV
e=   Vf9   Vf5   \        W P                  P                  V P                  P                  4      pV P
                  ! V3RVRVRVRVRVRVR	VR
V	/VB pV P                  VP                  4      pRpV
e5   V P                  ! RRTRRRV P                  P                  RVe   TMT
/VB p\        VVVP                  VP                  VP                  VP                  VP                  VP                   VP"                  R7	      # )ah  
input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
    Float values of the raw speech waveform. Raw speech waveform can be
    obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
    `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
    the soundfile library (`pip install soundfile`). To prepare the array into
    `input_values`, the [`AutoFeatureExtractor`] should be used for padding
    and conversion into a tensor of type `torch.FloatTensor`.
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

Example:

```python
>>> import torch
>>> from transformers import AutoProcessor, MoonshineForConditionalGeneration
>>> from datasets import load_dataset

>>> processor = AutoProcessor.from_pretrained("UsefulSensors/moonshine-tiny")
>>> model = MoonshineForConditionalGeneration.from_pretrained("UsefulSensors/moonshine-tiny")

>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")

>>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
>>> input_values = inputs.input_values

>>> generated_ids = model.generate(input_values, max_new_tokens=100)

>>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
>>> transcription
'Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
```Úshift_labelsNr'   rµ  r·  r¶  r÷   r¸  r¹  r  ÚlogitsrÛ  r�  )	ÚlossrÞ  r÷   r»  r¼  r‰  r½  r-  r¾  r1   )ÚpoprÇ  rB   r�  rÂ  r6  rÍ  rz  Úloss_functionr�  r   r÷   r»  r¼  r‰  r½  r-  r¾  )rK   r7  r'   rµ  r¶  r·  r÷   r¸  r¹  r  rÛ  rŽ   rÝ  ÚoutputsrÞ  rß  s   &&&&&&&&&&&,    r-   rU   Ú)MoonshineForConditionalGeneration.forwardX  s|  € ðh —z‘z .°$Ó7ˆàÒØ Ò(Ð-BÒ-JÜ$6ØŸK™K×4Ñ4°d·k±k×6XÑ6Xó%Ð!ð '+§j¢jØñ'
à)ð'
ð 0ð'
ð ,ð	'
ð
 $:ð'
ð ,ð'
ð #8ð'
ð "6ð'
ð  ð'
ð ñ'
ˆð —‘˜w×8Ñ8Ó9ˆàˆØÒð
 ×%Ò%ñ Øðàðð  Ÿ;™;×1Ñ1ðð .:Ò-E™\È6ð	ð
 ñˆDô ØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r0   )r6  rÍ  )
NNNNNNNNNN)r2   r3   r4   r5   Ú_tied_weights_keysrA   rÑ  rÕ  rm  r   r   rU   r8   r9   rY   rZ   s   @@r-   rÉ  rÉ  ?  s[   ù‡ € ð ,Ð-PÐQÐ÷ó òò'÷1ð 1ð Ø÷`
ñ `
ó ó ÷`
ð `
r0   rÉ  )rú   )rÊ   )rž  r5  rÉ  )EÚcollections.abcr   Údataclassesr   r)   Útorch.nnrD   Úactivationsr   Úcache_utilsr   r   r   Ú
generationr	   Úmasking_utilsr
   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_moonshiner!   r$   rº   r<   r\   ri   r²   rÈ   rÐ   rÞ   rà   r  r%  r5  rL  r…  rž  rÇ  rÉ  Ú__all__r1   r0   r-   Ú<module>rø     s›  ðõ* %Ý !ã Ý å !ß CÑ CÝ )ß JÝ BÝ 9÷õ ÷ Lß FÝ &ß IÑ IÝ 0ß Gß EÝ 4ñ ðôð
 ô
/ /ó 
/ó óð
/ô˜"Ÿ)™)ô ô˜"Ÿ)™)ô ô 8<˜rŸy™yô 8<õv	U÷%ò26ô%ôPw)˜Ÿ™ô w)ôt.Ð6ô .ôbEÐ6ô EðP ô#˜ó #ó ð#ô0d
Ð/ô d
ðN ô_
Ð/ó _
ó ð_
ðD ôb
Ð-ó b
ó ðb
õJñ  ðôô
v
Ð(@À/ó v
óð
v
òr ^‚r0   