+
    œ°³j	 ã                   óŒ  € R t ^ RIt^ RIHt ^ RIHt ^ RIt^ RIHt ^RIH	t
 ^RIHt ^RIHtHt ^RIHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHt ^RIHt ^RIHtH t H!t!H"t" ^RI#H$t$ ^RI%H&t& ^RI'H(t( ]"PR                  ! ]*4      t+]!] ! R R]4      4       4       t, ! R R4      t-]!] ! R R]4      4       4       t.]!] ! R R]4      4       4       t/ ! R R]P`                  4      t1 ! R R]P`                  4      t2 ! R R]P`                  4      t3 ! R  R!]P`                  4      t4 ! R" R#]P`                  4      t5 ! R$ R%]P`                  4      t6R& t7RDR' lt8 ! R( R)]P`                  4      t9R* R+ lt:RER, R- llt; ! R. R/]P`                  4      t< ! R0 R1]4      t= ! R2 R3]P`                  4      t> ! R4 R5]P`                  4      t? ! R6 R7]P`                  4      t@ ! R8 R9]P`                  4      tA ! R: R;]P`                  4      tB ! R< R=]P`                  4      tC]! ! R> R?]4      4       tD]!! R@RA7       ! RB RC]D4      4       tERCR?.tFR# )FzPyTorch Mimi model.N)ÚCallable)Ú	dataclass)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Ú!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)Údeprecate_kwarg)Úmaybe_autocast©Ú
MimiConfigc                   ó@   a € ] tR t^)t o RtRtRtRtRtV 3R lt	Rt
V tR# )Ú
MimiOutputa  
audio_codes (`torch.LongTensor`  of shape `(batch_size, num_quantizers, codes_length)`, *optional*):
    Discrete code embeddings computed using `model.encode`.
audio_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Decoded audio values, obtained using the decoder part of Mimi.
encoder_past_key_values (`Cache`, *optional*):
    Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the encoder transformer.
    This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

    The model will output the same cache format that is fed as input.

    If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't
    have their past key value states given to this model).
decoder_past_key_values (`Cache`, *optional*):
    Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the decoder transformer.
    This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

    The model will output the same cache format that is fed as input.

    If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't
    have their past key value states given to this model).
Nc                óª   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[R,          ;R&   S[R,          ;R&   # )é   NÚaudio_codesÚaudio_valuesÚencoder_past_key_valuesÚdecoder_past_key_values)ÚtorchÚ
LongTensorÚFloatTensorr   )ÚformatÚ__classdict__s   "€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mimi/modeling_mimi.pyÚ__annotate__ÚMimiOutput.__annotate__)   sW   ø‡ ‚ ñ4 ×!Ñ! DÕ(Ñ/ñ5 ñ6 ×#Ñ# dÕ*Ñ1ñ7 ñ8 # T�\Ñ0ñ9 ñ: # T�\Ñ0ò; ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   r   r    Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r%   s   @r&   r   r   )   s)   ø‡ € ñð. ,0€KØ-1€LØ,0ÐØ,0Ð÷; ƒ r)   r   c                   óZ   a € ] tR t^It o RtV 3R lR ltV 3R lR ltV 3R lR ltRtV t	R	# )
ÚMimiConv1dPaddingCachea{  
Padding cache for MimiConv1d causal convolutions in order to support streaming via cache padding.
See: https://huggingface.co/papers/2005.06720 & https://huggingface.co/papers/2204.07064

A padding cache is a list of cached partial hidden states for each convolution layer.
Hidden states are cached from the previous call to the MimiConv1d forward pass, given the padding size.
c          	      ób   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[S[,          RS[S[ ,          /# )r   Ú
num_layersÚper_layer_paddingÚper_layer_padding_modeÚper_layer_in_channels)ÚintÚlistÚstr)r$   r%   s   "€r&   r'   Ú#MimiConv1dPaddingCache.__annotate__R   s?   ø€ ÷ 1ñ 1áð1ñ  ¡�9ð1ñ !%¡S¥	ð	1ñ
  $¡C�yñ1r)   c                óð   € \        V4      \        V4      \        V4      0p\        V4      ^8w  g   VP                  4       V8w  d   \        RV R24      hW n        W0n        W@n        R.V,          V n        R# )é   zExpected `num_layers` (zU) values in `per_layer_padding`, `per_layer_padding_mode` and `per_layer_in_channels`N)ÚlenÚpopÚ
ValueErrorr8   r9   r:   Úpadding_cache)Úselfr7   r8   r9   r:   Úfrom_args_num_layerss   &&&&& r&   Ú__init__ÚMimiConv1dPaddingCache.__init__R   s†   € ô !$Ð$5Ó 6¼Ð<RÓ8SÔUXÐYnÓUoÐpÐäÐ#Ó$¨Ô)Ð-A×-EÑ-EÓ-GÈ:Ô-UÜØ)¨*¨ð  6Kð  Lóð ð "3ÔØ&<Ô#Ø%:Ô"à"˜V jÕ0ˆÖr)   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# ©r   Úhidden_statesÚ	layer_idx©r!   ÚTensorr;   )r$   r%   s   "€r&   r'   r>   g   s   ø€ ÷ ñ ©¯©ð Á#ñ r)   c                óˆ  € VP                   ^ ,          VP                  VP                  rTpV P                  V,          V P                  V,          V P
                  V,          r‡pVR8X  d   \        P                  ! W8WeVR7      p	V	# VR8X  d*   \        P                  ! W8WeVR7      VR,          ,          p	V	# \        RV R24      h)a  
Initialize the cache for a specific layer.

Parameters:
    hidden_states (`torch.Tensor`):
        The hidden states to initialize the cache with.
    layer_idx (`int`):
        The index of the layer to initialize the cache for.
Returns:
    `torch.Tensor`, the initialized cache.
Úconstant)ÚdeviceÚdtypeÚ	replicatezPadding mode z not supported).:Nr@   N)
ÚshaperR   rQ   r8   r9   r:   r!   ÚzerosÚonesÚNotImplementedError)
rE   rK   rL   Ú
batch_sizerR   rQ   ÚpaddingÚpadding_modeÚin_channelsÚcurrent_caches
   &&&       r&   Ú_cache_initÚ"MimiConv1dPaddingCache._cache_initg   sÇ   € ð %2×$7Ñ$7¸Õ$:¸M×<OÑ<OÐQ^×QeÑQe˜6ˆ
à×"Ñ" 9Õ-Ø×'Ñ'¨	Õ2Ø×&Ñ& yÕ1ð  +ˆð ˜:Ô%Ü!ŸKšK¨
ÀÐ_dÔeˆMð Ðð ˜[Ô(ä—
’
˜:°GÐRWÔXÐ[hÐipÕ[qÕqð ð Ðô &¨°l°^À>Ð&RÓSÐSr)   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# rJ   rM   )r$   r%   s   "€r&   r'   r>   …   s   ø€ ÷ ñ ¡E§L¡Lð ¹Sñ r)   c                ó8  € VP                   ^ ,          VP                  VP                  rTpV P                  V,          V P                  V,          rvV P
                  V,          f   V P                  W4      pMV P
                  V,          pV^ 8”  dc   \        ^ WaP                   R,          ,
          4      p	V	^ 8”  d)   \        P                  ! VRRV	) R13,          V.RR7      p
M)VRRV) R13,          p
M\        P                  ! W7^ WER7      p
W P
                  V&   V# )ad  
Updates the padding cache with the new padding states for the layer `layer_idx` and returns the current cache.

Parameters:
    hidden_states (`torch.Tensor`):
        The hidden states to be partially cached.
    layer_idx (`int`):
        The index of the layer to cache the states for.
Returns:
    `torch.Tensor` or `None`, the current padding cache.
NºNNN©Údim©rR   rQ   éÿÿÿÿ)rT   rR   rQ   r8   r:   rD   r]   Úmaxr!   ÚcatÚempty)rE   rK   rL   rX   rR   rQ   rY   r[   r\   Ú	shortfallÚpadding_statess   &&&        r&   ÚupdateÚMimiConv1dPaddingCache.update…   s   € ð %2×$7Ñ$7¸Õ$:¸M×<OÑ<OÐQ^×QeÑQe˜6ˆ
Ø#×5Ñ5°iÕ@À$×B\ÑB\Ð]fÕBg�à×Ñ˜iÕ(Ò0Ø ×,Ñ,¨]ÓF‰Mà ×.Ñ.¨yÕ9ˆMð �QŒ;Ü˜A˜w×)<Ñ)<¸RÕ)@Õ@ÓAˆIØ˜1Œ}Ü!&§¢¨M¸!¸QÀÀ
ÁÐ:KÕ,LÈmÐ+\ÐbdÔ!e‘à!.¨q°!°g°X±Y¨Õ!?‘ä"Ÿ[š[¨À!È5Ô`ˆNà(6×Ñ˜9Ñ%ØÐr)   )rD   r:   r8   r9   N)
r+   r,   r-   r.   r/   rG   r]   rk   r1   r2   r3   s   @r&   r5   r5   I   s(   ø‡ € ñ÷1ð 1÷*ð ÷<ö r)   r5   c                   ó<   a € ] tR t^§t o RtRtRtRtV 3R ltRt	V t
R# )ÚMimiEncoderOutputa¨  
audio_codes (`torch.LongTensor`  of shape `(batch_size, num_quantizers, codes_length)`, *optional*):
    Discrete code embeddings computed using `model.encode`.
encoder_past_key_values (`Cache`, *optional*):
    Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the encoder transformer.
    This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

    The model will output the same cache format that is fed as input.

    If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't
    have their past key value states given to this model).
padding_cache (`MimiConv1dPaddingCache`, *optional*):
    Padding cache for MimiConv1d causal convolutions in order to support streaming via cache padding.
Nc                ó|   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[R,          ;R&   S[R,          ;R&   # )r   Nr   r   rD   )r!   r"   r   r5   )r$   r%   s   "€r&   r'   ÚMimiEncoderOutput.__annotate__§   s@   ø‡ ‚ ñ$ ×!Ñ! DÕ(Ñ/ñ% ñ& # T�\Ñ0ñ' ñ( *¨DÕ0Ñ7ò) r)   r*   )r+   r,   r-   r.   r/   r   r   rD   r0   r1   r2   r3   s   @r&   rn   rn   §   s#   ø‡ € ñð ,0€KØ,0ÐØ37€M÷) ƒ r)   rn   c                   ó8   a € ] tR t^¾t o RtRtRtV 3R ltRtV t	R# )ÚMimiDecoderOutputa  
audio_values (`torch.FloatTensor`  of shape `(batch_size, segment_length)`, *optional*):
    Decoded audio values, obtained using the decoder part of Mimi.
decoder_past_key_values (`Cache`, *optional*):
    Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the decoder transformer.
    This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

    The model will output the same cache format that is fed as input.

    If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't
    have their past key value states given to this model).
Nc                ób   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[R,          ;R&   # )r   Nr   r    )r!   r#   r   )r$   r%   s   "€r&   r'   ÚMimiDecoderOutput.__annotate__¾   s.   ø‡ ‚ ñ  ×#Ñ# dÕ*Ñ1ñ! ñ" # T�\Ñ0ò# r)   r*   )
r+   r,   r-   r.   r/   r   r    r0   r1   r2   r3   s   @r&   rr   rr   ¾   s   ø‡ € ñð .2€LØ,0Ð÷# ƒ r)   rr   c                   ó    a a€ ] tR t^Òt oRtRV3R lV 3R llltR tR tV3R lR lt]	RV3R lR	 ll4       t
V3R
 lR ltRR ltRtVtV ;t# )Ú
MimiConv1dz;Conv1d with asymmetric or causal padding and normalization.c                ól   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[ RS[R,          R	S[R
S[ R,          /	# )r   r[   Úout_channelsÚkernel_sizeÚstrideÚdilationÚgroupsÚpad_modeNÚbiasrL   )r;   r=   Úbool)r$   r%   s   "€r&   r'   ÚMimiConv1d.__annotate__Õ   sz   ø€ ÷ +Dñ +Dñ ð+Dñ ð	+Dñ
 ð+Dñ ð+Dñ ð+Dñ ð+Dñ ˜•*ð+Dñ ð+Dñ ˜•:ñ+Dr)   c           
     ó¾  <€ \         SV `  4        VP                  V n        Vf   VP                  MTV n        W n        W n        V^8”  d'   V^8”  d    \        P                  RV RV RV R24       \        P                  ! W#WEWgV	R7      V n        V P                  P                  ^ ,          p\        P                  ! V P                  P                  ^ ,          \        P                   R7      pV P                  P"                  ^ ,          p\        P                  ! V^,
          V,          ^,           \        P                   R7      p\        P$                  ! VRR7      V n        \        P$                  ! VRR7      V n        \        P$                  ! WE,
          RR7      V n        V P&                  ^,          V n        V P&                  V P(                  ,
          V n        R # )	NzNMimiConv1d has been initialized with stride > 1 and dilation > 1 (kernel_size=z stride=z, dilation=ú).)r{   r|   r~   ©rR   F©Ú
persistent)ÚsuperrG   Úuse_causal_convÚcausalr}   rL   r[   ÚloggerÚwarningr   ÚConv1dÚconvry   r!   Útensorrz   Úint64r{   ÚBufferÚpadding_totalÚpadding_rightÚpadding_left)rE   Úconfigr[   rx   ry   rz   r{   r|   r}   r~   rL   Ú	__class__s   &&&&&&&&&&&€r&   rG   ÚMimiConv1d.__init__Õ   sj  ø€ ô 	‰ÑÔØ×,Ñ,ˆŒØ+3Ò+;˜ŸšÀˆŒØ"ŒØ&Ôð �AŒ:˜( Qœ,Ü�N‰Nð!Ø!, ¨X°f°X¸[ÈÈ
ÐRTðVôô
 —I’IØ {ÀXÐcgô
ˆŒ	ð —i‘i×+Ñ+¨AÕ.ˆÜ—’˜dŸi™i×.Ñ.¨qÕ1¼¿¹ÔEˆØ—9‘9×%Ñ% aÕ(ˆô —l’l K°!¥O°xÕ#?À!Õ#CÌ5Ï;É;ÔWˆä—i’i °5Ô9ˆŒÜŸ9š9 [¸UÔCˆÔÜŸYšY {Õ';ÈÔNˆÔð "×/Ñ/°1Õ4ˆÔØ ×.Ñ.°×1CÑ1CÕCˆÖr)   c                óú   € \         P                  P                  p\        \         P                  P                  R 4      '       d%   \         P                  P                  P                  pV! V P
                  4       R# ©Úweight_normN©r   Úutilsr˜   ÚhasattrÚparametrizationsrŒ   ©rE   r˜   s   & r&   Úapply_weight_normÚMimiConv1d.apply_weight_norm  óI   € Ü—h‘h×*Ñ*ˆÜ”2—8‘8×,Ñ,¨m×<Ò<ÜŸ(™(×3Ñ3×?Ñ?ˆKá�D—I‘IÖr)   c                óX   € \         P                  P                  V P                  4       R # ©N©r   rš   Úremove_weight_normrŒ   ©rE   s   &r&   r¤   ÚMimiConv1d.remove_weight_norm	  ó   € Ü
�‰×#Ñ# D§I¡IÖ.r)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   rK   Úreturn©r!   rN   )r$   r%   s   "€r&   r'   r€     s'   ø€ ÷ 
%ñ 
%á—|‘|ð
%ñ 
�‰ñ
%r)   c                óŠ  € VP                   R,          pW P                  ,
          V P                  ,           V P                  ,          ^,           p\        P
                  ! V4      P                  \        P                  4      ^,
          pW0P                  ,          V P                  ,           V P                  ,
          pWB,
          # )zSee `pad_for_conv1d`.re   )rT   ry   r�   rz   r!   ÚceilÚtorŽ   )rE   rK   ÚlengthÚn_framesÚideal_lengths   &&   r&   Ú_get_extra_padding_for_conv1dÚ(MimiConv1d._get_extra_padding_for_conv1d  s‰   € ð
 ×$Ñ$ RÕ(ˆØ×-Ñ-Õ-°×0BÑ0BÕBÀdÇkÁkÕQÐTUÕUˆÜ—:’:˜hÓ'×*Ñ*¬5¯;©;Ó7¸!Õ;ˆØ§+¡+Õ-°×0@Ñ0@Õ@À4×CUÑCUÕUˆàÕ$Ð$r)   c                ó\   <€ V ^8„  d   QhRS[ P                  RS[S[S[3,          RS[RS[/# )r   rK   ÚpaddingsÚmodeÚvalue)r!   rN   Útupler;   r=   Úfloat)r$   r%   s   "€r&   r'   r€     s6   ø€ ÷ !ñ !™eŸl™lð !±e¹CÁ¸Hµoð !ÉSð !Ñbgñ !r)   c                ó”  € V P                   R,          pVw  rVVR8w  d!   \        P                  P                  WW#4      # \	        WV4      p^ pWG8:  d2   Wt,
          ^,           p\        P                  P                  V ^ V34      p \        P                  P                  WW#4      p	V	P                   R,          V,
          p
V	RRV
13,          # )zºTiny wrapper around torch.nn.functional.pad, just to allow for reflect padding on small input.
If this is the case, we insert extra 0 padding to the right before the reflection happens.
Úreflect.Nre   )rT   r   Ú
functionalÚpadrf   )rK   rµ   r¶   r·   r¯   r’   r‘   Úmax_padÚ	extra_padÚpaddedÚends   &&&&       r&   Ú_pad1dÚMimiConv1d._pad1d  s³   € ð ×$Ñ$ RÕ(ˆØ&.Ñ#ˆØ�9ÔÜ—=‘=×$Ñ$ ]¸dÓJÐJä�lÓ2ˆØˆ	ØÔØÕ(¨1Õ,ˆIÜŸM™M×-Ñ-¨m¸aÀ¸^ÓLˆMÜ—‘×"Ñ" =¸DÓHˆØ�l‰l˜2Õ Õ*ˆØ�c˜4˜C˜4�iÕ Ð r)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   Úinput_lengthrª   ©r!   r"   )r$   r%   s   "€r&   r'   r€   -  s'   ø€ ÷ ñ ©u×/?Ñ/?ð ÁE×DTÑDTñ r)   c                óR  € WP                   ,
          V P                  ,           V P                  ,          ^,           p\        P                  ! V4      P                  \        P                  4      ^,
          pW P                  ,          V P                   ,           V P                  ,
          pW1,
          pV P                  '       d   V P                  pTpMV P                  pV P                  V,           pW,           V,           pV^V P                  P                  ^ ,          ,          ,           V P                  P                  ^ ,          V P                  P                   ^ ,          ^,
          ,          ,
          ^,
          V P                  P                  ^ ,          ,          ^,           pV# )z4
Return the length of the output of the MimiConv1d.
)ry   r�   rz   r!   r­   r®   rŽ   rˆ   r’   r‘   rŒ   rY   r{   )rE   rÆ   r°   r±   Úextra_paddingr’   r‘   Úoutput_lengths   &&      r&   Ú_get_output_lengthÚMimiConv1d._get_output_length-  s7  € ð
 !×#3Ñ#3Õ3°d×6HÑ6HÕHÈDÏKÉKÕWÐZ[Õ[ˆÜ—:’:˜hÓ'×*Ñ*¬5¯;©;Ó7¸!Õ;ˆØ§+¡+Õ-°×0@Ñ0@Õ@À4×CUÑCUÕUˆØ$Õ3ˆà�;�;ˆ;Ø×-Ñ-ˆLØ)‰Mà×,Ñ,ˆLØ ×.Ñ.°Õ>ˆMð $Õ2°]ÕBˆð ˜1˜tŸy™y×0Ñ0°Õ3Õ3Õ3°d·i±i×6HÑ6HÈÕ6KÈtÏyÉy×OdÑOdÐefÕOgÐjkÕOkÕ6lÕlÐopÕpØ�Y‰Y×Ñ˜aÕ õ!à#$õ%ˆð Ðr)   c                ó  € V P                  V4      pV P                  '       g   Ve   \        R4      hV P                  '       d:   Ve6   VP                  WP                  4      p\
        P                  ! WA.^R7      pMvV P                  '       d+   V P                  WP                  V3V P                  R7      pM:V P                  WP                  V P                  V,           3V P                  R7      pV P                  V4      pV# )Nz=`padding_cache` is not supported for non-causal convolutions.rb   )r¶   )r²   rˆ   rC   rk   rL   r!   rg   rÂ   r�   r}   r’   r‘   rŒ   )rE   rK   rD   rÉ   Úlayer_padding_caches   &&&  r&   ÚforwardÚMimiConv1d.forwardG  sÞ   € Ø×:Ñ:¸=ÓIˆà�{�{ˆ{˜}Ò8ÜÐ\Ó]Ð]à�;�;ˆ;˜=Ò4Ø"/×"6Ñ"6°}ÇnÁnÓ"UÐÜ!ŸIšIÐ':Ð&JÐPQÔR‰Mà�[�[ˆ[à ŸK™K¨×8JÑ8JÈMÐ7ZÐae×anÑan˜KÓo‰Mð !ŸK™KØ× 1Ñ 1°4×3EÑ3EÈÕ3UÐVÐ]a×]jÑ]jð (ó ˆMð Ÿ	™	 -Ó0ˆØÐr)   )
rˆ   rŒ   r[   ry   rL   r}   r’   r‘   r�   rz   )r@   r@   r@   NTN)Úzeroç        r¢   )r+   r,   r-   r.   r/   rG   rž   r¤   r²   ÚstaticmethodrÂ   rË   rÏ   r1   r2   Ú__classcell__©r”   r%   s   @@r&   rv   rv   Ò   sV   ù‡ € ÙE÷+Dõ +DòZò/÷
%ð 
%ð ÷!ñ !ó ð!÷$ð ÷4ô r)   rv   c                   óX   a a€ ] tR tRt oRtR	V3R lV 3R llltR tR tR tRt	Vt
V ;t# )
ÚMimiConvTranspose1di^  zDConvTranspose1d with asymmetric or causal padding and normalization.c          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ /# )r   r[   rx   ry   rz   r|   )r;   )r$   r%   s   "€r&   r'   Ú MimiConvTranspose1d.__annotate__a  s?   ø€ ÷ "?ñ "?ñ ð"?ñ ð	"?ñ
 ð"?ñ ð"?ñ ñ"?r)   c           	     óJ  <€ \         S	V `  4        VP                  V n        VP                  V n        \
        P                  ! W#WEWgR 7      V n        V P                  '       g   V P                  R8X  g   \        R4      hV P                  P                  ^ ,          pV P                  P                  ^ ,          pWE,
          pV P                  '       d-   \        P                  ! W€P                  ,          4      V n        MV^,          V n        W€P                  ,
          V n        R# ))r|   r~   ç      ð?zB`trim_right_ratio` != 1.0 only makes sense for causal convolutionsN)r†   rG   r‡   rˆ   Útrim_right_ratior   ÚConvTranspose1drŒ   rC   ry   rz   Úmathr­   r‘   r’   )
rE   r“   r[   rx   ry   rz   r|   r~   r�   r”   s
   &&&&&&&& €r&   rG   ÚMimiConvTranspose1d.__init__a  sÛ   ø€ ô 	‰ÑÔØ×,Ñ,ˆŒØ &× 7Ñ 7ˆÔÜ×&Ò& {À+Ð^dÔpˆŒ	à——�˜t×4Ñ4¸Ô;ÜÐaÓbÐbà—i‘i×+Ñ+¨AÕ.ˆØ—‘×!Ñ! !Õ$ˆØ#Õ,ˆð �;�;ˆ;ô "&§¢¨=×;PÑ;PÕ+PÓ!QˆDÕð "/°!Õ!3ˆDÔà)×,>Ñ,>Õ>ˆÖr)   c                óú   € \         P                  P                  p\        \         P                  P                  R 4      '       d%   \         P                  P                  P                  pV! V P
                  4       R# r—   r™   r�   s   & r&   rž   Ú%MimiConvTranspose1d.apply_weight_norm…  r    r)   c                óX   € \         P                  P                  V P                  4       R # r¢   r£   r¥   s   &r&   r¤   Ú&MimiConvTranspose1d.remove_weight_normŒ  r§   r)   c                óž   € V P                  V4      pVP                  R,          V P                  ,
          pVRV P                  V13,          pV# )r@   .re   )rŒ   rT   r‘   r’   )rE   rK   rÁ   s   && r&   rÏ   ÚMimiConvTranspose1d.forward�  sM   € ØŸ	™	 -Ó0ˆð ×!Ñ! "Õ%¨×(:Ñ(:Õ:ˆØ% c¨4×+<Ñ+<¸sÐ+BÐ&BÕCˆØÐr)   )rˆ   rŒ   r’   r‘   rÜ   )r@   r@   T)r+   r,   r-   r.   r/   rG   rž   r¤   rÏ   r1   r2   rÔ   rÕ   s   @@r&   r×   r×   ^  s'   ù‡ € ÙN÷"?õ "?òHò/÷ò r)   r×   c                   óL   a a€ ] tR tRt oRtV3R lV 3R lltRR ltRtVtV ;t	# )ÚMimiResnetBlocki˜  z3
Residual block from SEANet model as used by Mimi.
c                ó<   <€ V ^8„  d   QhRS[ RS[RS[S[,          /# )r   r“   rc   Ú	dilations)r   r;   r<   )r$   r%   s   "€r&   r'   ÚMimiResnetBlock.__annotate__�  s&   ø€ ÷ *ñ *™zð *±ð *ÁÁSÅ	ñ *r)   c           
     óZ  <€ \         SV `  4        VP                  ^3p\        V4      \        V4      8w  d   \	        R4      hW!P
                  ,          p. p\        \        WC4      4       Fa  w  pw  r‰V^ 8X  d   TMTp
V\        V4      ^,
          8X  d   TMTpV\        P                  ! 4       .,          pV\        WW¸V	R7      .,          pKc  	  \        P                  ! V4      V n        VP                  '       d   \        WV^R7      V n        R# \        P                  ! 4       V n        R# )r@   z7Number of kernel sizes should match number of dilations)r{   )ry   N)r†   rG   Úresidual_kernel_sizerA   rC   ÚcompressÚ	enumerateÚzipr   ÚELUrv   Ú
ModuleListÚblockÚuse_conv_shortcutÚshortcutÚIdentity)rE   r“   rc   ré   Úkernel_sizesÚhiddenrò   Úiry   r{   Úin_chsÚout_chsr”   s   &&&&        €r&   rG   ÚMimiResnetBlock.__init__�  sò   ø€ Ü‰ÑÔØ×3Ñ3°QÐ7ˆÜˆ|Ó¤ I£Ô.ÜÐVÓWÐWàŸ™Õ'ˆØˆÜ*3´C¸Ó4PÖ*QÑ&ˆAÑ&�Ø œF‘S¨ˆFØ¤# lÓ"3°aÕ"7Ô7‘c¸VˆGØ”b—f’f“h�ZÕˆEØ”j °ÐPXÔYÐZÕZŠEñ	 +Rô
 —]’] 5Ó)ˆŒ
à×#×#Ð#Ü& v°CÀQÔGˆDŽMäŸKšK›MˆDŽMr)   c                ó$  € TpV P                    F,  p\        V\        4      '       d   V! WR 7      pK$  V! V4      pK.  	  \        V P                  \        4      '       d   V P                  W2R 7      pW1,           # V P                  V4      pW1,           # ©©rD   )rò   Ú
isinstancerv   rô   )rE   rK   rD   ÚresidualÚlayers   &&&  r&   rÏ   ÚMimiResnetBlock.forward±  s}   € Ø ˆà—Z”ZˆEÜ˜%¤×,Ò,Ù % mÔ Q’á % mÓ 4’ñ	  ô �d—m‘m¤Z×0Ò0Ø—}‘} X�}ÓKˆHð Õ'Ð'ð —}‘} XÓ.ˆHàÕ'Ð'r)   )rò   rô   r¢   ©
r+   r,   r-   r.   r/   rG   rÏ   r1   r2   rÔ   rÕ   s   @@r&   rç   rç   ˜  s   ù‡ € ñ÷*ó *÷((ô (r)   rç   c                   óL   a a€ ] tR tRt oRtV3R lV 3R lltRR ltRtVtV ;t	# )ÚMimiEncoderiÂ  zSEANet encoder as used by Mimi.c                ó    <€ V ^8„  d   QhRS[ /# ©r   r“   r   )r$   r%   s   "€r&   r'   ÚMimiEncoder.__annotate__Å  s   ø€ ÷ 8ñ 8™zñ 8r)   c           
     ó4  <€ \         SV `  4        \        WP                  VP                  VP
                  4      .p^pR.p\        VP                  4       Fí  pW1P                  ,          p\        VP                  4       FW  pVP                  R\        V4       R2R\        V4       R2.4       V\        WVP                  V,          ^.4      .,          pKY  	  V\        P                  ! 4       .,          pVP!                  R\        V4       24       V\        WV^,          V^,          VR7      .,          pV^,          pKï  	  V\        P                  ! 4       .,          pVP!                  R\        V4       24       V\        WVP                  ,          VP"                  VP$                  4      .,          p\        P&                  ! V4      V n        W@n        \-        V P*                  4       F#  w  r‰V P/                  V	4      p
\1        V
RV4       K%  	  R# )r@   zlayers.0zlayers.z.block.1z.block.3©ry   rz   rL   N)r†   rG   rv   Úaudio_channelsÚnum_filtersry   ÚreversedÚupsampling_ratiosÚrangeÚnum_residual_layersÚextendrA   rç   Údilation_growth_rater   rð   ÚappendÚhidden_sizeÚlast_kernel_sizerñ   ÚlayersÚ_mimiconv1d_layer_namesrî   Úget_submoduleÚsetattr)rE   r“   ÚmodelÚscalingÚmimiconv1d_layer_namesÚratioÚcurrent_scaleÚjrL   Ú	layernameÚ
conv_layerr”   s   &&         €r&   rG   ÚMimiEncoder.__init__Å  sÖ  ø€ Ü‰ÑÔÜ˜F×$9Ñ$9¸6×;MÑ;MÈv×OaÑOaÓbÐcˆØˆð #- Ðô ˜f×6Ñ6Ö7ˆEØ#×&8Ñ&8Õ8ˆMä˜6×5Ñ5Ö6�Ø&×-Ñ-°¼¸U»¸ÀHÐ/MÐQXÔY\Ð]bÓYcÐXdÐdlÐOmÐ.nÔoØœ/¨&À&×B]ÑB]Ð_`ÕB`ÐbcÐAdÓeÐfÕf’ñ 7ð ”b—f’f“h�ZÕˆEØ"×)Ñ)¨G´C¸³J°<Ð*@ÔAØ”j ¸ÈÕ8IÐW\Ð_`ÕW`ÐinÔoÐpÕpˆEØ�q�LŠGñ 8ð 	”"—&’&“(�ÕˆØ×%Ñ%¨´°E³
¨|Ð&<Ô=Ø”*˜V¨v×/AÑ/AÕ%AÀ6×CUÑCUÐW]×WnÑWnÓoÐpÕpˆä—m’m EÓ*ˆŒØ'=Ô$ô %.¨d×.JÑ.JÖ$KÑ ˆIØ×+Ñ+¨IÓ6ˆJÜ�J ¨YÖ7ó %Lr)   c                óŠ   € V P                    F2  p\        V\        \        34      '       d   V! WR 7      pK*  V! V4      pK4  	  V# rý   )r  rÿ   rv   rç   )rE   rK   rD   r  s   &&& r&   rÏ   ÚMimiEncoder.forwardæ  s>   € Ø—[”[ˆEÜ˜%¤*¬oÐ!>×?Ò?Ù % mÔ Q’á % mÓ 4’ñ	 !ð
 Ðr)   )r  r  r¢   r  rÕ   s   @@r&   r  r  Â  s   ù‡ € Ù)÷8ó 8÷Bô r)   r  c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚMimiLayerScaleiï  z¡Layer scale from [Touvron et al 2021] (https://huggingface.co/papers/2103.17239).
This rescales diagonally the residual outputs close to 0, with a learnt scale.
c                óº   <€ \         SV `  4        VP                  pVP                  p\        P
                  ! \        P                  ! V3VR R7      4      V n        R# )T)Úrequires_gradN)	r†   rG   r  Úlayer_scale_initial_scaler   Ú	Parameterr!   ÚfullÚscale)rE   r“   ÚchannelsÚinitial_scaler”   s   &&  €r&   rG   ÚMimiLayerScale.__init__ô  sD   ø€ Ü‰ÑÔØ×%Ñ%ˆØ×8Ñ8ˆÜ—\’\¤%§*¢*¨h¨[¸-ÐW[Ô"\Ó]ˆŽ
r)   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   Úxr«   )r$   r%   s   "€r&   r'   ÚMimiLayerScale.__annotate__ú  s   ø€ ÷ ñ ™Ÿ™ñ r)   c                ó(   € V P                   V,          # r¢   ©r,  )rE   r1  s   &&r&   rÏ   ÚMimiLayerScale.forwardú  s   € Ø�z‰z˜A�~Ðr)   r4  r  rÕ   s   @@r&   r&  r&  ï  s   ù‡ € ñõ^÷÷ ð r)   r&  c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚMimiRotaryEmbeddingiÿ  rQ   z5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# r  r   )r$   r%   s   "€r&   r'   Ú MimiRotaryEmbedding.__annotate__  s   ø€ ÷ Oñ O™zñ Or)   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultFr„   N)r†   rG   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr“   Úrope_parametersr<  Úcompute_default_rope_parametersr   Úattention_scalingr   r�   Úinv_freqÚcloneÚoriginal_inv_freq)rE   r“   rQ   Úrope_init_fnrD  r”   s   &&&  €r&   rG   ÚMimiRotaryEmbedding.__init__   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr)   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r   r“   rª   )r   r¸   r!   rN   r¹   )r$   r%   s   "€r&   r'   r:    s.   ø€ ÷ 5ñ 5±
ð 5ÑV[Ñ\a×\hÑ\hÑjoÐ\oÕVpñ 5r)   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNrÛ   rƒ   )rA  Úgetattrr  Únum_attention_headsr!   Úaranger¹   r®   )r“   rQ   ÚkwargsÚbaserc   Úattention_factorrD  s   &&,    r&   rB  Ú3MimiRotaryEmbedding.compute_default_rope_parameters  s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r)   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)Nrd   ÚmpsÚcpuF)Údevice_typeÚenabledrb   rƒ   )Nra   Nre   )ra   Nra   )rD  ÚexpandrT   r®   r!   r¹   rQ   rÿ   Útyper=   r   Ú	transposerg   ÚcosrC  ÚsinrR   )
rE   r1  Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrW  ÚfreqsÚembr\  r]  s
   &&&       r&   rÏ   ÚMimiRotaryEmbedding.forward%  s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rC  r“   rD  r?  rF  r@  r<  r¢   )r+   r,   r-   r.   r   rG   rÓ   rB  r!   Úno_gradr   rÏ   r1   r2   rÔ   rÕ   s   @@r&   r7  r7  ÿ  su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r)   r7  c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrb   re   )rT   r!   rg   )r1  Úx1Úx2s   &  r&   Úrotate_halfrh  9  sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r)   c                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerh  )ÚqÚkr\  r]  Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r&   Úapply_rotary_pos_embrp  A  sS   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr)   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMimiMLPiZ  c                ó0  <€ \         SV `  4        Wn        \        VP                  ,          V n        \        P                  ! VP                  VP                  R R7      V n
        \        P                  ! VP                  VP                  R R7      V n        R# )F©r~   N)r†   rG   r“   r   Ú
hidden_actÚactivation_fnr   ÚLinearr  Úintermediate_sizeÚfc1Úfc2©rE   r“   r”   s   &&€r&   rG   ÚMimiMLP.__init__[  sh   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Õ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÐPUÔVˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÐPUÔVˆŽr)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r©   r«   )r$   r%   s   "€r&   r'   ÚMimiMLP.__annotate__c  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r)   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r¢   )ry  rv  rz  )rE   rK   s   &&r&   rÏ   ÚMimiMLP.forwardc  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr)   )rv  r“   ry  rz  ©	r+   r,   r-   r.   rG   rÏ   r1   r2   rÔ   rÕ   s   @@r&   rr  rr  Z  s   ù‡ € õW÷÷ ð r)   rr  c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r   rK   Ún_reprª   rM   )r$   s   "r&   r'   r'   k  s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur)   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)ra   ra   Nra   ra   )rT   rY  Úreshape)rK   rƒ  ÚbatchÚnum_key_value_headsÚslenrL  s   &&    r&   Ú	repeat_kvr‰  k  sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr)   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r   ÚmoduleÚqueryÚkeyr·   Úattention_maskNr  ÚdropoutrP  )r   ÚModuler!   rN   r¹   r   r   )r$   s   "r&   r'   r'   x  sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r)   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r   )rc   rR   )ÚpÚtrainingre   )r‰  Únum_key_value_groupsr!   Úmatmulr[  r   r¼   ÚsoftmaxÚfloat32r®   rR   r�  r“  Ú
contiguous)r‹  rŒ  r�  r·   rŽ  r  r�  rP  Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r&   Úeager_attention_forwardr�  x  sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r)   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚMimiAttentioni‘  z=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r   r“   rL   N©r   r;   )r$   r%   s   "€r&   r'   ÚMimiAttention.__annotate__”  s   ø€ ÷ 4ñ 4™zð 4±c¸Dµjñ 4r)   c                óÌ  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n	        V P                  V P                  ,          V n
        VP                  V n        R V n        ^\        P                  ! VP                  4      ,          V n        V P
                  V P                  ,          ^ 8w  d'   \!        RV P
                   RV P                   R24      h\"        P$                  ! V P
                  V P                  V P                  ,          VP&                  R7      V n        \"        P$                  ! V P
                  V P                  V P                  ,          VP&                  R7      V n        \"        P$                  ! V P
                  V P                  V P                  ,          VP&                  R7      V n        \"        P$                  ! V P                  V P                  ,          V P
                  VP&                  R7      V n        VP0                  V n        R# )Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: r‚   rt  N)r†   rG   r“   rL   Úattention_dropoutr  rN  Ú	num_headsrL  r‡  r”  r>  Ú	is_causalrÞ   Úsqrtr  rC   r   rw  Úattention_biasÚq_projÚk_projÚv_projÚo_projÚsliding_window©rE   r“   rL   r”   s   &&&€r&   rG   ÚMimiAttention.__init__”  s±  ø€ Ü‰ÑÔØŒØ"Œà!'×!9Ñ!9ˆÔØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØŸ™ˆŒØ#)×#=Ñ#=ˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø'-×'EÑ'EˆÔ$ØˆŒØœ4Ÿ9š9 V§_¡_Ó5Õ5ˆŒà×Ñ˜dŸn™nÕ,°Ô1ÜØQÐRV×RbÑRbÐQcØ$ T§^¡^Ð$4°Bð8óð ô
 —i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW]×WlÑWlÔmˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐag×avÑavÔwˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐag×avÑavÔwˆŒÜ—i’i §¡°·±Õ >À×@PÑ@PÐW]×WlÑWlÔmˆŒØ$×3Ñ3ˆÖr)   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[ P                  S[ P                  3,          /# )r   rK   rŽ  NÚpast_key_valuesÚposition_embeddingsrª   )r!   rN   r   r¸   )r$   r%   s   "€r&   r'   r¢  ¯  sw   ø€ ÷ ')ñ ')á—|‘|ð')ñ Ÿ™ tÕ+ð')ñ  �ð	')ñ
 #¡5§<¡<±·±Ð#=Õ>ÀÕEð')ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ')r)   c           	     ó(  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  RV P$                  /VB w  rïVP&                  ! . VORN5!  P)                  4       pV P+                  V4      pWï3# )Nr�  rÒ   r  r­  re   )rT   rL  r©  Úviewr[  rª  r«  rp  rk   rL   r   Úget_interfacer“   Ú_attn_implementationr�  r“  r¤  r  r­  r…  r˜  r¬  )rE   rK   rŽ  r±  r²  rP  Úinput_shapeÚhidden_shapeÚquery_statesr™  rš  r\  r]  Úattention_interfacerœ  r›  s   &&&&&,          r&   rÏ   ÚMimiAttention.forward¯  s—  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r)   )r¤  r“   rL  r  r¦  rª  rL   r>  r¥  r”  r‡  r¬  r©  r  r­  r«  r¢   ©NNNr  rÕ   s   @@r&   rŸ  rŸ  ‘  s!   ù‡ € ÙG÷4õ 4÷6')÷ ')ò ')r)   rŸ  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMimiTransformerLayeriÙ  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r“   rL   r¡  )r$   r%   s   "€r&   r'   Ú!MimiTransformerLayer.__annotate__Ú  s   ø€ ÷ 
6ñ 
6™zð 
6±cñ 
6r)   c                óŒ  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        P                  ! VP                  VP                  R7      V n
        \        P                  ! VP                  VP                  R7      V n        \        V4      V n        \        V4      V n        R# ))r“   rL   )ÚepsN)r†   rG   r  rŸ  Ú	self_attnrr  Úmlpr   Ú	LayerNormÚnorm_epsÚinput_layernormÚpost_attention_layernormr&  Úself_attn_layer_scaleÚmlp_layer_scaler®  s   &&&€r&   rG   ÚMimiTransformerLayer.__init__Ú  s‡   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä&¨fÔJˆŒä˜6“?ˆŒÜ!Ÿ|š|¨F×,>Ñ,>ÀFÇOÁOÔTˆÔÜ(*¯ª°V×5GÑ5GÈVÏ_É_Ô(]ˆÔ%Ü%3°FÓ%;ˆÔ"Ü-¨fÓ5ˆÖr)   c                óf  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[ P
                  S[S[ P
                  S[ P
                  3,          R,          3,          /# )	r   rK   rŽ  Nr±  Úoutput_attentionsÚ	use_cacher²  rª   )r!   rN   r   r   r¸   r#   )r$   r%   s   "€r&   r'   rÀ  æ  s­   ø€ ÷ %ñ %á—|‘|ð%ñ Ÿ™ tÕ+ð%ñ  �ð	%ñ
   $�;ð%ñ ˜$•;ð%ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð%ñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñ%r)   c                ó8  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW€P                  V4      ,           pTpV P                  V4      pV P	                  V4      pW€P                  V4      ,           pV3p
V'       d
   W©3,          p
V
# )rK   rŽ  r±  rÍ  rÎ  r²  r*   )rÇ  rÃ  rÉ  rÈ  rÄ  rÊ  )rE   rK   rŽ  r±  rÍ  rÎ  r²  rP  r   Úself_attn_weightsÚoutputss   &&&&&&&,   r&   rÏ   ÚMimiTransformerLayer.forwardæ  sÖ   € ð !ˆà×,Ñ,¨]Ó;ˆð ,0¯>ª>ñ ,
Ø'ð,
à)ð,
ð ,ð,
ð 0ð	,
ð
  ð,
ð !4ð,
ð ñ,
Ñ(ˆð !×#=Ñ#=¸mÓ#LÕLˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ ×#7Ñ#7¸Ó#FÕFˆà Ð"ˆçØÐ+Õ+ˆGàˆr)   )r  rÇ  rÄ  rÊ  rÈ  rÃ  rÉ  )NNFFNr�  rÕ   s   @@r&   r¾  r¾  Ù  s   ù‡ € ÷
6ó 
6÷%÷ %ò %r)   r¾  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚMimiTransformerModeli  z�
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MimiTransformerLayer`]

Args:
    config: MimiConfig
c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r$   r%   s   "€r&   r'   Ú!MimiTransformerModel.__annotate__  s   ø€ ÷ 	ñ 	™zñ 	r)   c           	     óö   <€ \         SV `  4        \        P                  ! \	        VP
                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        V4      V n	        R V n
        Wn        R# u upi )FN)r†   rG   r   rñ   r  Únum_hidden_layersr¾  r  r7  Ú
rotary_embÚgradient_checkpointingr“   r®  s   && €r&   rG   ÚMimiTransformerModel.__init__  sd   ø€ Ü‰ÑÔä—m’mÜFKÈF×LdÑLdÔFeÓfÑFe¸Ô! &Ö4ÑFeÑfó
ˆŒô .¨fÓ5ˆŒà&+ˆÔ#ØŽùò gs   ·A6c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# )r   rK   NrŽ  r^  r±  rÎ  rÍ  Úoutput_hidden_statesÚreturn_dictrª   )r!   r"   rN   r   r   r¸   r   )r$   r%   s   "€r&   r'   rÖ  !  s©   ø€ ÷ 
ñ 
á×'Ñ'¨$Õ.ð
ñ Ÿ™ tÕ+ð
ñ ×&Ñ&¨Õ-ð	
ñ
  �ð
ñ ˜$•;ð
ñ   $�;ð
ñ # T�kð
ñ ˜D•[ð
ñ 
Ñ(Õ	(ñ
r)   c	                óZ  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pV P
                  '       d2   V P                  '       d    V'       d   \        P                  R4       RpV'       d   Vf   \        V P                   R7      pVfb   Ve   VP                  4       M^ p
\        P                  ! VP                  ^,          VP                  R7      V
,           pVP                  ^ 4      p\!        V P                   VVVVR7      pV P#                  W4      pV'       d   R
MRpV'       d   R
MRpV P$                   FG  pV'       d
   WÑ3,          pV! VVVVVVVR7      pV^ ,          pV'       g   K6  VV^,          3,          pKI  	  V'       d
   WÑ3,          pV'       g1   \&        ;QJ d    . R WWÞ3 4       F  NK  	  5# ! R WWÞ3 4       4      # \)        VVVVR	7      # )aL
  
Args:
    hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
        Embedded representation that will be contextualized by the model
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)

        Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
        [`PreTrainedTokenizer.__call__`] for details.

        If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
        `past_key_values`).

        If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
        and modify to your needs. See diagram 1 in [the paper](https://huggingface.co/papers/1910.13461) for more
        information on the default strategy.

        - 1 indicates the head is **not masked**,
        - 0 indicates the head is **masked**.
    position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
        Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
        config.n_positions - 1]`.

        [What are position IDs?](../glossary#position-ids)
    past_key_values (`Cache`, *optional*):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
        have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
        of shape `(batch_size, sequence_length)`.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
        `past_key_values`).
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
        tensors for more detail.
    output_hidden_states (`bool`, *optional*):
        Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
        more detail.
    return_dict (`bool`, *optional*):
        Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
NzX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)r“   ©rQ   )r“   Úinputs_embedsrŽ  r±  r^  )rŽ  r^  r±  rÍ  rÎ  r²  c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5ir¢   r*   )Ú.0Úvs   & r&   Ú	<genexpr>Ú/MimiTransformerModel.forward.<locals>.<genexpr>—  s   é € ð Ù^�a—’Ó^ùs   ‚Œ
)Úlast_hidden_stater±  rK   Ú
attentionsr*   )r“   rÍ  rÝ  rÎ  rÞ  rÚ  r“  r‰   Úwarning_oncer   Úget_seq_lengthr!   rO  rT   rQ   rj  r	   rÙ  r  r¸   r   )rE   rK   rŽ  r^  r±  rÎ  rÍ  rÝ  rÞ  rP  Úpast_seen_tokensÚcausal_maskr²  Úall_hidden_statesÚall_self_attnsÚdecoder_layerÚlayer_outputss   &&&&&&&&&,       r&   rÏ   ÚMimiTransformerModel.forward!  s  € ðv 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	à%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆà×&×&Ð&¨4¯=¯=¨=¿YÜ×ÑØjôð ˆIç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä7Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð #Ÿo™o¨mÓJÐ÷ #7™B¸DÐß0™°dˆà!Ÿ[œ[ˆMß#Ø!Ð%5Õ5Ð!á)ØØ*Ø)Ø /Ø"3Ø#Ø$7ôˆMð *¨!Õ,ˆMç Ñ Ø =°Õ#3Ð"5Õ5’ñ# )÷(  ØÐ!1Õ1Ðçß”5ñ Ø)Ð<MÑ^ó—5ð �5ñ Ø)Ð<MÑ^óó ð ô 'Ø+Ø+Ø+Ø%ô	
ð 	
r)   )r“   rÚ  r  rÙ  )NNNNNNNNr  rÕ   s   @@r&   rÔ  rÔ    s&   ù‡ € ñ÷	ó 	÷
÷ 
ò 
r)   rÔ  c                   óH   a a€ ] tR tRt oRtV3R lV 3R lltR tRtVtV ;t	# )ÚMimiDecoderi£  zSEANet decoder as used by Mimi.c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r$   r%   s   "€r&   r'   ÚMimiDecoder.__annotate__¦  s   ø€ ÷ +ñ +™zñ +r)   c           
     ó  <€ \         SV `  4        \        ^\        VP                  4      ,          4      p\        WP                  W!P                  ,          VP                  4      .pVP                   Fª  pW!P                  ,          pV\        P                  ! 4       .,          pV\        WV^,          V^,          VR7      .,          p\        VP                  4       F1  pV\        W^,          VP                  V,          ^34      .,          pK3  	  V^,          pK¬  	  V\        P                  ! 4       .,          pV\        WP                  VP                   VP"                  4      .,          p\        P$                  ! V4      V n        R# )r   r
  N)r†   rG   r;   rA   r  rv   r  r  ry   r   rð   r×   r  r  rç   r  r  r  rñ   r  )rE   r“   r  r  r  r  r  r”   s   &&     €r&   rG   ÚMimiDecoder.__init__¦  sC  ø€ Ü‰ÑÔÜ�aœ3˜v×7Ñ7Ó8Õ8Ó9ˆÜ˜F×$6Ñ$6¸×BTÑBTÕ8TÐV\×VhÑVhÓiÐjˆð ×-Ô-ˆEØ#×&8Ñ&8Õ8ˆMà”b—f’f“h�ZÕˆEØÜ# F¸=ÈAÕ;MÐ[`ÐcdÕ[dÐmrÔsðõ ˆEô ˜6×5Ñ5Ö6�Øœ/¨&À1Õ2DÀv×GbÑGbÐdeÕGeÐghÐFiÓjÐkÕk’ñ 7à˜�MŠGñ .ð 	”"—&’&“(�ÕˆØ”*˜V×%7Ñ%7¸×9NÑ9NÐPV×PgÑPgÓhÐiÕiˆÜ—m’m EÓ*ˆŽr)   c                ó<   € V P                    F  pV! V4      pK  	  V# r¢   ©r  )rE   rK   r  s   && r&   rÏ   ÚMimiDecoder.forward¾  s    € Ø—[”[ˆEÙ! -Ó0ŠMñ !àÐr)   rù  r  rÕ   s   @@r&   ró  ró  £  s   ù‡ € Ù)÷+ó +÷0ò r)   ró  c                   ót   a a€ ] tR tRt oRtRV3R lV 3R lllt]V3R lR l4       tR tR t	R	 t
R
tVtV ;t# )ÚMimiEuclideanCodebookiÄ  z!Codebook with Euclidean distance.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r“   Úepsilon)r   r¹   )r$   r%   s   "€r&   r'   Ú"MimiEuclideanCodebook.__annotate__Ç  s   ø€ ÷ 
ñ 
™zð 
±Eñ 
r)   c                óÞ  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      pVP                  V n        \        P                  ! \        P                  ! R .\        P                  R7      4      V n
        \        P                  ! \        P                  ! VP                  4      4      V n        \        P                  ! V4      V n        RV n        W n        R# )Trƒ   N)r†   rG   r!   rU   Úcodebook_sizeÚcodebook_dimr   r�   r�   r—  ÚinitializedrV   Úcluster_usageÚ	embed_sumÚ_embedrþ  )rE   r“   rþ  Úembedr”   s   &&& €r&   rG   ÚMimiEuclideanCodebook.__init__Ç  s–   ø€ Ü‰ÑÔÜ—’˜F×0Ñ0°&×2EÑ2EÓFˆà#×1Ñ1ˆÔäŸ9š9¤U§\¢\°4°&ÄÇÁÔ%NÓOˆÔÜŸYšY¤u§z¢z°&×2FÑ2FÓ'GÓHˆÔÜŸš 5Ó)ˆŒØˆŒØŽr)   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rª   r«   )r$   r%   s   "€r&   r'   rÿ  Ô  s   ø€ ÷ ñ ‘u—|‘|ñ r)   c                ó¼   € V P                   fD   V P                  V P                  P                  V P                  R7      R,          ,          V n         V P                   # )N)Úmin)ra   N)r  r  r  Úclamprþ  r¥   s   &r&   r  ÚMimiEuclideanCodebook.embedÓ  sD   € à�;‰;ÒØŸ.™.¨4×+=Ñ+=×+CÑ+CÈÏÉÐ+CÓ+UÐV]Õ+^Õ^ˆDŒKØ�{‰{Ðr)   c                óÒ   € \         P                  ! VR ,          P                  4       V P                  R ,          P                  4       ^R7      ^ ,          pVP	                  RR7      pV# )N)r’  rb   re   )r!   Úcdistr¹   r  Úargmin)rE   rK   ÚdistsÚ	embed_inds   &&  r&   ÚquantizeÚMimiEuclideanCodebook.quantizeÙ  sS   € ô —’˜M¨$Õ/×5Ñ5Ó7¸¿¹ÀDÕ9I×9OÑ9OÓ9QÐUVÔWÐXYÕZˆØ—L‘L R�LÓ(ˆ	ØÐr)   c                óš   € VP                   pVP                  RVR,          34      pV P                  V4      pVP                  ! VRR !  pV# )r@   Nre   )rT   r…  r  r´  )rE   rK   rT   r  s   &&  r&   ÚencodeÚMimiEuclideanCodebook.encodeá  sM   € Ø×#Ñ#ˆà%×-Ñ-¨r°5¸µ9¨oÓ>ˆà—M‘M -Ó0ˆ	à—N’N E¨#¨2 JÑ/ˆ	ØÐr)   c                óX   € \         P                  P                  WP                  4      pV# r¢   )r   r¼   Ú	embeddingr  ©rE   r  r  s   && r&   ÚdecodeÚMimiEuclideanCodebook.decodeì  s   € Ü—=‘=×*Ñ*¨9·j±jÓAˆØˆr)   )r  r  r  r  rþ  r  )gñhãˆµøä>)r+   r,   r-   r.   r/   rG   Úpropertyr  r  r  r  r1   r2   rÔ   rÕ   s   @@r&   rü  rü  Ä  s:   ù‡ € Ù+÷
õ 
ð ÷ó ðò
ò÷ò r)   rü  c                   óN   a a€ ] tR tRt oRtV3R lV 3R lltR tR tRtVt	V ;t
# )ÚMimiVectorQuantizationiò  zQ
Vector quantization implementation. Currently supports only euclidean distance.
c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r$   r%   s   "€r&   r'   Ú#MimiVectorQuantization.__annotate__÷  s   ø€ ÷ 6ñ 6™zñ 6r)   c                óD   <€ \         SV `  4        \        V4      V n        R # r¢   )r†   rG   rü  Úcodebookr{  s   &&€r&   rG   ÚMimiVectorQuantization.__init__÷  s   ø€ Ü‰ÑÔÜ-¨fÓ5ˆŽr)   c                ób   € VP                  ^ ^^4      pV P                  P                  V4      pV# ©é    )Úpermuter#  r  )rE   rK   Úembed_ins   && r&   r  ÚMimiVectorQuantization.encodeû  s/   € Ø%×-Ñ-¨a°°AÓ6ˆØ—=‘=×'Ñ'¨Ó6ˆØˆr)   c                ób   € V P                   P                  V4      pVP                  ^ ^^4      pV# r&  )r#  r  r(  r  s   && r&   r  ÚMimiVectorQuantization.decode   s/   € Ø—=‘=×'Ñ'¨	Ó2ˆØ×#Ñ# A q¨!Ó,ˆØˆr)   )r#  ©r+   r,   r-   r.   r/   rG   r  r  r1   r2   rÔ   rÕ   s   @@r&   r  r  ò  s#   ù‡ € ñ÷6ó 6ò÷
ò r)   r  c                   ón   a a€ ] tR tRt oRtR
V3R lV 3R llltR
V3R lR lltV3R lR ltR	tVt	V ;t
# )ÚMimiResidualVectorQuantizeri  zResidual Vector Quantizer.c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r   r“   Únum_quantizersNr¡  )r$   r%   s   "€r&   r'   Ú(MimiResidualVectorQuantizer.__annotate__	  s   ø€ ÷ ñ ™zð ¹3À½:ñ r)   c                ót  <€ \         SV `  4        VP                  V n        VP                  V n        Ve   TMVP                  V n        \
        P                  ! \        V P                  4       Uu. uF  p\        V4      NK  	  up4      V n	        R V n
        R V n        VP                  VP                  8w  d{   \        P
                  P                  VP                  VP                  ^RR7      V n
        \        P
                  P                  VP                  VP                  ^RR7      V n        R # R # u upi )NFrt  )r†   rG   r  Ú
frame_rater1  r   rñ   r  r  r  Ú
input_projÚoutput_projÚ$vector_quantization_hidden_dimensionr  r!   r‹   )rE   r“   r1  Ú_r”   s   &&& €r&   rG   Ú$MimiResidualVectorQuantizer.__init__	  s  ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ ×+Ñ+ˆŒØ0>Ò0J™nÐPV×PeÑPeˆÔÜ—m’mÌUÐSW×SfÑSfÔMgÓ$hÑMgÈÔ%;¸FÖ%CÑMgÑ$hÓiˆŒàˆŒØˆÔØ×6Ñ6¸&×:LÑ:LÔLÜ#Ÿh™hŸo™oØ×"Ñ" F×$OÑ$OÐQRÐY^ð .ó ˆDŒOô  %Ÿx™xŸ™Ø×;Ñ;¸V×=OÑ=OÐQRÐY^ð  /ó  ˆDÖñ	 Mùò	 %is   Á0D5c                ób   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  /# ©r   Ú
embeddingsr1  Nrª   rM   )r$   r%   s   "€r&   r'   r2    s1   ø€ ÷ ñ ¡§¡ð ¹sÀT½zð ÑUZ×UaÑUañ r)   c                ó>  € V P                   e   V P                  V4      pVe   TMV P                  pTp. pV P                  RV  F>  pVP                  V4      pVP	                  V4      pW7,
          pVP                  V4       K@  	  \        P                  ! V4      pV# )úÙ
Encode a given input tensor with the specified frame rate at the given number of quantizers / codebooks. The RVQ encode method sets
the appropriate number of quantizers to use and returns indices for each quantizer.
N)r5  r1  r  r  r  r  r!   Ústack)	rE   r<  r1  r   Úall_indicesr  ÚindicesÚ	quantizedÚout_indicess	   &&&      r&   r  Ú"MimiResidualVectorQuantizer.encode  s•   € ð
 �?‰?Ò&ØŸ™¨Ó4ˆJà+9Ò+E™È4×K^ÑK^ˆàˆØˆØ—[‘[  .Ó1ˆEØ—l‘l 8Ó,ˆGØŸ™ WÓ-ˆIØÕ+ˆHØ×Ñ˜wÖ'ñ	 2ô
 —k’k +Ó.ˆØÐr)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   Úcodesrª   r«   )r$   r%   s   "€r&   r'   r2  .  s#   ø€ ÷ ñ ™EŸL™Lð ©U¯\©\ñ r)   c                ó.  € \         P                  ! RRVP                  R7      pVP                  ^ ^4      p\	        V4       F1  w  r4V P
                  V,          pVP                  V4      pW&,           pK3  	  V P                  e   V P                  V4      pV# )zJDecode the given codes of shape [B, K, T] to the quantized representation.rÒ   rà  r*   )r!   r+  rQ   r[  rî   r  r  r6  )rE   rG  Úquantized_outrø   rA  r  rB  s   &&     r&   r  Ú"MimiResidualVectorQuantizer.decode.  s€   € äŸ
š
 2 s°5·<±<Ô@ˆØ—‘  1Ó%ˆÜ# EÖ*‰JˆAØ—K‘K •NˆEØŸ™ WÓ-ˆIØ)Õ5ŠMñ +ð
 ×ÑÒ'Ø ×,Ñ,¨]Ó;ˆMØÐr)   )r  r4  r5  r  r1  r6  r¢   r-  rÕ   s   @@r&   r/  r/    s+   ù‡ € Ù$÷õ ÷"ò ÷(÷ ð r)   r/  c                   ój   a a€ ] tR tRt oRtV3R lV 3R lltR
V3R lR lltV3R lR ltR	tVt	V ;t
# )Ú MimiSplitResidualVectorQuantizeri<  z Split Residual Vector Quantizer.c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r$   r%   s   "€r&   r'   Ú-MimiSplitResidualVectorQuantizer.__annotate__?  s   ø€ ÷ 
tñ 
t™zñ 
tr)   c                óX  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  VP                  ,
          V n        \        WP                  4      V n	        \        WP                  4      V n
        R # r¢   )r†   rG   r  r4  r1  Úmax_num_quantizersÚnum_semantic_quantizersÚnum_acoustic_quantizersr/  Ú"semantic_residual_vector_quantizerÚ"acoustic_residual_vector_quantizerr{  s   &&€r&   rG   Ú)MimiSplitResidualVectorQuantizer.__init__?  s„   ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ ×+Ñ+ˆŒØ"(×"7Ñ"7ˆÔà'-×'EÑ'EˆÔ$Ø'-×'<Ñ'<¸v×?]Ñ?]Õ']ˆÔ$ä2MÈf×VrÑVrÓ2sˆÔ/Ü2MÈf×VrÑVrÓ2sˆÖ/r)   c                ób   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  /# r;  )r!   rN   r¹   )r$   r%   s   "€r&   r'   rN  K  s1   ø€ ÷ ñ ¡§¡ð ¹uÀt½|ð ÑW\×WcÑWcñ r)   c                ó¼  € Vf   V P                   MTpW P                   8”  d   \        RV P                    RV R24      hW P                  8  d   \        RV P                   RV R24      hV P                  P	                  V4      pW P                  8”  dG   V P
                  P	                  WV P                  ,
          R7      p\        P                  ! W4.^ R7      pV# )r>  úcThe number of quantizers (i.e codebooks) asked should be lower than the total number of quantizers ú, but is currently Ú.zgThe number of quantizers (i.e codebooks) asked should be higher than the number of semantic quantizers )r1  rb   )rP  rC   rQ  rS  r  rT  r!   rg   )rE   r<  r1  rG  Úacoustic_codess   &&&  r&   r  Ú'MimiSplitResidualVectorQuantizer.encodeK  s.  € ð 5CÒ4J˜×0Ò0ÐP^ˆà×3Ñ3Ô3ÜØuÐvz÷  wNñ  wNð  vOð  Obð  cqð  brð  rsð  tóð ð ×8Ñ8Ô8ÜØyÐz~÷  {Wñ  {Wð  zXð  Xkð  lzð  k{ð  {|ð  }óð ð
 ×7Ñ7×>Ñ>¸zÓJˆà×8Ñ8Ô8Ø!×DÑD×KÑKØ¸D×<XÑ<XÕ+Xð Ló ˆNô —I’I˜uÐ5¸1Ô=ˆEàˆr)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rF  r«   )r$   r%   s   "€r&   r'   rN  h  s#   ø€ ÷ 	ñ 	™EŸL™Lð 	©U¯\©\ñ 	r)   c                ó  € V P                   P                  VRRV P                  13,          4      pVP                  ^,          V P                  8”  d7   W P                  P                  VRV P                  R13,          4      ,          pV# )z7Decode the given codes to the quantized representation.ra   N)rS  r  rQ  rT   rT  )rE   rG  rI  s   && r&   r  Ú'MimiSplitResidualVectorQuantizer.decodeh  s}   € ð ×?Ñ?×FÑFÀuÈQÐPnÐRV×RnÑRnÐPnÐMnÕGoÓpˆð �;‰;�q�>˜D×8Ñ8Ô8Ø×DÑD×KÑKÈEÐRSÐUY×UqÑUqÑUsÐRsÕLtÓuÕuˆMØÐr)   )rT  r  r4  rP  rR  rQ  rS  r¢   r-  rÕ   s   @@r&   rL  rL  <  s-   ù‡ € Ù*÷
tó 
t÷ò ÷:	÷ 	ð 	r)   rL  c                   ó˜   a a€ ] tR tRt oRtRtRtRtRR.tR.t	Rt
RtRtRtRt]P                   ! 4       V 3R	 l4       tV3R
 ltRtVtV ;t# )ÚMimiPreTrainedModelit  ÚmimiÚinput_valuesÚaudioTrL  r¾  r±  c                ó,  <€ \         SV `  V4       \        V\        P                  \        P
                  34      '       d    \        P                  ! VP                  4       VP                  ep   \        P                  ! VP                  VP                  VP                  ^ ,          ,          ,          4      p\        P                  ! VP                  V) VR7       R# R# \        V\         4      '       d8   \        P"                  ! VP$                  V P&                  P(                  4       R# \        V\*        4      '       dÚ   VP,                  P                  ^ ,          pVP,                  P.                  ^ ,          pVP,                  P0                  ^ ,          pV^,
          V,          ^,           p\        P"                  ! VP.                  V4       \        P"                  ! VP                  V4       \        P"                  ! VP2                  W4,
          4       R# \        V\4        4      '       dc   \        P6                  ! VP8                  4       \        P6                  ! VP:                  4       \        P<                  ! VP>                  4       R# R# )zInitialize the weightsN)ÚaÚb) r†   Ú_init_weightsrÿ   r   r‹   rÝ   ÚinitÚkaiming_normal_Úweightr~   rÞ   r§  r|   r[   ry   Úuniform_r&  Ú	constant_r,  r“   r)  rv   rŒ   rz   r{   r�   rü  Úones_r  r  Úzeros_r  )rE   r‹  rl  ry   rz   r{   r”   s   &&    €r&   rh  Ú!MimiPreTrainedModel._init_weights„  s   ø€ ô 	‰Ñ˜fÔ%Ü�fœrŸy™y¬"×*<Ñ*<Ð=×>Ò>Ü× Ò  §¡Ô/Ø�{‰{Ò&Ü—I’I˜fŸm™m¨v×/AÑ/AÀF×DVÑDVÐWXÕDYÕ/YÕZÓ[�Ü—’˜fŸk™k¨a¨R°1×5ñ 'ô ˜¤×/Ò/Ü�NŠN˜6Ÿ<™<¨¯©×)NÑ)NÖOÜ˜¤
×+Ò+Ø Ÿ+™+×1Ñ1°!Õ4ˆKØ—[‘[×'Ñ'¨Õ*ˆFØ—{‘{×+Ñ+¨AÕ.ˆHØ&¨�?¨hÕ6¸Õ:ˆKÜ�NŠN˜6Ÿ=™=¨&Ô1Ü�NŠN˜6×-Ñ-¨{Ô;Ü�NŠN˜6×/Ñ/°Õ1EÖFÜ˜Ô 5×6Ò6Ü�JŠJ�v×)Ñ)Ô*Ü�JŠJ�v×+Ñ+Ô,Ü�KŠK˜×(Ñ(Ö)ñ 7r)   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r  r   )r$   r%   s   "€r&   r'   Ú MimiPreTrainedModel.__annotate__t  s   ø‡ ‚ áÑò r)   r*   )r+   r,   r-   r.   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendÚ_can_compile_fullgraphr!   rd  rh  r0   r1   r2   rÔ   rÕ   s   @@r&   ra  ra  t  so   ù‡ € ð ÐØ$€OØÐØ&*Ð#Ø;Ð=SÐTÐØ#4Ð"5ÐØÐØ€NØÐØ"&Ðà!Ðà
‡]‚]ƒ_ô*ó ð*÷# … r)   ra  z,
    The Mimi neural audio codec model.
    )Úcustom_introc                   óÞ   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltV3R lR ltRV3R lR	 lltRV3R
 lR lltRV3R lR llt	RV3R lR llt
]RV3R lR ll4       tRtVtV ;t# )Ú	MimiModeli�  c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r$   r%   s   "€r&   r'   ÚMimiModel.__annotate__£  s   ø€ ÷ )ñ )™zñ )r)   c                óÊ  <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        R V n        R V n        VP                  VP                  8w  dÕ   \        VVP                  VP                  ^\        VP                  VP                  ,          4      ,          ^RR\        V P                  P                  4      R7      V n        \!        VVP                  VP                  ^\        VP                  VP                  ,          4      ,          ^RVP"                  R7      V n        \        V4      V n        \'        V4      V n        \+        V4      V n        \        \.        P0                  ! V P                  P2                  4      4      V n        ^V P4                  ,          V P                  P2                  8w  d   \7        R4      hV P9                  4        R # )NFrS   )ry   rz   r~   r}   rL   )ry   rz   r~   r|   z'The codebook_size must be a power of 2.)r†   rG   r“   r  ÚencoderrÔ  Úencoder_transformerÚ
downsampleÚupsampler4  Úencodec_frame_raterv   r  r;   rA   r  r×   Úupsample_groupsÚdecoder_transformerró  ÚdecoderrL  Ú	quantizerrÞ   Úlog2r  Úbits_per_codebookrC   Ú	post_initr{  s   &&€r&   rG   ÚMimiModel.__init__£  sv  ø€ Ü‰Ñ˜Ô ØŒä" 6Ó*ˆŒÜ#7¸Ó#?ˆÔ àˆŒØˆŒØ×Ñ × 9Ñ 9Ô9Ü(ØØ×"Ñ"Ø×"Ñ"Ø¤ F×$=Ñ$=À×@QÑ@QÕ$QÓ RÕRØØØ$Ü˜dŸl™l×BÑBÓCô	ˆDŒOô 0ØØ×"Ñ"Ø×"Ñ"Ø¤ F×$=Ñ$=À×@QÑ@QÕ$QÓ RÕRØØØ×-Ñ-ôˆDŒMô $8¸Ó#?ˆÔ Ü" 6Ó*ˆŒä9¸&ÓAˆŒä!$¤T§Y¢Y¨t¯{©{×/HÑ/HÓ%IÓ!JˆÔØˆd×$Ñ$Õ$¨¯©×(AÑ(AÔAÜÐFÓGÐGð 	�‰Ör)   c                óâ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[R,          RS[R,          RS[R,          RS[R,          R	S[S[ P                  S[ P                  R,          3,          /# )
r   rc  r1  Úpadding_maskr±  NrD   Úuse_streamingrÞ  rª   )r!   rN   r;   r   r5   r   r¸   )r$   r%   s   "€r&   r'   r‚  Î  s‡   ø€ ÷ !5ñ !5á—l‘lð!5ñ ð!5ñ ð	!5ñ
  �ð!5ñ .°Õ4ð!5ñ ˜d•{ð!5ñ ˜D•[ð!5ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ!5r)   c                óŽ  € V P                  WR7      pV P                  VP                  ^^4      VVVR7      p	V'       d   V	P                  R4      pM\	        V	4      ^8”  d
   V	^,          pV	^ ,          P                  ^^4      pV P                  W…R7      pV P                  P                  W‚4      p
V
P                  ^ ^4      p
W¤V3# )zp
Encodes the given input using the underlying VQVAE. The padding mask is required to compute the correct scale.
rþ   )r±  rÎ  rÞ  r±  )r„  r…  r[  ÚgetrA   r†  rŒ  r  )rE   rc  r1  r’  r±  rD   r“  rÞ  r<  Úencoder_outputsrG  s   &&&&&&&&   r&   Ú_encode_frameÚMimiModel._encode_frameÎ  sÇ   € ð —\‘\ ,�\ÓLˆ
ð ×2Ñ2Ø× Ñ   AÓ&Ø+Ø#Ø#ð	 3ó 
ˆ÷ Ø-×1Ñ1Ð2CÓD‰OÜ�Ó! AÔ%Ø-¨aÕ0ˆOØ$ QÕ'×1Ñ1°!°QÓ7ˆ
Ø—_‘_ Z�_ÓMˆ
à—‘×%Ñ% jÓAˆØ—‘  1Ó%ˆØ }Ð4Ð4r)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÅ   rÇ   )r$   r%   s   "€r&   r'   r‚  ñ  s'   ø€ ÷ ñ ©u×/?Ñ/?ð ÁE×DTÑDTñ r)   c                óÎ   € TpV P                   P                   F-  pV P                   P                  V4      P                  V4      pK/  	  V P                  P                  V4      pV# )z<
Return the number of frames of the encoded audio waveform.
)r„  r  r  rË   r†  )rE   rÆ   rÊ   Ú
layer_names   &&  r&   Úget_encoded_lengthÚMimiModel.get_encoded_lengthñ  s[   € ð %ˆð Ÿ,™,×>Ô>ˆJØ ŸL™L×6Ñ6°zÓB×UÑUÐVcÓdŠMñ ?ð Ÿ™×:Ñ:¸=ÓIˆàÐr)   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# )r   r’  Úpadding_side)r!   rN   r=   )r$   r%   s   "€r&   r'   r‚     s   ø€ ÷ 4ñ 4±·±ð 4ÉSñ 4r)   c                ój  € V P                  VP                  RR7      4      p\        P                  ! VP	                  4       VP
                  R7      P                  \        V4      R4      pWCP                  ^4      8  pVP                  VP
                  4      pVR8X  d   V# VP                  R.R7      # )zB
Get the mask for the audio codes from the original padding mask.
rb   rà  Úright)Údimsre   )rœ  Úsumr!   rO  rf   rQ   rY  rA   rj  r®   Úflip)rE   r’  rŸ  Úencoded_lengthsÚaudio_codes_masks   &&&  r&   Úget_audio_codes_maskÚMimiModel.get_audio_codes_mask   s¨   € ð ×1Ñ1°,×2BÑ2BÀrÐ2BÓ2JÓKˆä Ÿ<š<¨×(;Ñ(;Ó(=Ào×F\ÑF\Ô]×dÑdÜ�Ó  "ó
Ðð ,×.GÑ.GÈÓ.JÑJÐØ+×.Ñ.¨|×/BÑ/BÓCÐà˜7Ô"Ø#Ð#à#×(Ñ(¨r¨dÐ(Ó3Ð3r)   c                ó"  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[R,          R	S[S[ P                  S[ P                  R,          3,          S[,          /# )
r   rc  r’  Nr1  r   rD   r“  rÞ  rª   )r!   rN   r¹   r   r5   r   r¸   rn   )r$   r%   s   "€r&   r'   r‚    s®   ø€ ÷ YYñ YYá—l‘lðYYñ —l‘l TÕ)ðYYñ  �ð	YYñ
 "'¨¥ðYYñ .°Õ4ðYYñ ˜d•{ðYYñ ˜D•[ðYYñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1Ñ4EÕ	EñYYr)   c           
     ó0  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVf   V P                   P                  MTpW0P                   P                  8”  d'   \	        RV P                   P                   RV R24      hVP
                  w  r‰p
V	^8  g   V	^8”  d   \	        RV	 24      hVf%   \        P                  ! V4      P                  4       pV'       Edf   VEfa   . . . rÜpV P                  P                   FŸ  pVP                  V P                  P                  V4      P                  4       VP                  V P                  P                  V4      P                  4       VP                  V P                  P                  V4      P                  4       K¡  	  VP                  V P                   P                  4       VP                  V P                   P                  4       VP                  V P                   P                  4       \#        \%        V P                  P                  4      ^,           VVVR7      pV P'                  VVVP                  4       VVVVR7      w  rôpV'       g   VVV3# \)        WôV4      # )a¥  
Encodes the input audio waveform into discrete codes.

Args:
    input_values (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
        Float values of the input audio waveform.
    padding_mask (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
        Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked* or 0
        for *masked*.
    num_quantizers (`int`, *optional*):
        Number of quantizers (i.e codebooks) to use. By default, all quantizers are used.
    encoder_past_key_values (`Cache`, *optional*):
        Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the encoder transformer.
        This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

        The model will output the same cache format that is fed as input.

        If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't
        have their past key value states given to this model).
    return_dict (`bool`, *optional*):
        Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

Returns:
    `codebook` of shape `[batch_size, num_codebooks, frames]`, the discrete encoded codes for the input audio waveform.
rX  rY  rZ  z1Number of audio channels must be 1 or 2, but got )r7   r8   r9   r:   )r±  rD   r“  rÞ  )r“   rÞ  r“  r1  rC   rT   r!   Ú	ones_liker   r„  r  r  r  r�   r}   r[   r†  r5   rA   r—  rn   )rE   rc  r’  r1  r   rD   r“  rÞ  r8  r-  rÆ   r8   r9   r:   r›  Úencoded_framess   &&&&&&&&        r&   r  ÚMimiModel.encode  sr  € ðF &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆØ)6Ò)B™ÈÏÉ×HaÑHaˆà7EÒ7M˜Ÿ™×3Ò3ÐSaˆàŸK™K×6Ñ6Ô6ÜØuÐvz÷  wBñ  wB÷  wQñ  wQð  vRð  Reð  ftð  euð  uvð  wóð ð %1×$6Ñ$6Ñ!ˆ�\à�aŒ<˜8 aœ<ÜÐPÐQYÐPZÐ[Ó\Ð\àÒÜ Ÿ?š?¨<Ó8×=Ñ=Ó?ˆLçˆ=˜]Ó2ØOQÐSUÐWYÐ7LÐØ"Ÿl™l×BÔB�
Ø!×(Ñ(¨¯©×)CÑ)CÀJÓ)O×)]Ñ)]Ô^Ø&×-Ñ-¨d¯l©l×.HÑ.HÈÓ.T×.]Ñ.]Ô^Ø%×,Ñ,¨T¯\©\×-GÑ-GÈ
Ó-S×-_Ñ-_Ö`ñ Cð ×$Ñ$ T§_¡_×%BÑ%BÔCØ"×)Ñ)¨$¯/©/×*BÑ*BÔCØ!×(Ñ(¨¯©×)DÑ)DÔEä2Ü˜tŸ|™|×CÑCÓDÀqÕHØ"3Ø'=Ø&;ô	ˆMð BF×ASÑASØØØ×ÑÓØ3Ø'Ø'Ø#ð BTó B
Ñ>ˆ°÷ àØ'Øðð ô ! È-ÓXÐXr)   c                óv   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[R,          RS[ P                  /# )r   rG  r±  NrÞ  rª   )r!   rN   r   r   )r$   r%   s   "€r&   r'   r‚  l  sC   ø€ ÷ (ñ (á�|‰|ð(ñ  �ð(ñ ˜D•[ð	(ñ
 
�‰ñ(r)   c                ó`  € V P                   P                  V4      pV P                  V4      pV P                  VP	                  ^^4      W#R7      pV'       d   VP                  R4      pM\        V4      ^8”  d
   V^,          pV^ ,          P	                  ^^4      pV P                  V4      pWb3# )r@   ©r±  rÞ  r±  )rŒ  r  r‡  rŠ  r[  r•  rA   r‹  )rE   rG  r±  rÞ  r<  Údecoder_outputsrÑ  s   &&&&   r&   Ú_decode_frameÚMimiModel._decode_framel  s¨   € ð —^‘^×*Ñ*¨5Ó1ˆ
à—]‘] :Ó.ˆ
Ø×2Ñ2Ø× Ñ   AÓ&¸ð 3ó 
ˆ÷ Ø-×1Ñ1Ð2CÓD‰OÜ�Ó! AÔ%Ø-¨aÕ0ˆOØ$ QÕ'×1Ñ1°!°QÓ7ˆ
Ø—,‘,˜zÓ*ˆØÐ'Ð'r)   c                óØ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          S[,          /# )r   r   r’  Nr    rÞ  rª   )r!   rN   r   r   r¸   rr   )r$   r%   s   "€r&   r'   r‚  €  sq   ø€ ÷ .Hñ .Há—\‘\ð.Hñ —l‘l TÕ)ð.Hñ "'¨¥ð	.Hñ
 ˜D•[ð.Hñ 
‰u�|‰|™UŸ\™\Ð)Õ	*Ñ->Õ	>ñ.Hr)   c                ó(  € Ve   TMV P                   P                  pV P                  WVR7      w  rSVeH   VP                  R,          VP                  R,          8  d   VRRVP                  R,          13,          pV'       g   VV3# \	        WS4      # )a:  
Decodes the given frames into an output audio waveform.

Note that the output might be a bit bigger than the input. In that case, any extra steps at the end can be
trimmed.

Args:
    audio_codes (`torch.LongTensor`  of shape `(batch_size, num_quantizers, codes_length)`, *optional*):
        Discrete code embeddings computed using `model.encode`.
    padding_mask (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
        Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked* or 0
        for *masked*.
    decoder_past_key_values (`Cache`, *optional*):
        Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the decoder transformer.
        This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

        The model will output the same cache format that is fed as input.

        If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't
        have their past key value states given to this model).
    return_dict (`bool`, *optional*):
        Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

Nr°  .re   )r“   rÞ  r²  rT   rr   )rE   r   r’  r    rÞ  r   s   &&&&& r&   r  ÚMimiModel.decode€  sž   € ð> &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà04×0BÑ0BØÈkð 1Có 1
Ñ-ˆð
 Ò#¨×(:Ñ(:¸2Õ(>À×ASÑASÐTVÕAWÔ(WØ'¨Ð-E¨|×/AÑ/AÀ"Õ/EÐ-EÐ(EÕFˆLçàØ'ðð ô ! ÓGÐGr)   c                ó(  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          R	S[S[ P                  S[ P                  3,          S[,          /# )
r   rc  r’  Nr1  r   r   r    rÞ  rª   )r!   rN   r;   r   r   r¸   r   )r$   r%   s   "€r&   r'   r‚  ±  s¢   ø€ ÷ W
ñ W
á—l‘lðW
ñ —l‘l TÕ)ðW
ñ ˜d�
ð	W
ñ
 —\‘\ DÕ(ðW
ñ "'¨¥ðW
ñ "'¨¥ðW
ñ ˜D•[ðW
ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*©ZÕ	7ñW
r)   c                ó  € Ve   TMV P                   P                  pVf%   \        P                  ! V4      P	                  4       pVfQ   V P                  WW5VR7      p	V	^ ,          pV'       d   V	P                  R4      pM\        V	4      ^8”  d
   V	^,          pV P                  WBWgR7      p
V
^ ,          pV'       d   V
P                  R4      pM\        V
4      ^8”  d
   V
^,          pV'       g   WKWV3# \        VVVVR7      # )a¢	  
input_values (`torch.FloatTensor` of shape `(batch_size, channels, sequence_length)`, *optional*):
    Raw audio input converted to Float.
padding_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked* or 0
    for *masked*.
num_quantizers (`int`, *optional*):
    Number of quantizers (i.e codebooks) to use. By default, all quantizers are used.
audio_codes (`torch.LongTensor`  of shape `(batch_size, num_quantizers, codes_length)`, *optional*):
    Discrete code embeddings computed using `model.encode`.
encoder_past_key_values (`Cache`, *optional*):
    Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the encoder transformer.
    This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

    The model will output the same cache format that is fed as input.

    If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't
    have their past key value states given to this model).
decoder_past_key_values (`Cache`, *optional*):
    Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the decoder transformer.
    This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

    The model will output the same cache format that is fed as input.

    If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't
    have their past key value states given to this model).

Examples:

```python
>>> from datasets import load_dataset
>>> from transformers import AutoFeatureExtractor, MimiModel

>>> dataset = load_dataset("hf-internal-testing/ashraq-esc50-1-dog-example")
>>> audio_sample = dataset["train"]["audio"][0]["array"]

>>> model_id = "kyutai/mimi"
>>> model = MimiModel.from_pretrained(model_id)
>>> feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)

>>> inputs = feature_extractor(raw_audio=audio_sample, return_tensors="pt")

>>> outputs = model(**inputs)
>>> audio_codes = outputs.audio_codes
>>> audio_values = outputs.audio_values
```)rÞ  r±  )r   r   r   r    )
r“   rÞ  r!   r«  r   r  r•  rA   r  r   )rE   rc  r’  r1  r   r   r    rÞ  rP  r–  r±  r   s   &&&&&&&&,   r&   rÏ   ÚMimiModel.forward°  s  € ðt &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒÜ Ÿ?š?¨<Ó8×=Ñ=Ó?ˆLàÒØ"Ÿk™kØ¨NÐalð *ó ˆOð *¨!Õ,ˆKßØ*9×*=Ñ*=Ð>OÓ*PÑ'Ü�_Ó%¨Ô)Ø*9¸!Õ*<Ð'àŸ+™+ kÐAX˜+ÓrˆØ& qÕ)ˆßØ&5×&9Ñ&9Ð:KÓ&LÑ#Ü�Ó! AÔ%Ø&5°aÕ&8Ð#çØÐ/FÐ`Ð`äØ#Ø%Ø$;Ø$;ô	
ð 	
r)   )	rŽ  r“   r‹  rŠ  r†  r„  r…  rŒ  r‡  )NNNN)r¡  )NNNNNN)NNr¼  )r+   r,   r-   r.   rG   r—  rœ  r§  r  r²  r  r   rÏ   r1   r2   rÔ   rÕ   s   @@r&   r€  r€  �  sw   ù‡ € ÷)ó )÷V!5ò !5÷Fð ÷4ò 4÷"YYò YY÷v(ò (÷(.Hò .Hð` ÷W
ñ W
ó ÷W
ð W
r)   r€  )r@   )rÒ   )Gr/   rÞ   Úcollections.abcr   Údataclassesr   r!   r   Ú r   ri  Úactivationsr   Úcache_utilsr   r   Úmasking_utilsr	   Úmodeling_layersr
   Úmodeling_outputsr   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   rš   r   r   r   r   Úutils.deprecationr   Úutils.genericr   Úconfiguration_mimir   Ú
get_loggerr+   r‰   r   r5   rn   rr   r�  rv   r×   rç   r  r&  r7  rh  rp  rr  r‰  r�  rŸ  r¾  rÔ  ró  rü  r  r/  rL  ra  r€  Ú__all__r*   r)   r&   Ú<module>rÊ     s2  ðñ ã Ý $Ý !ã Ý å &Ý !ß .Ý >Ý 9Ý 7ß Kß FÝ &ß MÓ MÝ 0Ý +Ý *ð 
×	Ò	˜HÓ	%€ð Ø
ô1�ó 1ó ó ð1÷<[ñ [ð| Ø
ô8˜ó 8ó ó ð8ð* Ø
ô1˜ó 1ó ó ð1ô$I�—‘ô IôX7˜"Ÿ)™)ô 7ôt'(�b—i‘iô '(ôT*�"—)‘)ô *ôZ�R—Y‘Yô ô 6<˜"Ÿ)™)ô 6<òt(ôô2ˆb�i‰iô õ"	U÷%ô2E)�B—I‘Iô E)ôP2Ð5ô 2ôjR
˜2Ÿ9™9ô R
ôj�"—)‘)ô ôB*˜BŸI™Iô *ô\˜RŸY™Yô ô(3 "§)¡)ô 3ôl5 r§y¡yô 5ðp ô%*˜/ó %*ó ð%*ñP ðôô
f
Ð#ó f
óð
f
ðR Ð-Ð
.‚r)   