+
    œ°³j’  ã                   óž  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHt ^RI	H
t
HtHt ^RIHtHt ^RIHtHt ^R	IHt ^R
IHt ^RIHtHtHtHt ^RIHtHt ^RIH t H!t! ^RI"H#t# ^RI$H%t%H&t&H't'H(t(H)t) ^RI*H+t+ ^RI,H-t-H.t. ^RI/H0t0 ^RI1H2t2H3t3H4t4 ^RI5H6t6 ])Pn                  ! ]84      t9]& ! R R]!4      4       t: ! R R]Pv                  4      t< ! R R]Pv                  4      t=]! R4       ! R R]Pv                  4      4       t> ! R R]Pv                  4      t?R  t@]! R!4      R:R" l4       tAR# R$ ltBR;R% R& lltC]! ]A4       ! R' R(]Pv                  4      4       tD ! R) R*]Pv                  4      tE ! R+ R,]4      tF ! R- R.]:4      tG ! R/ R0]4      tH ! R1 R2]:4      tI]&! R3R47       ! R5 R6]:4      4       tJ]&! R7R47       ! R8 R9]:]64      4       tK. R<OtLR# )=é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs)Ú	DiaConfigÚDiaDecoderConfigÚDiaEncoderConfig)ÚDiaGenerationMixinc                   ób   a a€ ] tR t^5t oRtRtRtRtRtRt	Rt
RR.tV 3R ltV3R ltRtVtV ;t# )	ÚDiaPreTrainedModelÚmodelTÚ	input_idsÚDiaEncoderLayerÚDiaDecoderLayerc                óB  <€ \         SV `  V4       \        V\        4      '       dy   \        P
                  ! V P                  P                  \        P                  R 7      V P                  P                  ,          p\        P                  ! VP                  V4       R# R# )©ÚdtypeN)ÚsuperÚ_init_weightsÚ
isinstanceÚDiaMultiChannelEmbeddingÚtorchÚarangeÚconfigÚnum_channelsÚlongÚ
vocab_sizeÚinitÚcopy_Úoffsets)ÚselfÚmoduler;   Ú	__class__s   && €Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/dia/modeling_dia.pyr0   Ú DiaPreTrainedModel._init_weightsA   se   ø€ Ü‰Ñ˜fÔ%Ü�fÔ6×7Ò7Ü—l’l 4§;¡;×#;Ñ#;Ä5Ç:Á:ÔNÐQU×Q\ÑQ\×QgÑQgÕgˆGÜ�JŠJ�v—~‘~ wÖ/ñ 8ó    c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©é   r5   ©r"   )ÚformatÚ__classdict__s   "€r?   Ú__annotate__ÚDiaPreTrainedModel.__annotate__5   s   ø‡ ‚ áÑò rA   © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚmain_input_nameÚ_no_split_modulesr0   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r>   rG   s   @@r?   r'   r'   5   sG   ù‡ € ð  ÐØ&*Ð#ØÐØ€NØÐØ!ÐØ!€OØ*Ð,=Ð>Ðõ0÷ … rA   r'   c                   óT   a a€ ] tR t^Ht oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )r2   aâ  In order to efficiently compute the audio embedding from the 9 different channels,
we vectorize the embedding process by using a single embedding layer and an offset.
Example:
- num_embeds = 4
- vocab_size = 8
- num_channels = 3
We would have offsets = [0, 8, 16]
If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
then tokens = audio_codes + offsets
            = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
This allows us to use a single embedding layer for all channels.
c                ó    <€ V ^8„  d   QhRS[ /# rC   ©r#   )rF   rG   s   "€r?   rH   Ú%DiaMultiChannelEmbedding.__annotate__V   s   ø€ ÷ <ñ <Ñ/ñ <rA   c                ó¦  <€ \         SV `  4        \        P                  ! VP                  VP
                  ,          VP                  4      V n        VP                  V n        VP
                  V n        \        P                  ! VP
                  \        P                  R 7      VP                  ,          p\        P                  ! VRR7      V n        R# )r-   F©Ú
persistentN)r/   Ú__init__r   Ú	Embeddingr8   r6   Úhidden_sizeÚembedr3   r4   r7   ÚBufferr;   )r<   r5   r;   r>   s   && €r?   rc   Ú!DiaMultiChannelEmbedding.__init__V   sŠ   ø€ Ü‰ÑÔÜ—\’\ &×"3Ñ"3°f×6IÑ6IÕ"IÈ6×K]ÑK]Ó^ˆŒ
Ø!×-Ñ-ˆÔØ"×/Ñ/ˆÔÜ—,’,˜v×2Ñ2¼%¿*¹*ÔEÈ×HYÑHYÕYˆÜ—y’y °UÔ;ˆŽrA   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rD   Úaudio_codesÚreturn©r3   ÚTensor)rF   rG   s   "€r?   rH   r_   ^   s#   ø€ ÷ !ñ !¡5§<¡<ð !±E·L±Lñ !rA   c                óŒ  € WP                   P                  VP                  4      ,           P                  RVP                  ^,          VP                  ^,          ,          4      pV P                  V4      P                  VP                  ^ ,          VP                  ^,          RV P                  4      pVP                  ^R7      # )é   ©Údiméÿÿÿÿ)r;   ÚtoÚdeviceÚviewÚshaperf   re   Úsum)r<   rj   ÚtokensÚembedss   &&  r?   ÚforwardÚ DiaMultiChannelEmbedding.forward^   s•   € Ø§¡§¡°×0BÑ0BÓ CÕC×IÑIØ�×!Ñ! !Õ$ {×'8Ñ'8¸Õ';Õ;ó
ˆð —‘˜FÓ#×(Ñ(¨¯©°a­¸+×:KÑ:KÈAÕ:NÐPRÐTX×TdÑTdÓeˆØ�z‰z˜aˆzÓ Ð rA   )rf   re   r6   r;   ©
rK   rL   rM   rN   Ú__doc__rc   rz   rX   rY   rZ   r[   s   @@r?   r2   r2   H   s#   ù‡ € ñ÷<ó <÷!÷ !ð !rA   r2   c                   óD   a a€ ] tR t^ft oV 3R ltV3R lR ltRtVtV ;t# )ÚDiaMLPc                ó>  <€ \         SV `  4        Wn        \        P                  ! VP
                  ^VP                  ,          RR7      V n        \        P                  ! VP                  VP
                  RR7      V n        \        VP                  ,          V n        R# )rD   F©ÚbiasN)r/   rc   r5   r   ÚLinearre   Úintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn©r<   r5   r>   s   &&€r?   rc   ÚDiaMLP.__init__g   sn   ø€ Ü‰ÑÔàŒÜŸIšI f×&8Ñ&8¸!¸f×>VÑ>VÕ:VÐ]bÔcˆÔÜŸš 6×#;Ñ#;¸V×=OÑ=OÐV[Ô\ˆŒÜ# F×$5Ñ$5Õ6ˆÖrA   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rD   Úhidden_statesrk   )r3   ÚFloatTensor)rF   rG   s   "€r?   rH   ÚDiaMLP.__annotate__o   s'   ø€ ÷ )ñ )¡U×%6Ñ%6ð )¹5×;LÑ;Lñ )rA   c                óž   € V P                  V4      pVP                  ^RR7      w  r2W P                  V4      ,          pV P                  V4      # )rD   rp   rr   )r…   Úchunkrˆ   r†   )r<   r�   Ú	up_statesÚgates   &&  r?   rz   ÚDiaMLP.forwardo   sH   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ó4‰ˆØ× 2Ñ 2°4Ó 8Õ8ˆ	à�~‰~˜iÓ(Ð(rA   )rˆ   r5   r†   r…   ©	rK   rL   rM   rN   rc   rz   rX   rY   rZ   r[   s   @@r?   r   r   f   s   ù‡ € õ7÷)÷ )ð )rA   r   ÚRMSNormc                   óZ   a a€ ] tR t^xt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )Ú
DiaRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )rD   Úepsrk   N)Úfloat)rF   rG   s   "€r?   rH   ÚDiaRMSNorm.__annotate__z   s   ø€ ÷ $ñ $©ð $¸$ñ $rA   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z)
DiaRMSNorm is equivalent to T5LayerNorm
N)r/   rc   r   Ú	Parameterr3   ÚonesÚweightÚvariance_epsilon)r<   re   rš   r>   s   &&&€r?   rc   ÚDiaRMSNorm.__init__z   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #ÖrA   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rŒ   rl   )rF   rG   s   "€r?   rH   rœ   ‚   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;rA   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )rD   T)Úkeepdimrr   )	r.   rs   r3   Úfloat32ÚpowÚmeanÚrsqrtr¡   r    )r<   r�   Úinput_dtypeÚvariances   &&  r?   rz   ÚDiaRMSNorm.forward‚   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:rA   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler    rv   r¡   )r<   s   &r?   Ú
extra_reprÚDiaRMSNorm.extra_repr‰   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIrA   )r¡   r    )g�íµ ÷Æ°>)
rK   rL   rM   rN   rc   rz   r¯   rX   rY   rZ   r[   s   @@r?   r˜   r˜   x   s%   ù‡ € ÷$õ $÷;ð ;÷Jò JrA   r˜   c                   óÎ   a a€ ] tR t^�t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚDiaRotaryEmbeddingrt   z5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# rC   rE   )rF   rG   s   "€r?   rH   ÚDiaRotaryEmbedding.__annotate__�   s   ø€ ÷ Oñ O™yñ OrA   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultFra   N)r/   rc   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr5   Úrope_parametersr·   Úcompute_default_rope_parametersr   Úattention_scalingr   rg   Úinv_freqÚcloneÚoriginal_inv_freq)r<   r5   rt   Úrope_init_fnr¿   r>   s   &&&  €r?   rc   ÚDiaRotaryEmbedding.__init__Ž   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖrA   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )rD   r5   rk   )r"   r®   r3   rm   r›   )rF   rG   s   "€r?   rH   rµ   ¡   s.   ø€ ÷ 5ñ 5±	ð 5ÑUZÑ[`×[gÑ[gÑinÐ[nÕUoñ 5rA   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?r-   )r¼   Úgetattrre   Únum_attention_headsr3   r4   r›   rs   )r5   rt   ÚkwargsÚbaserq   Úattention_factorr¿   s   &&,    r?   r½   Ú2DiaRotaryEmbedding.compute_default_rope_parametersŸ   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4rA   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)r.   rt   ÚmpsÚcpuF)Údevice_typeÚenabledrp   r-   )NºNNNNrr   )rÓ   NrÓ   )r¿   Úexpandrv   rs   r3   r›   rt   r1   ÚtypeÚstrr   Ú	transposeÚcatÚcosr¾   Úsinr.   )
r<   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrÑ   ÚfreqsÚembrÙ   rÚ   s
   &&&       r?   rz   ÚDiaRotaryEmbedding.forward³   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r¾   r5   r¿   rº   rÁ   r»   r·   ©N)rK   rL   rM   rN   r   rc   Ústaticmethodr½   r3   Úno_gradr   rz   rX   rY   rZ   r[   s   @@r?   r²   r²   �   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <rA   r²   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrp   rr   )rv   r3   rØ   )rÛ   Úx1Úx2s   &  r?   Úrotate_halfrè   Æ   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'rA   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerè   )ÚqÚkrÙ   rÚ   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r?   Úapply_rotary_pos_embrñ   Í   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐrA   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )rD   r�   Ún_reprk   )r3   rm   Úint)rF   s   "r?   rH   rH   ç   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	UrA   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rÓ   rÓ   NrÓ   rÓ   )rv   rÔ   Úreshape)r�   ró   ÚbatchÚnum_key_value_headsÚslenrÇ   s   &&    r?   Ú	repeat_kvrú   ç   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTrA   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
rD   r=   ÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrÊ   )r   ÚModuler3   rm   r›   r   r   )rF   s   "r?   rH   rH   ó   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%rA   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )rD   )rq   r.   )ÚpÚtrainingrr   )rú   Únum_key_value_groupsr3   Úmatmulr×   r   Ú
functionalÚsoftmaxr¦   rs   r.   r  r  Ú
contiguous)r=   rü   rý   rþ   rÿ   r   r  rÊ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r?   Úeager_attention_forwardr  ó   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$rA   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚDiaSelfAttentioni  ú=Multi-headed attention from 'Attention Is All You Need' paperc                ó<   <€ V ^8„  d   QhRS[ S[,          RS[RS[/# )rD   r5   Ú	layer_idxÚ	is_causal)r$   r#   rô   Úbool)rF   rG   s   "€r?   rH   ÚDiaSelfAttention.__annotate__  s/   ø€ ÷ ^ñ ^Ñ/Ñ2BÕBð ^Ésð ^Ñ_cñ ^rA   c                óÐ  <€ \         SV `  4        Wn        W n        VP                  V n        V P                  P
                  V n        V P                  P                  ;'       g    V P                  V n        V P                  V P                  ,          V n        \        VR VP                  V P                  ,          4      V n
        ^V n        RV n        W0n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  ,          V P                  RR7      V n        R# )rÇ   ç        Fr�   N)r/   rc   r5   r  re   rÉ   Ú	num_headsrø   r  rÈ   rÇ   r   Úattention_dropoutr  r   rƒ   Úq_projÚk_projÚv_projÚo_proj)r<   r5   r  r  r>   s   &&&&€r?   rc   ÚDiaSelfAttention.__init__  sC  ø€ Ü‰ÑÔØŒØ"ŒØ!×-Ñ-ˆÔØŸ™×8Ñ8ˆŒØ#'§;¡;×#BÑ#B×#TÐ#TÀdÇnÁnˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ü ¨
°F×4FÑ4FÈ$Ï.É.Õ4XÓYˆŒØˆŒØ!$ˆÔØ"Œä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i §¡°·±Õ >À×@PÑ@PÐW\Ô]ˆŽrA   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )rD   r�   Úposition_embeddingsNrÿ   Úpast_key_valuesrÊ   rk   )r3   rm   r®   r   r   r   )rF   rG   s   "€r?   rH   r  "  s‡   ø€ ÷ &)ñ &)á—|‘|ð&)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð&)ñ Ÿ™ tÕ+ð	&)ñ
  �ð&)ñ Ñ+Õ,ð&)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ&)rA   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pWï3# )Nr  r  r   rr   )rv   rÇ   r  ru   r×   r  r  rñ   Úupdater  r   Úget_interfacer5   Ú_attn_implementationr  r  r  r   rö   r
  r  )r<   r�   r"  rÿ   r#  rÊ   Úinput_shapeÚhidden_shapeÚquery_statesr  r  rÙ   rÚ   Úattention_interfacer  r  s   &&&&&,          r?   rz   ÚDiaSelfAttention.forward"  s‡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(rA   )r  r5   rÇ   re   r  r  r  r  r  rø   r  r  r   r  )F)NNNr|   r[   s   @@r?   r  r    s#   ù‡ € áG÷^õ ^÷$&)÷ &)ò &)rA   r  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚDiaCrossAttentioniK  r  c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©rD   r5   r  ©r#   rô   )rF   rG   s   "€r?   rH   ÚDiaCrossAttention.__annotate__N  s    ø€ ÷ ^ñ ^Ñ/ð ^¹Cñ ^rA   c                ó”  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        V P                  P                  V n        V P                  P                  V n	        V P                  V P                  ,          V n
        VP                  V n        ^V n        RV n        RV n        \         P"                  ! V P                  V P                  V P                  ,          RR7      V n        \         P"                  ! V P
                  V P                  V P                  ,          RR7      V n        \         P"                  ! V P
                  V P                  V P                  ,          RR7      V n        \         P"                  ! V P                  V P                  ,          V P                  RR7      V n        R# )ro   r  Fr�   N)r/   rc   r5   r  re   Úcross_hidden_sizeÚcross_num_attention_headsr  Úcross_num_key_value_headsrø   r  Úcross_head_dimrÇ   r   r  r  r   rƒ   r  r  r  r  ©r<   r5   r  r>   s   &&&€r?   rc   ÚDiaCrossAttention.__init__N  s;  ø€ Ü‰ÑÔØŒØ"ŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØŸ™×>Ñ>ˆŒØ#'§;¡;×#HÑ#HˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø×-Ñ-ˆŒØˆŒØ!$ˆÔØˆŒä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Õ8`ÐglÔmˆŒÜ—i’i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Õ8`ÐglÔmˆŒÜ—i’i §¡°·±Õ >À×@PÑ@PÐW\Ô]ˆŽrA   c                óò   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )rD   r�   Úcross_attention_statesrÿ   Nr#  rÊ   rk   )r3   rm   r	   r   r   r®   )rF   rG   s   "€r?   rH   r2  a  sz   ø€ ÷ 1)ñ 1)á—|‘|ð1)ñ !&§¡ð1)ñ Ÿ™ tÕ+ð	1)ñ
 -¨tÕ3ð1)ñ Ñ-Õ.ð1)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ1)rA   c                ój  € VP                   R R p. VORNV P                  N5p. VP                   R R ORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p	Ve&   VP
                  P                  V P                  4      MRp
Vel   V
'       dd   VP                  P                  V P                  ,          P                  pVP                  P                  V P                  ,          P                  pM¦V P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pVeC   VP                  P                  VVV P                  4      w  r¼RVP
                  V P                  &   \        P                   ! V P"                  P$                  \&        4      pV! V V	VVV3RV P(                  /VB w  rïVP+                  . VORN54      P-                  4       pV P/                  V4      pWï3# )NFTr   rr   )rv   rÇ   r  ru   r×   Ú
is_updatedÚgetr  Úcross_attention_cacheÚlayersÚkeysÚvaluesr  r  r%  r   r&  r5   r'  r  r   rö   r
  r  )r<   r�   r;  rÿ   r#  rÊ   r(  r)  Úcross_shaper*  r=  r  r  r+  r  r  s   &&&&&,          r?   rz   ÚDiaCrossAttention.forwarda  s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØMÐ.×4Ñ4°S°bÐ9ÐM¸2ÐM¸t¿}¹}ÑMˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàGVÒGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÒ&¯:à(×>Ñ>×EÑEÀdÇnÁnÕU×ZÑZˆJØ*×@Ñ@×GÑGÈÏÉÕW×^Ñ^‰LàŸ™Ð%;Ó<×AÑAÀ+ÓN×XÑXÐYZÐ\]Ó^ˆJØŸ;™;Ð'=Ó>×CÑCÀKÓP×ZÑZÐ[\Ð^_Ó`ˆLàÒ*à+:×+PÑ+P×+WÑ+WØØ Ø—N‘Nó,Ñ(�
ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð —L‘Lð%
ð ñ%
Ñ!ˆð "×)Ñ)Ð*<¨KÐ*<¸Ñ*<Ó=×HÑHÓJˆØ—k‘k +Ó.ˆØÐ(Ð(rA   )r  r5   r4  rÇ   re   r  r  r  r  r  rø   r  r  r   r  ©NNr|   r[   s   @@r?   r.  r.  K  s#   ù‡ € ÙG÷^ó ^÷&1)÷ 1)ò 1)rA   r.  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )r*   i•  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r0  )r$   rô   )rF   rG   s   "€r?   rH   ÚDiaEncoderLayer.__annotate__–  s   ø€ ÷ "ñ "Ñ/ð "¹Cñ "rA   c                ó   <€ \         SV `  4        \        VP                  VP                  R 7      V n        \        WRR7      V n        \        VP                  VP                  R 7      V n        \        V4      V n
        R# )©rš   F©r  N)r/   rc   r˜   re   Únorm_epsÚpre_sa_normr  Úself_attentionÚpost_sa_normr   Úmlpr8  s   &&&€r?   rc   ÚDiaEncoderLayer.__init__–  sZ   ø€ Ü‰ÑÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ.¨vÈEÔRˆÔÜ& v×'9Ñ'9¸v¿¹ÔOˆÔÜ˜&“>ˆŽrA   c          
      óÞ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[S[,          RS[ P                  /# )rD   r�   r"  Nrÿ   rÊ   rk   )r3   rm   r®   r   r   )rF   rG   s   "€r?   rH   rH  �  sk   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 Ñ-Õ.ðñ 
�‰ñrA   c                óÆ   € TpV P                  V4      pV P                  ! V3R VRV/VB w  rxWW,           pTpV P                  V4      pV P                  V4      p	WY,           pV# )r"  rÿ   )rM  rN  rO  rP  )
r<   r�   r"  rÿ   rÊ   ÚresidualÚnormed_statesÚself_attn_outputÚ_Úmlp_outs
   &&&&,     r?   rz   ÚDiaEncoderLayer.forward�  s‡   € ð !ˆØ×(Ñ(¨Ó7ˆØ"×1Ò1Øñ
à 3ð
ð *ð
ð ñ	
ÑÐð !Õ3ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ Õ*ˆàÐrA   )rP  rO  rM  rN  rE  r•   r[   s   @@r?   r*   r*   •  s   ù‡ € ÷"ó "÷÷ ò rA   r*   c                   ó~   a a€ ] tR tRt oR]R]/tV3R lV 3R llt]]	]
R	V3R lR ll4       4       4       tRtVtV ;t# )
Ú
DiaEncoderi¶  r�   Ú
attentionsc                ó    <€ V ^8„  d   QhRS[ /# rC   )r$   )rF   rG   s   "€r?   rH   ÚDiaEncoder.__annotate__¼  ó   ø€ ÷ ñ Ñ/ñ rA   c           	     ó¸  <€ \         SV `  V4       Wn        \        P                  ! VP
                  VP                  4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \!        VR7      V n        V P%                  4        R# u upi ©rJ  ©r5   N)r/   rc   r5   r   rd   r8   re   Ú	embeddingÚ
ModuleListÚrangeÚnum_hidden_layersr*   r@  r˜   rL  Únormr²   Ú
rotary_embÚ	post_initr8  s   && €r?   rc   ÚDiaEncoder.__init__¼  s�   ø€ Ü‰Ñ˜Ô ØŒäŸš f×&7Ñ&7¸×9KÑ9KÓLˆŒÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÑA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô ˜v×1Ñ1°v·±ÔGˆŒ	Ü,°FÔ;ˆŒà�‰Öùò bs   Á.Cc                óx   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[/# )rD   r)   rÿ   NrÊ   rk   )r3   rm   r   r   r   )rF   rG   s   "€r?   rH   r^  Ì  sL   ø€ ÷ @ñ @á—<‘<ð@ñ Ÿ™ tÕ+ð@ñ Ñ+Õ,ð	@ñ
 
ñ@rA   c           
     ój  € V P                  V4      p\        P                  ! VP                  R,          VP                  R7      R	,          p\        V P                  VVR7      pV P                  WER7      pV P                   F  pV! V3RVRVRV/VB pK  	  V P                  V4      p\        VR7      # )
ro   ©rt   )r5   Úinputs_embedsrÿ   ©rÜ   rÿ   rÜ   r"  )Úlast_hidden_staterr   )NrÓ   )rc  r3   r4   rv   rt   r   r5   rh  r@  rg  r   )r<   r)   rÿ   rÊ   r�   rÜ   r"  Úencoder_layers   &&&,    r?   rz   ÚDiaEncoder.forwardÉ  s¿   € ð Ÿ™ yÓ1ˆô
 —|’| I§O¡O°BÕ$7À	×@PÑ@PÔQÐRYÕZˆä2Ø—;‘;Ø'Ø)ô
ˆð
 #Ÿo™o¨m˜oÓWÐà!Ÿ[œ[ˆMÙ)Øñà-ðð *ðð %8ð	ð
 ñŠMñ )ð Ÿ	™	 -Ó0ˆä°Ô?Ð?rA   )r5   rc  r@  rg  rh  râ   )rK   rL   rM   rN   r*   r  Ú_can_record_outputsrc   r    r!   r   rz   rX   rY   rZ   r[   s   @@r?   r[  r[  ¶  sT   ù‡ € à˜ØÐ&ðÐ÷
ó ð  ØØ÷@ñ @ó ó ó  ÷@ð @rA   r[  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )r+   iî  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r0  r1  )rF   rG   s   "€r?   rH   ÚDiaDecoderLayer.__annotate__ï  s   ø€ ÷ "ñ "Ñ/ð "¹Cñ "rA   c                óŽ  <€ \         SV `  4        VP                  V n        \	        WR R7      V n        \        W4      V n        \        VP                  VP                  R7      V n
        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \        V4      V n        R# )TrK  rJ  N)r/   rc   re   Ú	embed_dimr  rN  r.  Úcross_attentionr˜   rL  rM  Úpre_ca_normÚpre_mlp_normr   rP  r8  s   &&&€r?   rc   ÚDiaDecoderLayer.__init__ï  s�   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ.¨vÈDÔQˆÔÜ0°ÓCˆÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ& v×'9Ñ'9¸v¿¹ÔOˆÔÜ˜&“>ˆŽrA   c                óŠ  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          S[ P                  R,          3,          /# )	rD   r�   r"  Nrÿ   Úencoder_hidden_statesÚencoder_attention_maskr#  rk   )r3   rm   r®   r	   )rF   rG   s   "€r?   rH   rv  ù  s®   ø€ ÷ +ñ +á—|‘|ð+ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð+ñ Ÿ™ tÕ+ð	+ñ
  %Ÿ|™|¨dÕ2ð+ñ !&§¡¨tÕ 3ð+ñ -¨tÕ3ð+ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%·,±,ÀÕ2EÐEÕ	Fñ+rA   c                óv  € Tp\        V\        4      '       d   VP                  pTp	V P                  V4      p
V P                  ! V
VVV3/ VB w  r¼W›,           pTp	V P                  V4      p
V P                  ! V
V3R VRV/VB w  rÜW�,           pTp	V P                  V4      p
V P                  V
4      pWž,           pV# )rÿ   r#  )	r1   r	   Úself_attention_cacherM  rN  rz  ry  r{  rP  )r<   r�   r"  rÿ   r~  r  r#  rÊ   Úself_attn_cacherT  rU  rV  rW  Úcross_statesrX  s   &&&&&&&,       r?   rz   ÚDiaDecoderLayer.forwardù  só   € ð *ˆÜ�oÔ':×;Ò;Ø-×BÑBˆOà ˆØ×(Ñ(¨Ó7ˆØ"×1Ò1ØØØð ñ
ð ñ
ÑÐð !Õ3ˆà ˆØ×(Ñ(¨Ó7ˆØ×.Ò.ØØ!ñ
ð 2ð
ð ,ð	
ð
 ñ
‰ˆð !Õ/ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ Õ*ˆàÐrA   )ry  rx  rP  rz  r{  rM  rN  ©NNNNNr•   r[   s   @@r?   r+   r+   î  s   ù‡ € ÷"ó "÷+÷ +ò +rA   r+   c                   ó†   a a€ ] tR tRt oRtR]R]R]/tV3R lV 3R llt	]
]]RV3R lR	 ll4       4       4       tR
tVtV ;t# )Ú
DiaDecoderi'  z-Transformer Decoder Stack using DenseGeneral.r�   r\  Úcross_attentionsc                ó    <€ V ^8„  d   QhRS[ /# rC   r^   )rF   rG   s   "€r?   rH   ÚDiaDecoder.__annotate__0  r_  rA   c           	     ó°  <€ \         SV `  V4       VP                  V n        VP                  V n        \	        V4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \!        VR7      V n        V P%                  4        R# u upi ra  )r/   rc   r6   r8   r2   Ú
embeddingsr   rd  re  rf  r+   r@  r˜   re   rL  rg  r²   rh  ri  r8  s   && €r?   rc   ÚDiaDecoder.__init__0  s¡   ø€ Ü‰Ñ˜Ô Ø"×/Ñ/ˆÔØ ×+Ñ+ˆŒÜ2°6Ó:ˆŒÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÑA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô ˜v×1Ñ1°v·±ÔGˆŒ	Ü,°FÔ;ˆŒà�‰Öùò bs   Á*Cc                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[S[,          /# )
rD   r)   rÜ   Nrÿ   r~  r  r#  rÊ   rk   )	r3   rm   Ú
LongTensorrŽ   r	   r   r   r   r®   )rF   rG   s   "€r?   rH   rŠ  @  s£   ø€ ÷ A
ñ A
á—<‘<ðA
ñ ×&Ñ&¨Õ-ðA
ñ Ÿ™ tÕ+ð	A
ñ
  %×0Ñ0°4Õ7ðA
ñ !&× 0Ñ 0°4Õ 7ðA
ñ -¨tÕ3ðA
ñ Ñ+Õ,ðA
ñ 
3±UÕ	:ñA
rA   c           
     ó~  € VP                  4       RR
 w  r‰Ve   VP                  4       M^ p
Vf:   \        P                  ! W‘P                  R7      V
,           pVP                  ^ 4      pV P                  V4      pVf;   \        4       '       g+   W©,           p\        P                  ! WŒVP                  R7      p\        V P                  VVVR7      p\        V P                  VVVR7      pV P                  W²R7      pV P                   F  pV! VVVV3RVRVRV/VB pK  	  V P                  V4      p\        VVR	7      # )zç
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
    The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

    [What are input IDs?](../glossary#input-ids)
Nrm  )r5   rn  rÿ   r#  )r5   rn  rÿ   r~  ro  r  r#  rÜ   )rp  r#  rr   )ÚsizeÚget_seq_lengthr3   r4   rt   rë   rŒ  r   rŸ   r   r5   r   rh  r@  rg  r   )r<   r)   rÜ   rÿ   r~  r  r#  rÊ   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthr�   Úmask_seq_lengthr"  Úlayers   &&&&&&&,       r?   rz   ÚDiaDecoder.forward=  sY  € ð( "+§¡Ó!1°#°2Ð!6Ñˆ
ØETÒE` ×!?Ñ!?Ô!AÐfgÐàÒÜ Ÿ<š<¨
×;KÑ;KÔLÐOeÕeˆLØ'×1Ñ1°!Ó4ˆLð Ÿ™¨	Ó2ˆàÒ!Ô*B×*DÒ*Dà4ÕAˆOÜ"ŸZšZ¨
ÈI×L\ÑL\Ô]ˆNä+Ø—;‘;Ø'Ø)Ø+ô	
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð #Ÿo™o¨m˜oÓWÐà—[”[ˆEÙ!Øð $ØØ%ñð (>ðð !0ðð *ðð ñŠMñ !ð Ÿ	™	 -Ó0ˆä8Ø+Ø+ô
ð 	
rA   )rŒ  r@  rg  r6   rh  r8   r…  )rK   rL   rM   rN   r}   r+   r  r.  rs  rc   r    r!   r   rz   rX   rY   rZ   r[   s   @@r?   r‡  r‡  '  s_   ù‡ € Ù7ð 	˜ØÐ&ØÐ-ðÐ÷ó ð  ØØ÷A
ñ A
ó ó ó  ÷A
ð A
rA   r‡  z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )Úcustom_introc                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚDiaModeli„  c                ó    <€ V ^8„  d   QhRS[ /# rC   rE   )rF   rG   s   "€r?   rH   ÚDiaModel.__annotate__Š  s   ø€ ÷ ñ ™yñ rA   c                óº   <€ \         SV `  V4       Wn        \        VP                  4      V n        \        VP                  4      V n        V P                  4        R # râ   )
r/   rc   r5   r[  Úencoder_configÚencoderr‡  Údecoder_configÚdecoderri  r‰   s   &&€r?   rc   ÚDiaModel.__init__Š  sC   ø€ Ü‰Ñ˜Ô ØŒÜ! &×"7Ñ"7Ó8ˆŒÜ! &×"7Ñ"7Ó8ˆŒØ�‰ÖrA   c                óD  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R,          RS[R,          R	S[R,          R
S[S[,          /	# )rD   r)   Nrÿ   Údecoder_input_idsÚdecoder_position_idsÚdecoder_attention_maskÚencoder_outputsr#  Ú	use_cacherk   )r3   r�  r   r®   r	   r  r   )rF   rG   s   "€r?   rH   r�  “  sÀ   ø€ ÷ ]
ñ ]
á×#Ñ# dÕ*ð]
ñ ×(Ñ(¨4Õ/ð]
ñ !×+Ñ+¨dÕ2ð	]
ñ
 $×.Ñ.°Õ5ð]
ñ !&× 0Ñ 0°4Õ 7ð]
ñ )©5Õ0°4Õ7ð]
ñ -¨tÕ3ð]
ñ ˜$•;ð]
ñ 
Ñ#Õ	#ñ]
rA   c	                ó˜  € Vf   Vf   \        R4      hV P                  '       d2   V P                  '       d    V'       d   \        P	                  R4       RpV'       d9   Vf5   \        \        V P                  R7      \        V P                  R7      4      pVf   V P                  ! RRVRV/V	B pM]\        V\        4      '       gH   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV^ ,          P                  ^ ,          RV P                  P                  P                  rËp
VfE   \        P                   ! V
^V3V P                  P                  P"                  V P$                  R	7      pVP&                  ^8X  d#   VP)                  W¬V4      P+                  ^^4      pV P,                  ! RRVR
VRVRV^ ,          RVRVRV/V	B p\/        VP0                  VP2                  VP4                  VP6                  VP8                  V^ ,          VP4                  VP6                  R7      # )aÜ  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
    1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
    the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
    tened audio logits which are used to calculate the loss.

    2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
    Dia to calculate embeddings and subsequent steps more efficiently.

    If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
    `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
    [`DiaProcessor.__call__`] for more details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

    [What are position IDs?](../glossary#position-ids)
NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frb  r)   rÿ   )rp  r�   r\  )r‘  Ú
fill_valuert   rÜ   r~  r  r#  r©  )rp  r#  Údecoder_hidden_statesÚdecoder_attentionsrˆ  Úencoder_last_hidden_stater~  Úencoder_attentionsrJ   rr   )Ú
ValueErrorÚis_gradient_checkpointingr  ÚloggerÚwarning_oncer	   r   r5   r   r1   r   Úlenrv   r¡  r6   r3   ÚfullÚbos_token_idrt   Úndimrö   r×   r¢  r   rp  r#  r�   r\  rˆ  )r<   r)   rÿ   r¥  r¦  r§  r¨  r#  r©  rÊ   ÚbszÚseq_lenÚchannelsÚdecoder_outputss   &&&&&&&&&,    r?   rz   ÚDiaModel.forward‘  sE  € ðH Ò Ò!8ÜØjóð ð ×)×)Ð)¨d¯m¯m¨mßÜ×#Ñ#Øpôð "�	ç˜Ò0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàÒ"Ø"Ÿlšlñ Ø#ðà-ðð ñ‰Oô ˜O¬_×=Ò=Ü-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð #2°!Õ"4×":Ñ":¸1Õ"=¸rÀ4Ç;Á;×C]ÑC]×CjÑCj�hˆØÒ$Ü %§
¢
Ø˜1˜hÐ'°D·K±K×4NÑ4N×4[Ñ4[Ðdh×doÑdoô!Ðð ×!Ñ! QÔ&Ø 1× 9Ñ 9¸#ÈÓ Q× [Ñ [Ð\]Ð_`Ó aÐàŸ,š,ñ 	
Ø'ð	
à-ð	
ð 2ð	
ð #2°!Õ"4ð		
ð
 $2ð	
ð ,ð	
ð  ð	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5°aÕ&8Ø"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
rA   )r5   r¢  r   )NNNNNNNN)rK   rL   rM   rN   rc   r   r   rz   rX   rY   rZ   r[   s   @@r?   r›  r›  „  s9   ù‡ € ÷ó ð Ø÷]
ñ ]
ó ó ÷]
ð ]
rA   r›  zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   óp   a a€ ] tR tRt oRtRtV3R lV 3R llt]]R	V3R lR ll4       4       t	Rt
VtV ;t# )
ÚDiaForConditionalGenerationió  r(   c                ó    <€ V ^8„  d   QhRS[ /# rC   rE   )rF   rG   s   "€r?   rH   Ú(DiaForConditionalGeneration.__annotate__ü  s   ø€ ÷ ñ ™yñ rA   c                ó†  <€ \         SV `  V4       Wn        \        V4      V n        VP
                  P                  V n        VP
                  P                  V n        \        P                  ! VP
                  P                  V P                  V P                  ,          R R7      V n        RV n        V P                  4        R# )Fr�   ÚForMaskedLMN)r/   rc   r5   r›  r(   r¡  r6   r8   r   rƒ   re   Úlogits_denseÚ	loss_typeri  r‰   s   &&€r?   rc   Ú$DiaForConditionalGeneration.__init__ü  s‘   ø€ Ü‰Ñ˜Ô ØŒÜ˜fÓ%ˆŒ
à"×1Ñ1×>Ñ>ˆÔØ ×/Ñ/×:Ñ:ˆŒÜŸIšIØ×!Ñ!×-Ñ-°×0AÑ0AÀDÇOÁOÕ0SÐ[`ô
ˆÔð 'ˆŒð 	�‰ÖrA   c                ól  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R,          RS[R,          R	S[R,          R
S[ P                  R,          RS[S[,          /
# )rD   r)   Nrÿ   r¥  r¦  r§  r¨  r#  r©  Úlabelsrk   )r3   r�  r   r®   r	   r  r   )rF   rG   s   "€r?   rH   rÀ    sÕ   ø€ ÷ L
ñ L
á×#Ñ# dÕ*ðL
ñ ×(Ñ(¨4Õ/ðL
ñ !×+Ñ+¨dÕ2ð	L
ñ
 $×.Ñ.°Õ5ðL
ñ !&× 0Ñ 0°4Õ 7ðL
ñ )©5Õ0°4Õ7ðL
ñ -¨tÕ3ðL
ñ ˜$•;ðL
ñ × Ñ  4Õ'ðL
ñ 
‘Õ	 ñL
rA   c
                ó„  € V P                   ! RRVRVRVRVRVRVRVRV/V
B pV^ ,          pVP                  ^ ,          pV P                  V4      P                  VRV P                  V P
                  34      P                  ^^4      P                  4       P                  WÐP                  ,          RV P
                  4      pR	pV	e#   V P                  ! RR
VRV	RV P
                  /V
B p\        VVVP                  VP                  VP                  VP                  VP                  VP                  VP                   R7	      # )a   
decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
    1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
    the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
    tened audio logits which are used to calculate the loss.

    2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
    Dia to calculate embeddings and subsequent steps more efficiently.

    If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
    `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
    [`DiaProcessor.__call__`] for more details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

    [What are position IDs?](../glossary#position-ids)
labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in
    `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
    are ignored (masked).
r)   rÿ   r¥  r¦  r§  r¨  r#  r©  NÚlogitsrÇ  r8   )	ÚlossrÉ  r#  r¬  r­  rˆ  r®  r~  r¯  rJ   rr   )r(   rv   rÃ  ru   r6   r8   r×   r
  Úloss_functionr   r#  r¬  r­  rˆ  r®  r~  r¯  )r<   r)   rÿ   r¥  r¦  r§  r¨  r#  r©  rÇ  rÊ   Úoutputsrp  r“  Úaudio_logitsrÊ  s   &&&&&&&&&&,     r?   rz   Ú#DiaForConditionalGeneration.forward  sn  € ðR —*’*ñ 

Øð

à)ð

ð 0ð

ð "6ð	

ð
 $:ð

ð ,ð

ð ,ð

ð  ð

ð ñ

ˆð $ A�JÐØ&×,Ñ,¨QÕ/ˆ
ð ×ÑÐ/Ó0ß‰T�:˜r 4×#4Ñ#4°d·o±oÐFÓGß‰Y�q˜!‹_ß‰Z‹\ß‰T�*×0Ñ0Õ0°"°d·o±oÓFð 	ð ˆØÒØ×%Ò%Ño¨\ÐoÀ&ÐoÐUY×UdÑUdÐoÐhnÑoˆDäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
rA   )r5   rÃ  rÄ  r(   r6   r8   )Úaudio)	NNNNNNNNN)rK   rL   rM   rN   rO   Úoutput_modalitiesrc   r   r   rz   rX   rY   rZ   r[   s   @@r?   r¾  r¾  ó  sG   ù‡ € ð  ÐØ"Ð÷ó ð Ø÷L
ñ L
ó ó ÷L
ð L
rA   r¾  )ro   )r  )r›  r'   r¾  )MÚcollections.abcr   r3   r   Ú r   r9   Úactivationsr   Úcache_utilsr   r   r	   Úintegrationsr
   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.deprecationr   Úutils.genericr   r    Úutils.output_capturingr!   Úconfiguration_diar"   r#   r$   Úgeneration_diar%   Ú
get_loggerrK   r²  r'   r  r2   r   r˜   r²   rè   rñ   rú   r  r  r.  r*   r[  r+   r‡  r›  r¾  Ú__all__rJ   rA   r?   Ú<module>rå     sÙ  ðõ* %ã Ý å &Ý !ß CÑ Cß Lß JÝ BÝ 9÷ó ÷ Lß FÝ &ß lÕ lÝ 0ß GÝ 5ß LÑ LÝ .ð 
×	Ò	˜HÓ	%€ð ô0˜ó 0ó ð0ô$!˜rŸy™yô !ô<)ˆR�Y‰Yô )ñ$ ˜YÓ'ôJ�—‘ó Jó (ðJô(6<˜Ÿ™ô 6<òr(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô;)�r—y‘yó ;)ó +ð;)ô|G)˜Ÿ	™	ô G)ôTÐ0ô ôB5@Ð#ô 5@ôp6Ð0ô 6ôrZ
Ð#ô Z
ñz ðôô
g
Ð!ó g
óð
g
ñT ðôô
a
Ð"4Ð6Hó a
óð
a
òH L‚rA   