+
    œ°³jõk  ã                   óÈ  € R t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIH	t	H
t
 ^RIHtHt ^RIHt ^RIHt ^R	IHtHtHtHt ^R
IHtHt ^RIHt ^RIHtHtHtH t H!t! ^RI"H#t# ^RI$H%t% ^RI&H't'H(t(H)t)H*t* ^RI+H,t, ^RI-H.t.H/t/H0t0 ^RI1H2t2 ]!Pf                  ! ]44      t5] ! R R]4      4       t6 ! R R]Pn                  4      t8 ! R R],4      t9 ! R R](4      t: ! R R])4      t; ! R R]'4      t< ! R R ]Pn                  4      t= ! R! R"]4      t> ! R# R$]64      t? ! R% R&]4      t@ ! R' R(]64      tA]! R)R*7       ! R+ R,]64      4       tB]! R-R*7       ! R. R/]6]24      4       tC. R0OtDR# )1zPyTorch Dia model.)ÚCallableN)Únn)Úinitialization)ÚDynamicCacheÚEncoderDecoderCache)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmerge_with_config_defaults)Úcapture_outputs)ÚLlamaAttentionÚLlamaRMSNormÚLlamaRotaryEmbeddingÚeager_attention_forward)ÚPhi3MLP)Ú	DiaConfigÚDiaDecoderConfigÚDiaEncoderConfig)ÚDiaGenerationMixinc                   ób   a a€ ] tR t^3t oRtRtRtRtRtRt	Rt
RR.tV 3R ltV3R ltRtVtV ;t# )	ÚDiaPreTrainedModelÚmodelTÚ	input_idsÚDiaEncoderLayerÚDiaDecoderLayerc                óB  <€ \         SV `  V4       \        V\        4      '       dy   \        P
                  ! V P                  P                  \        P                  R 7      V P                  P                  ,          p\        P                  ! VP                  V4       R# R# )©ÚdtypeN)ÚsuperÚ_init_weightsÚ
isinstanceÚDiaMultiChannelEmbeddingÚtorchÚarangeÚconfigÚnum_channelsÚlongÚ
vocab_sizeÚinitÚcopy_Úoffsets)ÚselfÚmoduler7   Ú	__class__s   && €Úd/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/dia/modular_dia.pyr,   Ú DiaPreTrainedModel._init_weights?   se   ø€ Ü‰Ñ˜fÔ%Ü�fÔ6×7Ò7Ü—l’l 4§;¡;×#;Ñ#;Ä5Ç:Á:ÔNÐQU×Q\ÑQ\×QgÑQgÕgˆGÜ�JŠJ�v—~‘~ wÖ/ñ 8ó    c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©é   r1   ©r   )ÚformatÚ__classdict__s   "€r;   Ú__annotate__ÚDiaPreTrainedModel.__annotate__3   s   ø‡ ‚ áÑò r=   © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚmain_input_nameÚ_no_split_modulesr,   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r:   rC   s   @@r;   r#   r#   3   sG   ù‡ € ð  ÐØ&*Ð#ØÐØ€NØÐØ!ÐØ!€OØ*Ð,=Ð>Ðõ0÷ … r=   r#   c                   óT   a a€ ] tR t^Ft oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )r.   aâ  In order to efficiently compute the audio embedding from the 9 different channels,
we vectorize the embedding process by using a single embedding layer and an offset.
Example:
- num_embeds = 4
- vocab_size = 8
- num_channels = 3
We would have offsets = [0, 8, 16]
If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
then tokens = audio_codes + offsets
            = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
This allows us to use a single embedding layer for all channels.
c                ó    <€ V ^8„  d   QhRS[ /# r?   ©r   )rB   rC   s   "€r;   rD   Ú%DiaMultiChannelEmbedding.__annotate__T   s   ø€ ÷ <ñ <Ñ/ñ <r=   c                ó¦  <€ \         SV `  4        \        P                  ! VP                  VP
                  ,          VP                  4      V n        VP                  V n        VP
                  V n        \        P                  ! VP
                  \        P                  R 7      VP                  ,          p\        P                  ! VRR7      V n        R# )r)   F)Ú
persistentN)r+   Ú__init__r   Ú	Embeddingr4   r2   Úhidden_sizeÚembedr/   r0   r3   ÚBufferr7   )r8   r1   r7   r:   s   && €r;   r^   Ú!DiaMultiChannelEmbedding.__init__T   sŠ   ø€ Ü‰ÑÔÜ—\’\ &×"3Ñ"3°f×6IÑ6IÕ"IÈ6×K]ÑK]Ó^ˆŒ
Ø!×-Ñ-ˆÔØ"×/Ñ/ˆÔÜ—,’,˜v×2Ñ2¼%¿*¹*ÔEÈ×HYÑHYÕYˆÜ—y’y °UÔ;ˆŽr=   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r@   Úaudio_codesÚreturn)r/   ÚTensor)rB   rC   s   "€r;   rD   r[   \   s#   ø€ ÷ !ñ !¡5§<¡<ð !±E·L±Lñ !r=   c                óŒ  € WP                   P                  VP                  4      ,           P                  RVP                  ^,          VP                  ^,          ,          4      pV P                  V4      P                  VP                  ^ ,          VP                  ^,          RV P                  4      pVP                  ^R7      # )é   )Údiméÿÿÿÿ)r7   ÚtoÚdeviceÚviewÚshapera   r`   Úsum)r8   re   ÚtokensÚembedss   &&  r;   ÚforwardÚ DiaMultiChannelEmbedding.forward\   s•   € Ø§¡§¡°×0BÑ0BÓ CÕC×IÑIØ�×!Ñ! !Õ$ {×'8Ñ'8¸Õ';Õ;ó
ˆð —‘˜FÓ#×(Ñ(¨¯©°a­¸+×:KÑ:KÈAÕ:NÐPRÐTX×TdÑTdÓeˆØ�z‰z˜aˆzÓ Ð r=   )ra   r`   r2   r7   ©
rG   rH   rI   rJ   Ú__doc__r^   rs   rT   rU   rV   rW   s   @@r;   r.   r.   F   s#   ù‡ € ñ÷<ó <÷!÷ !ð !r=   r.   c                   ó   € ] tR t^dtRtR# )ÚDiaMLPrF   N©rG   rH   rI   rJ   rT   rF   r=   r;   rx   rx   d   ó   † Ûr=   rx   c                   ó   € ] tR t^htRtR# )Ú
DiaRMSNormrF   Nry   rF   r=   r;   r|   r|   h   rz   r=   r|   c                   ó   € ] tR t^ltRtR# )ÚDiaRotaryEmbeddingrF   Nry   rF   r=   r;   r~   r~   l   rz   r=   r~   c                   ó:   a € ] tR t^pt o RtRV 3R lR lltRtV tR# )ÚDiaSelfAttentionú=Multi-headed attention from 'Attention Is All You Need' paperc                ó<   <€ V ^8„  d   QhRS[ S[,          RS[RS[/# )r@   r1   Ú	layer_idxÚ	is_causal)r    r   ÚintÚbool)rB   rC   s   "€r;   rD   ÚDiaSelfAttention.__annotate__s   s/   ø€ ÷ ^ñ ^Ñ/Ñ2BÕBð ^Ésð ^Ñ_cñ ^r=   c                óð  € \         P                  P                  V 4       Wn        W n        VP
                  V n        V P                  P                  V n        V P                  P                  ;'       g    V P                  V n        V P                  V P                  ,          V n	        \        VR VP
                  V P                  ,          4      V n        ^V n        RV n        W0n        \         P                  ! V P
                  V P                  V P                  ,          RR7      V n        \         P                  ! V P
                  V P                  V P                  ,          RR7      V n        \         P                  ! V P
                  V P                  V P                  ,          RR7      V n        \         P                  ! V P                  V P                  ,          V P
                  RR7      V n        R# )Úhead_dimç        F©ÚbiasN)r   ÚModuler^   r1   rƒ   r`   Únum_attention_headsÚ	num_headsÚnum_key_value_headsÚnum_key_value_groupsÚgetattrr‰   ÚscalingÚattention_dropoutr„   ÚLinearÚq_projÚk_projÚv_projÚo_proj)r8   r1   rƒ   r„   s   &&&&r;   r^   ÚDiaSelfAttention.__init__s   sI  € Ü
�	‰	×Ñ˜4Ô ØŒØ"ŒØ!×-Ñ-ˆÔØŸ™×8Ñ8ˆŒØ#'§;¡;×#BÑ#B×#TÐ#TÀdÇnÁnˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ü ¨
°F×4FÑ4FÈ$Ï.É.Õ4XÓYˆŒØˆŒØ!$ˆÔØ"Œä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i §¡°·±Õ >À×@PÑ@PÐW\Ô]ˆŽr=   )r”   r1   r‰   r`   r„   r—   rƒ   r�   r‘   r�   r™   r–   r“   r˜   N)F)rG   rH   rI   rJ   rv   r^   rT   rU   )rC   s   @r;   r€   r€   p   s   ø‡ € ÙG÷^÷ ^ð ^r=   r€   c                   óX   a a€ ] tR t^†t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚDiaCrossAttentionr�   c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r@   r1   rƒ   ©r   r…   )rB   rC   s   "€r;   rD   ÚDiaCrossAttention.__annotate__‰   s    ø€ ÷ ^ñ ^Ñ/ð ^¹Cñ ^r=   c                ó”  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        V P                  P                  V n        V P                  P                  V n	        V P                  V P                  ,          V n
        VP                  V n        ^V n        RV n        RV n        \         P"                  ! V P                  V P                  V P                  ,          RR7      V n        \         P"                  ! V P
                  V P                  V P                  ,          RR7      V n        \         P"                  ! V P
                  V P                  V P                  ,          RR7      V n        \         P"                  ! V P                  V P                  ,          V P                  RR7      V n        R# )ri   rŠ   Fr‹   N)r+   r^   r1   rƒ   r`   Úcross_hidden_sizeÚcross_num_attention_headsr�   Úcross_num_key_value_headsr�   r‘   Úcross_head_dimr‰   r“   r”   r„   r   r•   r–   r—   r˜   r™   ©r8   r1   rƒ   r:   s   &&&€r;   r^   ÚDiaCrossAttention.__init__‰   s;  ø€ Ü‰ÑÔØŒØ"ŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØŸ™×>Ñ>ˆŒØ#'§;¡;×#HÑ#HˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø×-Ñ-ˆŒØˆŒØ!$ˆÔØˆŒä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW\Ô]ˆŒÜ—i’i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Õ8`ÐglÔmˆŒÜ—i’i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Õ8`ÐglÔmˆŒÜ—i’i §¡°·±Õ >À×@PÑ@PÐW\Ô]ˆŽr=   c                óò   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r@   Úhidden_statesÚcross_attention_statesÚattention_maskNÚpast_key_valuesÚkwargsrf   )r/   rg   r   r   r	   Útuple)rB   rC   s   "€r;   rD   r    œ   sz   ø€ ÷ 1)ñ 1)á—|‘|ð1)ñ !&§¡ð1)ñ Ÿ™ tÕ+ð	1)ñ
 -¨tÕ3ð1)ñ Ñ-Õ.ð1)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ1)r=   c                ój  € VP                   R R p. VORNV P                  N5p. VP                   R R ORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p	Ve&   VP
                  P                  V P                  4      MRp
Vel   V
'       dd   VP                  P                  V P                  ,          P                  pVP                  P                  V P                  ,          P                  pM¦V P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pVeC   VP                  P                  VVV P                  4      w  r¼RVP
                  V P                  &   \        P                   ! V P"                  P$                  \&        4      pV! V V	VVV3RV P(                  /VB w  rïVP+                  . VORN54      P-                  4       pV P/                  V4      pWï3# )NFTr“   rk   )ro   r‰   r–   rn   Ú	transposeÚ
is_updatedÚgetrƒ   Úcross_attention_cacheÚlayersÚkeysÚvaluesr—   r˜   Úupdater   Úget_interfacer1   Ú_attn_implementationr   r“   ÚreshapeÚ
contiguousr™   )r8   r©   rª   r«   r¬   r­   Úinput_shapeÚhidden_shapeÚcross_shapeÚquery_statesr±   Ú
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss   &&&&&,          r;   rs   ÚDiaCrossAttention.forwardœ   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØMÐ.×4Ñ4°S°bÐ9ÐM¸2ÐM¸t¿}¹}ÑMˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàGVÒGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÒ&¯:à(×>Ñ>×EÑEÀdÇnÁnÕU×ZÑZˆJØ*×@Ñ@×GÑGÈÏÉÕW×^Ñ^‰LàŸ™Ð%;Ó<×AÑAÀ+ÓN×XÑXÐYZÐ\]Ó^ˆJØŸ;™;Ð'=Ó>×CÑCÀKÓP×ZÑZÐ[\Ð^_Ó`ˆLàÒ*à+:×+PÑ+P×+WÑ+WØØ Ø—N‘Nó,Ñ(�
ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð —L‘Lð%
ð ñ%
Ñ!ˆð "×)Ñ)Ð*<¨KÐ*<¸Ñ*<Ó=×HÑHÓJˆØ—k‘k +Ó.ˆØÐ(Ð(r=   )r”   r1   r¢   r‰   r`   r„   r—   rƒ   r�   r‘   r�   r™   r–   r“   r˜   ©NNru   rW   s   @@r;   rœ   rœ   †   s#   ù‡ € ÙG÷^ó ^÷&1)÷ 1)ò 1)r=   rœ   c                   óT   a a€ ] tR t^Ðt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )r&   c                ó&   <€ V ^8„  d   QhRS[ RS[/# rž   )r    r…   )rB   rC   s   "€r;   rD   ÚDiaEncoderLayer.__annotate__Ñ   s   ø€ ÷ "ñ "Ñ/ð "¹Cñ "r=   c                ó   <€ \         SV `  4        \        VP                  VP                  R 7      V n        \        WRR7      V n        \        VP                  VP                  R 7      V n        \        V4      V n
        R# )©ÚepsF©r„   N)r+   r^   r|   r`   Únorm_epsÚpre_sa_normr€   Úself_attentionÚpost_sa_normrx   Úmlpr¦   s   &&&€r;   r^   ÚDiaEncoderLayer.__init__Ñ   sZ   ø€ Ü‰ÑÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ.¨vÈEÔRˆÔÜ& v×'9Ñ'9¸v¿¹ÔOˆÔÜ˜&“>ˆŽr=   c          
      óÞ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[S[,          RS[ P                  /# )r@   r©   Úposition_embeddingsNr«   r­   rf   )r/   rg   r®   r   r	   )rB   rC   s   "€r;   rD   rÉ   Ø   sk   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 Ñ-Õ.ðñ 
�‰ñr=   c                óÆ   € TpV P                  V4      pV P                  ! V3R VRV/VB w  rxWW,           pTpV P                  V4      pV P                  V4      p	WY,           pV# )rÕ   r«   )rÏ   rÐ   rÑ   rÒ   )
r8   r©   rÕ   r«   r­   ÚresidualÚnormed_statesÚself_attn_outputÚ_Úmlp_outs
   &&&&,     r;   rs   ÚDiaEncoderLayer.forwardØ   s‡   € ð !ˆØ×(Ñ(¨Ó7ˆØ"×1Ò1Øñ
à 3ð
ð *ð
ð ñ	
ÑÐð !Õ3ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ Õ*ˆàÐr=   )rÒ   rÑ   rÏ   rÐ   rÆ   ©	rG   rH   rI   rJ   r^   rs   rT   rU   rV   rW   s   @@r;   r&   r&   Ð   s   ù‡ € ÷"ó "÷÷ ò r=   r&   c                   ó~   a a€ ] tR t^ñt oR]R]/tV3R lV 3R llt]]	]
RV3R lR ll4       4       4       tRtVtV ;t# )	Ú
DiaEncoderr©   Ú
attentionsc                ó    <€ V ^8„  d   QhRS[ /# r?   )r    )rB   rC   s   "€r;   rD   ÚDiaEncoder.__annotate__÷   ó   ø€ ÷ ñ Ñ/ñ r=   c           	     ó¸  <€ \         SV `  V4       Wn        \        P                  ! VP
                  VP                  4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \!        VR7      V n        V P%                  4        R# u upi ©rË   ©r1   N)r+   r^   r1   r   r_   r4   r`   Ú	embeddingÚ
ModuleListÚrangeÚnum_hidden_layersr&   r´   r|   rÎ   Únormr~   Ú
rotary_embÚ	post_initr¦   s   && €r;   r^   ÚDiaEncoder.__init__÷   s�   ø€ Ü‰Ñ˜Ô ØŒäŸš f×&7Ñ&7¸×9KÑ9KÓLˆŒÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÑA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô ˜v×1Ñ1°v·±ÔGˆŒ	Ü,°FÔ;ˆŒà�‰Öùò bs   Á.Cc                óx   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[/# )r@   r%   r«   Nr­   rf   )r/   rg   r   r   r   )rB   rC   s   "€r;   rD   râ     sL   ø€ ÷ @ñ @á—<‘<ð@ñ Ÿ™ tÕ+ð@ñ Ñ+Õ,ð	@ñ
 
ñ@r=   c           
     ój  € V P                  V4      p\        P                  ! VP                  R,          VP                  R7      R	,          p\        V P                  VVR7      pV P                  WER7      pV P                   F  pV! V3RVRVRV/VB pK  	  V P                  V4      p\        VR7      # )
ri   ©rm   )r1   Úinputs_embedsr«   ©Úposition_idsr«   rô   rÕ   )Úlast_hidden_staterk   )N:NNN)rç   r/   r0   ro   rm   r   r1   rì   r´   rë   r   )r8   r%   r«   r­   r©   rô   rÕ   Úencoder_layers   &&&,    r;   rs   ÚDiaEncoder.forward  s¿   € ð Ÿ™ yÓ1ˆô
 —|’| I§O¡O°BÕ$7À	×@PÑ@PÔQÐRYÕZˆä2Ø—;‘;Ø'Ø)ô
ˆð
 #Ÿo™o¨m˜oÓWÐà!Ÿ[œ[ˆMÙ)Øñà-ðð *ðð %8ð	ð
 ñŠMñ )ð Ÿ	™	 -Ó0ˆä°Ô?Ð?r=   )r1   rç   r´   rë   rì   ©N)rG   rH   rI   rJ   r&   r€   Ú_can_record_outputsr^   r   r   r   rs   rT   rU   rV   rW   s   @@r;   rß   rß   ñ   sT   ù‡ € à˜ØÐ&ðÐ÷
ó ð  ØØ÷@ñ @ó ó ó  ÷@ð @r=   rß   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )r'   i)  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rž   rŸ   )rB   rC   s   "€r;   rD   ÚDiaDecoderLayer.__annotate__*  s   ø€ ÷ "ñ "Ñ/ð "¹Cñ "r=   c                óŽ  <€ \         SV `  4        VP                  V n        \	        WR R7      V n        \        W4      V n        \        VP                  VP                  R7      V n
        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \        V4      V n        R# )TrÍ   rË   N)r+   r^   r`   Ú	embed_dimr€   rÐ   rœ   Úcross_attentionr|   rÎ   rÏ   Úpre_ca_normÚpre_mlp_normrx   rÒ   r¦   s   &&&€r;   r^   ÚDiaDecoderLayer.__init__*  s�   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ.¨vÈDÔQˆÔÜ0°ÓCˆÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ& v×'9Ñ'9¸v¿¹ÔOˆÔÜ˜&“>ˆŽr=   c                óŠ  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          S[ P                  R,          3,          /# )	r@   r©   rÕ   Nr«   Úencoder_hidden_statesÚencoder_attention_maskr¬   rf   )r/   rg   r®   r   )rB   rC   s   "€r;   rD   rü   4  s®   ø€ ÷ +ñ +á—|‘|ð+ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð+ñ Ÿ™ tÕ+ð	+ñ
  %Ÿ|™|¨dÕ2ð+ñ !&§¡¨tÕ 3ð+ñ -¨tÕ3ð+ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%·,±,ÀÕ2EÐEÕ	Fñ+r=   c                óv  € Tp\        V\        4      '       d   VP                  pTp	V P                  V4      p
V P                  ! V
VVV3/ VB w  r¼W›,           pTp	V P                  V4      p
V P                  ! V
V3R VRV/VB w  rÜW�,           pTp	V P                  V4      p
V P                  V
4      pWž,           pV# )r«   r¬   )	r-   r   Úself_attention_cacherÏ   rÐ   r   rÿ   r  rÒ   )r8   r©   rÕ   r«   r  r  r¬   r­   Úself_attn_cacher×   rØ   rÙ   rÚ   Úcross_statesrÛ   s   &&&&&&&,       r;   rs   ÚDiaDecoderLayer.forward4  só   € ð *ˆÜ�oÔ':×;Ò;Ø-×BÑBˆOà ˆØ×(Ñ(¨Ó7ˆØ"×1Ò1ØØØð ñ
ð ñ
ÑÐð !Õ3ˆà ˆØ×(Ñ(¨Ó7ˆØ×.Ò.ØØ!ñ
ð 2ð
ð ,ð	
ð
 ñ
‰ˆð !Õ/ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ Õ*ˆàÐr=   )rÿ   rþ   rÒ   r   r  rÏ   rÐ   ©NNNNNrÝ   rW   s   @@r;   r'   r'   )  s   ù‡ € ÷"ó "÷+÷ +ò +r=   r'   c                   ó†   a a€ ] tR tRt oRtR]R]R]/tV3R lV 3R llt	]
]]RV3R lR	 ll4       4       4       tR
tVtV ;t# )Ú
DiaDecoderib  z-Transformer Decoder Stack using DenseGeneral.r©   rà   Úcross_attentionsc                ó    <€ V ^8„  d   QhRS[ /# r?   rZ   )rB   rC   s   "€r;   rD   ÚDiaDecoder.__annotate__k  rã   r=   c           	     ó°  <€ \         SV `  V4       VP                  V n        VP                  V n        \	        V4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \!        VR7      V n        V P%                  4        R# u upi rå   )r+   r^   r2   r4   r.   Ú
embeddingsr   rè   ré   rê   r'   r´   r|   r`   rÎ   rë   r~   rì   rí   r¦   s   && €r;   r^   ÚDiaDecoder.__init__k  s¡   ø€ Ü‰Ñ˜Ô Ø"×/Ñ/ˆÔØ ×+Ñ+ˆŒÜ2°6Ó:ˆŒÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÑA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô ˜v×1Ñ1°v·±ÔGˆŒ	Ü,°FÔ;ˆŒà�‰Öùò bs   Á*Cc                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[S[,          /# )
r@   r%   rô   Nr«   r  r  r¬   r­   rf   )	r/   rg   Ú
LongTensorÚFloatTensorr   r   r   r   r®   )rB   rC   s   "€r;   rD   r  {  s£   ø€ ÷ A
ñ A
á—<‘<ðA
ñ ×&Ñ&¨Õ-ðA
ñ Ÿ™ tÕ+ð	A
ñ
  %×0Ñ0°4Õ7ðA
ñ !&× 0Ñ 0°4Õ 7ðA
ñ -¨tÕ3ðA
ñ Ñ+Õ,ðA
ñ 
3±UÕ	:ñA
r=   c           
     ó~  € VP                  4       RR
 w  r‰Ve   VP                  4       M^ p
Vf:   \        P                  ! W‘P                  R7      V
,           pVP                  ^ 4      pV P                  V4      pVf;   \        4       '       g+   W©,           p\        P                  ! WŒVP                  R7      p\        V P                  VVVR7      p\        V P                  VVVR7      pV P                  W²R7      pV P                   F  pV! VVVV3RVRVRV/VB pK  	  V P                  V4      p\        VVR	7      # )zç
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
    The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

    [What are input IDs?](../glossary#input-ids)
Nrñ   )r1   rò   r«   r¬   )r1   rò   r«   r  ró   r  r¬   rô   )rõ   r¬   rk   )ÚsizeÚget_seq_lengthr/   r0   rm   Ú	unsqueezer  r   Úonesr   r1   r   rì   r´   rë   r   )r8   r%   rô   r«   r  r  r¬   r­   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthr©   Úmask_seq_lengthrÕ   Úlayers   &&&&&&&,       r;   rs   ÚDiaDecoder.forwardx  sY  € ð( "+§¡Ó!1°#°2Ð!6Ñˆ
ØETÒE` ×!?Ñ!?Ô!AÐfgÐàÒÜ Ÿ<š<¨
×;KÑ;KÔLÐOeÕeˆLØ'×1Ñ1°!Ó4ˆLð Ÿ™¨	Ó2ˆàÒ!Ô*B×*DÒ*Dà4ÕAˆOÜ"ŸZšZ¨
ÈI×L\ÑL\Ô]ˆNä+Ø—;‘;Ø'Ø)Ø+ô	
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð #Ÿo™o¨m˜oÓWÐà—[”[ˆEÙ!Øð $ØØ%ñð (>ðð !0ðð *ðð ñŠMñ !ð Ÿ	™	 -Ó0ˆä8Ø+Ø+ô
ð 	
r=   )r  r´   rë   r2   rì   r4   r  )rG   rH   rI   rJ   rv   r'   r€   rœ   rù   r^   r   r   r   rs   rT   rU   rV   rW   s   @@r;   r  r  b  s_   ù‡ € Ù7ð 	˜ØÐ&ØÐ-ðÐ÷ó ð  ØØ÷A
ñ A
ó ó ó  ÷A
ð A
r=   r  z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )Úcustom_introc                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚDiaModeli¿  c                ó    <€ V ^8„  d   QhRS[ /# r?   rA   )rB   rC   s   "€r;   rD   ÚDiaModel.__annotate__Å  s   ø€ ÷ ñ ™yñ r=   c                óº   <€ \         SV `  V4       Wn        \        VP                  4      V n        \        VP                  4      V n        V P                  4        R # rø   )
r+   r^   r1   rß   Úencoder_configÚencoderr  Údecoder_configÚdecoderrí   ©r8   r1   r:   s   &&€r;   r^   ÚDiaModel.__init__Å  sC   ø€ Ü‰Ñ˜Ô ØŒÜ! &×"7Ñ"7Ó8ˆŒÜ! &×"7Ñ"7Ó8ˆŒØ�‰Ör=   c                óD  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R,          RS[R,          R	S[R,          R
S[S[,          /	# )r@   r%   Nr«   Údecoder_input_idsÚdecoder_position_idsÚdecoder_attention_maskÚencoder_outputsr¬   Ú	use_cacherf   )r/   r  r   r®   r   r†   r   )rB   rC   s   "€r;   rD   r&  Î  sÀ   ø€ ÷ ]
ñ ]
á×#Ñ# dÕ*ð]
ñ ×(Ñ(¨4Õ/ð]
ñ !×+Ñ+¨dÕ2ð	]
ñ
 $×.Ñ.°Õ5ð]
ñ !&× 0Ñ 0°4Õ 7ð]
ñ )©5Õ0°4Õ7ð]
ñ -¨tÕ3ð]
ñ ˜$•;ð]
ñ 
Ñ#Õ	#ñ]
r=   c	                ó˜  € Vf   Vf   \        R4      hV P                  '       d2   V P                  '       d    V'       d   \        P	                  R4       RpV'       d9   Vf5   \        \        V P                  R7      \        V P                  R7      4      pVf   V P                  ! RRVRV/V	B pM]\        V\        4      '       gH   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV^ ,          P                  ^ ,          RV P                  P                  P                  rËp
VfE   \        P                   ! V
^V3V P                  P                  P"                  V P$                  R	7      pVP&                  ^8X  d#   VP)                  W¬V4      P+                  ^^4      pV P,                  ! RRVR
VRVRV^ ,          RVRVRV/V	B p\/        VP0                  VP2                  VP4                  VP6                  VP8                  V^ ,          VP4                  VP6                  R7      # )aÜ  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
    1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
    the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
    tened audio logits which are used to calculate the loss.

    2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
    Dia to calculate embeddings and subsequent steps more efficiently.

    If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
    `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
    [`DiaProcessor.__call__`] for more details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

    [What are position IDs?](../glossary#position-ids)
NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fræ   r%   r«   )rõ   r©   rà   )r  Ú
fill_valuerm   rô   r  r  r¬   r3  )rõ   r¬   Údecoder_hidden_statesÚdecoder_attentionsr  Úencoder_last_hidden_stater  Úencoder_attentionsrF   rk   )Ú
ValueErrorÚis_gradient_checkpointingÚtrainingÚloggerÚwarning_oncer   r   r1   r)  r-   r   Úlenro   r*  r2   r/   ÚfullÚbos_token_idrm   Úndimrº   r°   r+  r   rõ   r¬   r©   rà   r  )r8   r%   r«   r/  r0  r1  r2  r¬   r3  r­   ÚbszÚseq_lenÚchannelsÚdecoder_outputss   &&&&&&&&&,    r;   rs   ÚDiaModel.forwardÌ  sE  € ðH Ò Ò!8ÜØjóð ð ×)×)Ð)¨d¯m¯m¨mßÜ×#Ñ#Øpôð "�	ç˜Ò0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàÒ"Ø"Ÿlšlñ Ø#ðà-ðð ñ‰Oô ˜O¬_×=Ò=Ü-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð #2°!Õ"4×":Ñ":¸1Õ"=¸rÀ4Ç;Á;×C]ÑC]×CjÑCj�hˆØÒ$Ü %§
¢
Ø˜1˜hÐ'°D·K±K×4NÑ4N×4[Ñ4[Ðdh×doÑdoô!Ðð ×!Ñ! QÔ&Ø 1× 9Ñ 9¸#ÈÓ Q× [Ñ [Ð\]Ð_`Ó aÐàŸ,š,ñ 	
Ø'ð	
à-ð	
ð 2ð	
ð #2°!Õ"4ð		
ð
 $2ð	
ð ,ð	
ð  ð	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5°aÕ&8Ø"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r=   )r1   r+  r)  )NNNNNNNN)rG   rH   rI   rJ   r^   r   r   rs   rT   rU   rV   rW   s   @@r;   r$  r$  ¿  s9   ù‡ € ÷ó ð Ø÷]
ñ ]
ó ó ÷]
ð ]
r=   r$  zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   óp   a a€ ] tR tRt oRtRtV3R lV 3R llt]]R	V3R lR ll4       4       t	Rt
VtV ;t# )
ÚDiaForConditionalGenerationi.  r$   c                ó    <€ V ^8„  d   QhRS[ /# r?   rA   )rB   rC   s   "€r;   rD   Ú(DiaForConditionalGeneration.__annotate__7  s   ø€ ÷ ñ ™yñ r=   c                ó†  <€ \         SV `  V4       Wn        \        V4      V n        VP
                  P                  V n        VP
                  P                  V n        \        P                  ! VP
                  P                  V P                  V P                  ,          R R7      V n        RV n        V P                  4        R# )Fr‹   ÚForMaskedLMN)r+   r^   r1   r$  r$   r*  r2   r4   r   r•   r`   Úlogits_denseÚ	loss_typerí   r,  s   &&€r;   r^   Ú$DiaForConditionalGeneration.__init__7  s‘   ø€ Ü‰Ñ˜Ô ØŒÜ˜fÓ%ˆŒ
à"×1Ñ1×>Ñ>ˆÔØ ×/Ñ/×:Ñ:ˆŒÜŸIšIØ×!Ñ!×-Ñ-°×0AÑ0AÀDÇOÁOÕ0SÐ[`ô
ˆÔð 'ˆŒð 	�‰Ör=   c                ól  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R,          RS[R,          R	S[R,          R
S[ P                  R,          RS[S[,          /
# )r@   r%   Nr«   r/  r0  r1  r2  r¬   r3  Úlabelsrf   )r/   r  r   r®   r   r†   r   )rB   rC   s   "€r;   rD   rK  H  sÕ   ø€ ÷ L
ñ L
á×#Ñ# dÕ*ðL
ñ ×(Ñ(¨4Õ/ðL
ñ !×+Ñ+¨dÕ2ð	L
ñ
 $×.Ñ.°Õ5ðL
ñ !&× 0Ñ 0°4Õ 7ðL
ñ )©5Õ0°4Õ7ðL
ñ -¨tÕ3ðL
ñ ˜$•;ðL
ñ × Ñ  4Õ'ðL
ñ 
‘Õ	 ñL
r=   c
                ó„  € V P                   ! RRVRVRVRVRVRVRVRV/V
B pV^ ,          pVP                  ^ ,          pV P                  V4      P                  VRV P                  V P
                  34      P                  ^^4      P                  4       P                  WÐP                  ,          RV P
                  4      pR	pV	e#   V P                  ! RR
VRV	RV P
                  /V
B p\        VVVP                  VP                  VP                  VP                  VP                  VP                  VP                   R7	      # )a   
decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
    1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
    the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
    tened audio logits which are used to calculate the loss.

    2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
    Dia to calculate embeddings and subsequent steps more efficiently.

    If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
    `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
    [`DiaProcessor.__call__`] for more details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

    [What are position IDs?](../glossary#position-ids)
labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in
    `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
    are ignored (masked).
r%   r«   r/  r0  r1  r2  r¬   r3  NÚlogitsrR  r4   )	ÚlossrT  r¬   r6  r7  r  r8  r  r9  rF   rk   )r$   ro   rN  rn   r2   r4   r°   r»   Úloss_functionr   r¬   r6  r7  r  r8  r  r9  )r8   r%   r«   r/  r0  r1  r2  r¬   r3  rR  r­   Úoutputsrõ   r  Úaudio_logitsrU  s   &&&&&&&&&&,     r;   rs   Ú#DiaForConditionalGeneration.forwardF  sn  € ðR —*’*ñ 

Øð

à)ð

ð 0ð

ð "6ð	

ð
 $:ð

ð ,ð

ð ,ð

ð  ð

ð ñ

ˆð $ A�JÐØ&×,Ñ,¨QÕ/ˆ
ð ×ÑÐ/Ó0ß‰T�:˜r 4×#4Ñ#4°d·o±oÐFÓGß‰Y�q˜!‹_ß‰Z‹\ß‰T�*×0Ñ0Õ0°"°d·o±oÓFð 	ð ˆØÒØ×%Ò%Ño¨\ÐoÀ&ÐoÐUY×UdÑUdÐoÐhnÑoˆDäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r=   )r1   rN  rO  r$   r2   r4   )Úaudio)	NNNNNNNNN)rG   rH   rI   rJ   rK   Úoutput_modalitiesr^   r   r   rs   rT   rU   rV   rW   s   @@r;   rI  rI  .  sG   ù‡ € ð  ÐØ"Ð÷ó ð Ø÷L
ñ L
ó ó ÷L
ð L
r=   rI  )r$  r#   rI  )Erv   Úcollections.abcr   r/   r   Ú r   r5   Úcache_utilsr   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   r   Úphi3.modeling_phi3r   Úconfiguration_diar   r   r    Úgeneration_diar!   Ú
get_loggerrG   r=  r#   r�   r.   rx   r|   r~   r€   rœ   r&   rß   r'   r  r$  rI  Ú__all__rF   r=   r;   Ú<module>rn     sw  ðñ å $ã Ý å &ß <ß JÝ BÝ 9÷ó ÷ GÝ &ß lÕ lÝ 7Ý 5÷ó õ )ß LÑ LÝ .ð 
×	Ò	˜HÓ	%€ð ô0˜ó 0ó ð0ô$!˜rŸy™yô !ô<	ˆWô 	ô	�ô 	ô	Ð-ô 	ô^�~ô ^ô,G)˜Ÿ	™	ô G)ôTÐ0ô ôB5@Ð#ô 5@ôp6Ð0ô 6ôrZ
Ð#ô Z
ñz ðôô
g
Ð!ó g
óð
g
ñT ðôô
a
Ð"4Ð6Hó a
óð
a
òH L‚r=   