+
    œ°³jBc  ã                   ó  € R t ^ RIt^ RIt^ RIHt ^RIHt ^RIHt ^RI	H
t
HtHt ^RIHt ^RIHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHt ^RIHtHtHt ^RIH t  ^RI!H"t"H#t# ^RI$H%t% ]PL                  ! ]'4      t( ! R R]PR                  4      t* ! R R]PV                  4      t, ! R R]PV                  4      t- ! R R]4      t.] ! R R]4      4       t/] ! R R]/4      4       t0]! RR7       ! R R]/]4      4       t1. R Ot2R# )!zPyTorch XGLM model.N)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentions)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©Ú
XGLMConfigc                   ó^   a a€ ] tR t^(t oRtRV3R lV 3R llltV3R lV 3R lltRtVtV ;t	# )ÚXGLMScaledWordEmbeddingzT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
c          	      ó@   <€ V ^8„  d   QhRS[ RS[ RS[ RS[R,          /# )é   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scaleN)ÚintÚfloat)ÚformatÚ__classdict__s   "€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/xglm/modeling_xglm.pyÚ__annotate__Ú$XGLMScaledWordEmbedding.__annotate__-   s0   ø€ ÷ 'ñ '¡sð '¹3ð 'ÉSð 'Ñ_dÐgkÕ_kñ 'ó    c                ó4   <€ \         SV `  WV4       W@n        R # ©N)ÚsuperÚ__init__r   )Úselfr   r   r   r   Ú	__class__s   &&&&&€r$   r+   Ú XGLMScaledWordEmbedding.__init__-   s   ø€ Ü‰Ñ˜¸ÔDØ&Ör'   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   Ú	input_ids)ÚtorchÚTensor)r"   r#   s   "€r$   r%   r&   1   s   ø€ ÷ =ñ =¡§¡ñ =r'   c                óD   <€ \         SV `  V4      V P                  ,          # r)   )r*   Úforwardr   )r,   r0   r-   s   &&€r$   r4   ÚXGLMScaledWordEmbedding.forward1   s   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<Õ<Ð<r'   ©r   )ç      ð?©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r+   r4   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r-   r#   s   @@r$   r   r   (   s#   ù‡ € ñ÷'õ '÷=÷ =ó =r'   r   c                   óº   a a€ ] tR t^5t oRtRV3R lV 3R llltRV3R lR llt]RV3R lR ll4       t]	P                  ! 4       RV3R lR	 ll4       tR
tVtV ;t# )Ú!XGLMSinusoidalPositionalEmbeddingzDThis module produces sinusoidal positional embeddings of any length.c                ó:   <€ V ^8„  d   QhRS[ RS[ RS[ R,          /# )r   Únum_positionsr   r   N©r    )r"   r#   s   "€r$   r%   Ú.XGLMSinusoidalPositionalEmbedding.__annotate__8   s,   ø€ ÷ Sñ S¡cð S¹#ð SÉCÐRVÍJñ Sr'   c                óš   <€ \         SV `  4        ^V n        Wn        W n        W0n        V P                  WP                  ,           W#4       R# )r   N)r*   r+   ÚoffsetrE   r   r   Úmake_weights)r,   rE   r   r   r-   s   &&&&€r$   r+   Ú*XGLMSinusoidalPositionalEmbedding.__init__8   s>   ø€ Ü‰ÑÔØˆŒØ*ÔØ*ÔØ&ÔØ×Ñ˜-¯+©+Õ5°}ÖRr'   c                ó:   <€ V ^8„  d   QhRS[ RS[ RS[ R,          /# ©r   r   r   r   NrF   )r"   r#   s   "€r$   r%   rG   @   s.   ø€ ÷ @ñ @©3ð @¹sð @ÑQTÐW[ÕQ[ñ @r'   c                óþ   € V P                  WV4      p\        V R 4      '       d<   VP                  V P                  P                  V P                  P
                  R7      p\        P                  ! VRR7      V n        R# )Úweights©ÚdtypeÚdeviceF)Ú
persistentN)Úget_embeddingÚhasattrÚtorO   rQ   rR   r   ÚBuffer)r,   r   r   r   Úemb_weightss   &&&& r$   rJ   Ú.XGLMSinusoidalPositionalEmbedding.make_weights@   sY   € Ø×(Ñ(¨ÈÓTˆÜ�4˜×#Ò#à%Ÿ.™.¨t¯|©|×/AÑ/AÈ$Ï,É,×J]ÑJ]˜.Ó^ˆKä—y’y ¸Ô?ˆŽr'   c                ó:   <€ V ^8„  d   QhRS[ RS[ RS[ R,          /# rM   rF   )r"   r#   s   "€r$   r%   rG   I   s'   ø€ ÷ 1ñ 1¡cð 1¹#ð 1ÉCÐRVÍJñ 1r'   c                ó(  € V^,          p\         P                  ! R4      V^,
          ,          p\        P                  ! \        P                  ! V\        P
                  R7      P                  4       V) ,          4      p\        P                  ! V \        P
                  R7      P                  4       P                  ^4      VP                  ^ 4      ,          p\        P                  ! \        P                  ! V4      \        P                  ! V4      .^R7      P                  V R4      pV^,          ^8X  d0   \        P                  ! V\        P                  ! V ^4      .^R7      pVe   ^ WBR3&   VP                  \        P                  ! 4       4      # )zª
Build sinusoidal embeddings.

This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
"Attention Is All You Need".
i'  )rQ   ©ÚdimºNNNéÿÿÿÿ)ÚmathÚlogr1   ÚexpÚarangeÚint64r!   Ú	unsqueezeÚcatÚsinÚcosÚviewÚzerosrV   Úget_default_dtype)r   r   r   Úhalf_dimÚembs   &&&  r$   rT   Ú/XGLMSinusoidalPositionalEmbedding.get_embeddingH   s  € ð ! AÕ%ˆÜ�hŠh�u‹o ¨A¥Õ.ˆÜ�iŠiœŸš X´U·[±[ÔA×GÑGÓIÈSÈDÕPÓQˆÜ�lŠl˜>´·±Ô=×CÑCÓE×OÑOÐPQÓRÐUX×UbÑUbÐcdÓUeÕeˆÜ�iŠiœŸš 3›¬¯ª°3«Ð8¸aÔ@×EÑEÀnÐVXÓYˆØ˜1Õ Ô!ä—)’)˜S¤%§+¢+¨n¸aÓ"@ÐAÀqÔIˆCØÒ"Ø"#ˆC˜Q�Ñà�v‰v”e×-Ò-Ó/Ó0Ð0r'   c                óH   <€ V ^8„  d   QhRS[ P                  R,          RS[/# )r   Úposition_idsNÚpast_key_values_length)r1   r2   r    )r"   r#   s   "€r$   r%   rG   ^   s)   ø€ ÷ wñ w¡E§L¡L°4Õ$7ð wÑX[ñ wr'   c                ó¼  € VP                  4       w  r4WP                  ,           p^V,           V,           pWPP                  P                  ^ 4      8”  d'   V P                  WPP                  V P
                  4       V P                  P                  ^ VP                  R4      4      P                  W4V P                  P                  R,          4      P                  4       # )r   r_   )
ÚsizerI   rO   rJ   r   r   Úindex_selectri   ÚshapeÚdetach)r,   rp   rq   ÚbszÚseq_lenÚmax_poss   &&&   r$   r4   Ú)XGLMSinusoidalPositionalEmbedding.forward]   s©   € à#×(Ñ(Ó*‰ˆØ#§k¡kÕ1ˆà�g•+Ð 6Õ6ˆØ—\‘\×&Ñ& qÓ)Ô)Ø×Ñ˜g×'9Ñ'9¸4×;KÑ;KÔLà�|‰|×(Ñ(¨¨L×,=Ñ,=¸bÓ,AÓB×GÑGÈÐVZ×VbÑVb×VhÑVhÐikÕVlÓm×tÑtÓvÐvr'   )r   rE   rI   r   rO   r)   )Né    )r9   r:   r;   r<   r=   r+   rJ   ÚstaticmethodrT   r1   Úno_gradr4   r>   r?   r@   rA   s   @@r$   rC   rC   5   sa   ù‡ € ÙN÷Sõ S÷@ò @ð ÷1ñ 1ó ð1ð( ‡]‚]ƒ_÷wñ wó ÷wð wr'   rC   c                   ó\   a a€ ] tR t^it oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚXGLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                óv   <€ V ^8„  d   QhRS[ RS[ RS[R,          RS[R,          RS[R,          RS[R,          /# )r   Ú	embed_dimÚ	num_headsÚdropoutNÚ
is_decoderÚbiasÚ	layer_idx)r    r!   Úbool)r"   r#   s   "€r$   r%   ÚXGLMAttention.__annotate__l   s_   ø€ ÷ Cñ CáðCñ ðCñ ˜•ð	Cñ
 ˜4•KðCñ �T�kðCñ ˜$•;ñCr'   c                ó  <€ \         SV `  4        Wn        W n        W0n        W,          V n        V P
                  V,          V P                  8w  d   \        R V P                   RV R24      hV P
                  R,          V n        W@n        W`n	        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        R# )z;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).©r…   Ng      à¿)r*   r+   r�   r‚   rƒ   Úhead_dimÚ
ValueErrorÚscalingr„   r†   r   ÚLinearÚk_projÚv_projÚq_projÚout_proj)r,   r�   r‚   rƒ   r„   r…   r†   r-   s   &&&&&&&€r$   r+   ÚXGLMAttention.__init__l   sÉ   ø€ ô 	‰ÑÔØ"ŒØ"ŒØŒØ!Õ.ˆŒà�M‰M˜IÕ%¨$¯.©.Ô8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÕ*ˆŒØ$ŒØ"Œä—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜŸ	š	 )¸TÔBˆŽr'   c                ó6  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r   Úhidden_statesÚkey_value_statesNÚpast_key_valuesÚattention_maskÚkwargsÚreturn)r1   r2   r   r   r   Útuple)r"   r#   s   "€r$   r%   rˆ   ‰   s–   ø€ ÷ l2ñ l2á—|‘|ðl2ñ  Ÿ,™,¨Õ-ðl2ñ  �ð	l2ñ
 Ÿ™ tÕ+ðl2ñ Ñ+Õ,ðl2ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñl2r'   c           	     óÄ  € VRJpVP                  4       w  rxp	V'       d   VP                  ^,          MTp
V P                  V4      V P                  ,          pRpVe`   \	        V\
        4      '       dH   VP                  P                  V P                  4      pV'       d   VP                  pMVP                  pMTpV'       d   TMTpV'       d\   VeX   V'       dP   XP                  V P                  ,          P                  pVP                  V P                  ,          P                  pM×V P                  V4      pV P                  V4      pVP!                  WzRV P"                  4      P%                  ^^4      pVP!                  WzRV P"                  4      P%                  ^^4      pVeX   XP'                  VVV P                  4      w  ppV'       d0   \	        V\
        4      '       d   RVP                  V P                  &   WpP(                  ,          RV P"                  3pVP!                  WxV P(                  V P"                  4      P%                  ^^4      pVP*                  ! V!  pVP*                  ! V!  pVP*                  ! V!  pVP                  ^4      p
\,        P.                  ! W¿P%                  ^^4      4      pVP                  4       WpP(                  ,          WŠ38w  d2   \1        RWpP(                  ,          WŠ3 RVP                  4        24      hVeà   VP                  4       V^WŠ38w  d#   \1        RV^WŠ3 RVP                  4        24      hVP!                  WpP(                  WŠ4      V,           p\,        P2                  ! V\,        P4                  ! R\,        P6                  ! VP8                  4      P:                  VP<                  R7      4      pVP!                  WpP(                  ,          WŠ4      pVP8                  \,        P>                  8X  dO   \@        PB                  PE                  VR\,        PF                  R7      PI                  \,        P>                  4      pM!\@        PB                  PE                  VRR	7      pVP!                  WpP(                  WŠ4      pVP!                  WpP(                  ,          WŠ4      p\@        PB                  PK                  VV PJ                  V PL                  R
7      p\,        P.                  ! VV4      pVP                  4       WpP(                  ,          W€P"                  38w  d6   \1        RWpP(                  W€P"                  3 RVP                  4        24      hVP!                  WpP(                  W€P"                  4      pVP%                  ^^4      pVP+                  WxV PN                  4      pV PQ                  V4      pVV3# )z#Input shape: Batch x Time x ChannelNFTz$Attention weights should be of size z	, but is z!Attention mask should be of size )rR   )r]   rQ   r\   ©ÚpÚtrainingz `attn_output` should be of size r_   © ))rs   ru   r‘   r�   Ú
isinstancer   Ú
is_updatedÚgetr†   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesr�   r�   ri   r‹   Ú	transposeÚupdater‚   Úreshaper1   ÚbmmrŒ   ÚmaxÚfullÚfinforQ   ÚminrR   Úfloat16r   Ú
functionalÚsoftmaxÚfloat32rV   rƒ   rŸ   r�   r’   )r,   r•   r–   r—   r˜   r™   Úis_cross_attentionrw   Útgt_lenÚ_Úsrc_lenÚquery_statesr¢   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚ
proj_shapeÚattn_weightsÚattn_weights_reshapedÚ
attn_probsÚattn_outputs   &&&&&,                r$   r4   ÚXGLMAttention.forward‰   sÜ  € ð .°TÐ9Ðà'×,Ñ,Ó.‰ˆ�aß/AÐ"×(Ñ(¨Ö+Àwˆð —{‘{ =Ó1°D·L±LÕ@ˆàˆ
ØÒ&Ü˜/Ô+>×?Ò?Ø,×7Ñ7×;Ñ;¸D¿N¹NÓK�
ß%à+:×+PÑ+PÑ(à+:×+OÑ+OÑ(à'6Ð$ç-?Ñ)À]ˆß /Ò"=Ç*à-×4Ñ4°T·^±^ÕD×IÑIˆJØ/×6Ñ6°t·~±~ÕF×MÑM‰LàŸ™ ^Ó4ˆJØŸ;™; ~Ó6ˆLØ#Ÿ™¨°r¸4¿=¹=ÓI×SÑSÐTUÐWXÓYˆJØ'×,Ñ,¨S¸2¸t¿}¹}ÓM×WÑWÐXYÐ[\Ó]ˆLàÒ*à+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lç%¬*°_ÔFY×*ZÒ*ZØAE�O×.Ñ.¨t¯~©~Ñ>àŸN™NÕ*¨B°·±Ð>ˆ
Ø#×(Ñ(¨°t·~±~ÀtÇ}Á}ÓU×_Ñ_Ð`aÐcdÓeˆØ#×+Ò+¨ZÑ8ˆØ×'Ò'¨Ñ4ˆ
Ø#×+Ò+¨ZÑ8ˆà—/‘/ !Ó$ˆÜ—y’y ×/CÑ/CÀAÀqÓ/IÓJˆà×ÑÓ 3¯©Õ#7¸Ð"JÔJÜØ6¸¿n¹nÕ8LÈgÐ7_Ð6`ð aØ ×%Ñ%Ó'Ð(ð*óð ð
 Ò%Ø×"Ñ"Ó$¨¨a°Ð(BÔBÜ Ø7¸¸aÀÐ8RÐ7SÐS\Ð]k×]pÑ]pÓ]rÐ\sÐtóð ð (×,Ñ,¨S·.±.À'ÓSÐVdÕdˆLÜ Ÿ9š9ØœeŸjšj¨¬U¯[ª[¸×9KÑ9KÓ-L×-PÑ-PÐYe×YlÑYlÔmóˆLð (×,Ñ,¨S·>±>Õ-AÀ7ÓTˆLð ×Ñ¤§¡Ô.ÜŸ=™=×0Ñ0°À2ÌUÏ]É]Ð0Ó[×^Ñ^Ô_d×_lÑ_lÓm‰LäŸ=™=×0Ñ0°À2Ð0ÓFˆLð !-× 1Ñ 1°#·~±~ÀwÓ XÐØ,×1Ñ1°#¿¹Õ2FÈÓYˆä—]‘]×*Ñ*¨<¸4¿<¹<ÐRV×R_ÑR_Ð*Ó`ˆ
ä—i’i 
¨LÓ9ˆà×ÑÓ #¯©Õ"6¸ÇÁÐ!OÔOÜØ2°C¿¹È×R_ÑR_Ð3`Ð2að bØ×$Ñ$Ó&Ð'ð)óð ð
 "×&Ñ& s¯N©N¸GÇ]Á]ÓSˆØ!×+Ñ+¨A¨qÓ1ˆð "×)Ñ)¨#¸¿¹ÓGˆà—m‘m KÓ0ˆàÐ1Ð1Ð1r'   )rƒ   r�   r‹   r„   r�   r†   r‚   r’   r‘   r�   r�   )g        FTN)NNNr8   rA   s   @@r$   r   r   i   s&   ù‡ € ÙG÷Cõ C÷:l2÷ l2ò l2r'   r   c                   óX   a a€ ] tR t^øt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚXGLMDecoderLayerc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfigr   )r"   r#   s   "€r$   r%   ÚXGLMDecoderLayer.__annotate__ù   s   ø€ ÷ =ñ =™zñ =r'   c                óP  <€ \         SV `  4        VP                  V n        \	        V P                  VP
                  VP                  R VR7      V n        VP                  V n        \        VP                  ,          V n        VP                  V n        VP                  '       dY   \	        V P                  VP
                  VP                  R VR7      V n        \        P                   ! V P                  4      V n        \        P                   ! V P                  4      V n        \        P&                  ! V P                  VP(                  4      V n        \        P&                  ! VP(                  V P                  4      V n        \        P                   ! V P                  4      V n        R# )T)r�   r‚   rƒ   r„   r†   N)r*   r+   Úd_modelr�   r   Úattention_headsÚattention_dropoutÚ	self_attnrƒ   r   Úactivation_functionÚactivation_fnÚactivation_dropoutÚadd_cross_attentionÚencoder_attnr   Ú	LayerNormÚencoder_attn_layer_normÚself_attn_layer_normrŽ   Úffn_dimÚfc1Úfc2Úfinal_layer_norm)r,   rÈ   r†   r-   s   &&&€r$   r+   ÚXGLMDecoderLayer.__init__ù   s  ø€ Ü‰ÑÔØŸ™ˆŒä&Ø—n‘nØ×,Ñ,Ø×,Ñ,ØØô
ˆŒð —~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔà×%×%Ð%Ü -ØŸ.™.Ø ×0Ñ0Ø×0Ñ0ØØ#ô!ˆDÔô ,.¯<ª<¸¿¹Ó+GˆDÔ(ä$&§L¢L°·±Ó$@ˆÔ!Ü—9’9˜TŸ^™^¨V¯^©^Ó<ˆŒÜ—9’9˜VŸ^™^¨T¯^©^Ó<ˆŒÜ "§¢¨T¯^©^Ó <ˆÖr'   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[ P                  /# )
r   r•   r˜   NÚencoder_hidden_statesÚencoder_attention_maskr—   Ú	use_cacher™   rš   )r1   r2   r   r‡   r   r   )r"   r#   s   "€r$   r%   rÉ     sŒ   ø€ ÷ :ñ :á—|‘|ð:ñ Ÿ™ tÕ+ð:ñ  %Ÿ|™|¨dÕ2ð	:ñ
 !&§¡¨tÕ 3ð:ñ  �ð:ñ ˜$•;ð:ñ Ñ+Õ,ð:ñ 
�‰ñ:r'   c           	     ó"  € TpV P                  V4      pV P                  ! V3RVRV/VB w  r\        P                  P	                  WP                  V P
                  R7      pW�,           pVel   TpV P                  V4      pV P                  ! V3RVRVRV/VB w  r\        P                  P	                  WP                  V P
                  R7      pW�,           pTpV P                  V4      pV P                  V P                  V4      4      p\        P                  P	                  WP                  V P
                  R7      pV P                  V4      p\        P                  P	                  WP                  V P
                  R7      pW�,           pV# )a   
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`): attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
    encoder_hidden_states (`torch.FloatTensor`):
        cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
    encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
    past_key_values (`Cache`): cached past key and value projection states
r—   r˜   r�   r–   )rÖ   rÎ   r   r²   rƒ   rŸ   rÕ   rÓ   rÚ   rÐ   rØ   rÑ   rÙ   )
r,   r•   r˜   rÝ   rÞ   r—   rß   r™   Úresidualr·   s
   &&&&&&&,  r$   r4   ÚXGLMDecoderLayer.forward  s–  € ð* !ˆØ×1Ñ1°-Ó@ˆð  Ÿ>š>Øñ
à+ð
ð *ð
ð ñ	
Ñˆô Ÿ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆð !Ò,Ø$ˆHØ ×8Ñ8¸ÓGˆMà#×0Ò0Øñ à!6ð ð  6ð ð !0ð	 ð
 ñ ÑˆMô ŸM™M×1Ñ1°-Ç<Á<ÐZ^×ZgÑZgÐ1ÓhˆMØ$Õ4ˆMð !ˆØ×-Ñ-¨mÓ<ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆàÐr'   )rÑ   rÐ   rƒ   r�   rÓ   rÕ   rØ   rÙ   rÚ   rÎ   rÖ   r)   )NNNNT)	r9   r:   r;   r<   r+   r4   r>   r?   r@   rA   s   @@r$   rÅ   rÅ   ø   s   ù‡ € ÷=õ =÷>:÷ :ò :r'   rÅ   c                   óL   a a€ ] tR tRt oRtRtR.tV 3R ltV3R ltRt	Vt
V ;t# )ÚXGLMPreTrainedModeliU  ÚmodelTrÅ   c                ó  <€ \         SV `  V4       \        V\        4      '       df   VP	                  VP
                  VP                  ,           VP                  VP                  4      p\        P                  ! VP                  V4       R # R # r)   )r*   Ú_init_weightsr¡   rC   rT   rE   rI   r   r   ÚinitÚcopy_rO   )r,   ÚmodulerX   r-   s   && €r$   rç   Ú!XGLMPreTrainedModel._init_weights\  sj   ø€ Ü‰Ñ˜fÔ%Ü�fÔ?×@Ò@Ø ×.Ñ.Ø×$Ñ$ v§}¡}Õ4°f×6JÑ6JÈF×L^ÑL^óˆKô �JŠJ�v—~‘~ {Ö3ñ	 Ar'   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rÇ   r   )r"   r#   s   "€r$   r%   Ú XGLMPreTrainedModel.__annotate__U  s   ø‡ ‚ áÑò r'   r    )r9   r:   r;   r<   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesrç   Ú__annotate_func__r>   r?   r@   rA   s   @@r$   rä   rä   U  s(   ù‡ € ð  ÐØ&*Ð#Ø+Ð,Ðõ4÷ … r'   rä   c                   ó¦   a a€ ] tR tRt oR]R]! ]^RR7      R]! ]^RR7      /tV3R lV 3R	 llt]	]
]RV3R
 lR ll4       4       4       tRtVtV ;t# )Ú	XGLMModelie  r•   Ú
attentionsrÎ   )ÚindexÚ
layer_nameÚcross_attentionsrÓ   c                ó    <€ V ^8„  d   QhRS[ /# rÇ   r   )r"   r#   s   "€r$   r%   ÚXGLMModel.__annotate__m  s   ø€ ÷ ñ ™zñ r'   c           
     óì  <€ \         SV `  V4       VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  '       d!   \        P                  ! VP                  4      MR p\        VP                  VP                  V P
                  VR7      V n        \        VP                  VP                  VP                  4      V n        \"        P$                  ! \'        VP(                  4       Uu. uF  p\+        WR7      NK  	  up4      V n        \"        P.                  ! VP                  4      V n        RV n        V P5                  4        R# u upi )r7   r6   )r†   FN)r*   r+   rƒ   Ú	layerdropÚpad_token_idr   Úmax_position_embeddingsÚmax_target_positionsÚscale_embeddingr`   ÚsqrtrË   r   Ú
vocab_sizeÚembed_tokensrC   Úembed_positionsr   Ú
ModuleListÚrangeÚ
num_layersrÅ   r¦   rÔ   Ú
layer_normÚgradient_checkpointingÚ	post_init)r,   rÈ   r   Úir-   s   &&  €r$   r+   ÚXGLMModel.__init__m  s  ø€ Ü‰Ñ˜Ô Ø—~‘~ˆŒØ×)Ñ)ˆŒØ!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø39×3I×3IÐ3I”d—i’i §¡Ô/Èsˆä3Ø×Ñ˜vŸ~™~¨t×/?Ñ/?È[ô
ˆÔô  AØ×*Ñ*Ø�N‰NØ×Ñó 
ˆÔô
 —m’mÔTYÐZ`×ZkÑZkÔTlÓ$mÑTlÈqÔ%5°f×%JÑTlÑ$mÓnˆŒÜŸ,š, v§~¡~Ó6ˆŒà&+ˆÔ#à�‰Öùò %ns   ÄE1c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[R,          R
S[S[,          RS[S[ P                  ,          S[,          /
# )r   r0   Nr˜   rp   rÝ   rÞ   r—   Úinputs_embedsrß   r™   rš   )r1   r2   r   r‡   r   r   r›   r   )r"   r#   s   "€r$   r%   rù   ˆ  sÎ   ø€ ÷ ]
ñ ]
á—<‘< $Õ&ð]
ñ Ÿ™ tÕ+ð]
ñ —l‘l TÕ)ð	]
ñ
  %Ÿ|™|¨dÕ2ð]
ñ !&§¡¨tÕ 3ð]
ñ  �ð]
ñ —|‘| dÕ*ð]
ñ ˜$•;ð]
ñ Ñ+Õ,ð]
ñ 
‰u�|‰|Õ	ÑHÕ	Hñ]
r'   c	           
     ó¾  € VRJ VRJ,          '       d   \        R4      hVf   V P                  V4      pV'       do   Vfk   Vf   V P                  P                  '       d5   \	        \        V P                  R7      \        V P                  R7      4      M\        V P                  R7      pVe   VP                  4       M^ p
\        V P                  VVVR7      pVfl   \        P                  ! T
VP                  ^,          V
,           \        P                  Ve   VP                  MVP                  R7      pVP                  ^ 4      pVe   Ve   \        V P                  VVVR7      pWpP                  W:4      P!                  VP                  4      ,           p\"        P$                  P'                  V\)        V P&                  4      V P*                  R7      p\-        V P.                  4       FP  w  rÍV P*                  '       d)   \        P0                  ! . 4      pWàP2                  8  d   K?  V! VVV3RVR	VR
V/V	B pKR  	  V P5                  V4      p\7        VVR7      # )au  
encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
    Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of
    the decoder.
encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
    Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
    selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

    [What are attention masks?](../glossary#attention-mask)
Nz:You must specify exactly one of input_ids or inputs_embeds)rÈ   )rÈ   r  r˜   r—   rP   )rÈ   r  r˜   rÝ   r�   rÞ   r—   rß   )Úlast_hidden_stater—   )rŒ   r  rÈ   Úis_encoder_decoderr   r   Úget_seq_lengthr
   r1   rc   ru   ÚlongrR   re   r	   r  rV   r   r²   rƒ   r!   rŸ   Ú	enumerater¦   Úrandrû   r  r   )r,   r0   r˜   rp   rÝ   rÞ   r—   r  rß   r™   rq   r•   ÚidxÚdecoder_layerÚdropout_probabilitys   &&&&&&&&&,     r$   r4   ÚXGLMModel.forward…  s8  € ð8 ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆM÷ ˜Ò0ð )Ò4¸¿¹×8V×8VÐ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÒE` ×!?Ñ!?Ô!AÐfgÐä+Ø—;‘;Ø'Ø)Ø+ô	
ˆð ÒÜ Ÿ<š<Ø&Ø×#Ñ# AÕ&Ð)?Õ?Ü—j‘jØ+4Ò+@�y×'Ò'Àm×FZÑFZô	ˆLð (×1Ñ1°!Ó4ˆLð !Ò,Ð1GÒ1SÜ%>Ø—{‘{Ø+Ø5Ø&;ô	&Ð"ð &×(<Ñ(<¸\Ó(b×(eÑ(eØ× Ñ ó)
õ 
ˆô Ÿ™×-Ñ-¨m¼uÀTÇ\Á\Ó?RÐ]a×]jÑ]jÐ-Ókˆä"+¨D¯K©KÖ"8ÑˆCà�}�}ˆ}Ü&+§j¢j°£nÐ#Ø&¯©Ô7Ùá)ØØØ%ñð (>ð	ð
 !0ðð $ðð ñŠMñ #9ð" Ÿ™¨Ó6ˆä8Ø+Ø+ô
ð 	
r'   )	rƒ   r  r  r  r  rû   r¦   rþ   r   )NNNNNNNN)r9   r:   r;   r<   rÅ   r   r   Ú_can_record_outputsr+   r   r   r   r4   r>   r?   r@   rA   s   @@r$   ró   ró   e  sm   ù‡ € ð 	Ð)Ø‘n ]¸!ÈÔTØ™N¨=ÀÈnÔ]ðÐ÷ó ð0  ØØ÷]
ñ ]
ó ó ó  ÷]
ð ]
r'   ró   z‡
    The XGLM Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    )Úcustom_introc                   ór   a a€ ] tR tRt oRtRR/tV 3R lt]]]	R	V3R lR ll4       4       4       t
RtVtV ;t# )
ÚXGLMForCausalLMiè  rå   zlm_head.weightzmodel.embed_tokens.weightc                óÊ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        V P                  4        R# )FrŠ   N)
r*   r+   ró   rå   r   rŽ   Úhidden_sizer  Úlm_headr	  )r,   rÈ   r-   s   &&€r$   r+   ÚXGLMForCausalLM.__init__ò  sH   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ü—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör'   c                óÔ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[S[ P                  ,          RS[S[,          RS[S[ P                  ,          S[,          /# )r   r0   Nr˜   rp   rÝ   rÞ   r—   r  Úlabelsrß   Úlogits_to_keepr™   rš   )	r1   r2   r   r‡   r    r   r   r›   r   )r"   r#   s   "€r$   r%   ÚXGLMForCausalLM.__annotate__ý  sö   ø€ ÷ B
ñ B
á—<‘< $Õ&ðB
ñ Ÿ™ tÕ+ðB
ñ —l‘l TÕ)ð	B
ñ
  %Ÿ|™|¨dÕ2ðB
ñ !&§¡¨tÕ 3ðB
ñ  �ðB
ñ —|‘| dÕ*ðB
ñ —‘˜tÕ#ðB
ñ ˜$•;ðB
ñ ™eŸl™lÕ*ðB
ñ Ñ+Õ,ðB
ñ 
‰u�|‰|Õ	Ñ@Õ	@ñB
r'   c                óÜ  € V P                   ! RRVRVRVRVRVRVRVRV	/VB pVP                  p\        V
\        4      '       d   \	        V
) R	4      MT
pV P                  VR
VR
3,          4      pR	pVeA   V P                  ! VV3RV P                  P                  RV P                  P                  /VB p\        VVVP                  VP                  VP                  VP                  R7      # )aù  
encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
    Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of
    the decoder.
encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
    Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
    selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

    [What are attention masks?](../glossary#attention-mask)
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
r0   r˜   rp   rÝ   rÞ   r—   r  rß   Nr^   r  rü   )ÚlossÚlogitsr—   r•   rô   r÷   r    )rå   r  r¡   r    Úslicer  Úloss_functionrÈ   r  rü   r   r—   r•   rô   r÷   )r,   r0   r˜   rp   rÝ   rÞ   r—   r  r"  rß   r#  r™   Úoutputsr•   Úslice_indicesr'  r&  s   &&&&&&&&&&&,     r$   r4   ÚXGLMForCausalLM.forwardú  s;  € ðF >B¿ZºZñ 
>
Øð
>
à)ð
>
ð &ð
>
ð #8ð	
>
ð
 $:ð
>
ð ,ð
>
ð (ð
>
ð  ð
>
ð ñ
>
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%ØØñð  Ÿ;™;×1Ñ1ðð "Ÿ[™[×5Ñ5ð	ð
 ñˆDô 1ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r'   )r  rå   )
NNNNNNNNNr{   )r9   r:   r;   r<   rî   Ú_tied_weights_keysr+   r   r   r   r4   r>   r?   r@   rA   s   @@r$   r  r  è  sP   ù‡ € ð  ÐØ*Ð,GÐHÐõð  ØØ÷B
ñ B
ó ó ó  ÷B
ð B
r'   r  )r  ró   rä   )3r=   r`   r1   r   Ú r   rè   Úactivationsr   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr	   r
   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úconfiguration_xglmr   Ú
get_loggerr9   ÚloggerÚ	Embeddingr   ÚModulerC   r   rÅ   rä   ró   r  Ú__all__r    r'   r$   Ú<module>r@     s  ðñ ã ã Ý å &Ý !ß CÑ CÝ )ß JÝ 9ß lÝ -Ý &ß @Ñ @Ý 7ß EÝ *ð 
×	Ò	˜HÓ	%€ô
=˜bŸl™lô 
=ô1w¨¯	©	ô 1wôhL2�B—I‘Iô L2ô^ZÐ1ô Zðz ô4˜/ó 4ó ð4ð ô
Ð#ó 
ó ð
ñD ðôôQ
Ð)¨?ó Q
óðQ
òh B‚r'   