+
    œ°³jË|  ã                   ó@  € R t ^ RIt^ RIt^ RIHt ^ RIHtHtHtHt ^ RIH	t
 ^RIHtHt ^RIHt ^RIHt ^RIHt ^R	IHtHtHtHtHt ^R
IHt ^RIHtHt ^RIH t  ]PB                  ! ]"4      t#R#R lt$ ! R R]PJ                  4      t& ! R R]PJ                  4      t' ! R R]4      t(] ! R R]4      4       t)] ! R R])4      4       t*]! RR7       ! R R])]4      4       t+]! RR7       ! R R])4      4       t,] ! R R ])4      4       t-] ! R! R"])4      4       t.. R$Ot/R# )%zPyTorch MPT model.N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚ	LayerNormÚMSELoss)Ú
functional)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚlogging©Ú	MptConfigc                óˆ  € \         P                  ! ^V,
          ^\         P                  VR7      P                  ^^^V4      p^\        P
                  ! \        P                  ! V 4      4      ,          p\         P                  ! ^V^,           \         P                  VR7      P                  4       pWbV,          ,          pR\         P                  ! ^V4      ,          pVP                  ^V^^4      pWP8w  d5   \         P                  ! VR,          VR,          .^R7      RRV 1R3,          pWG,          pVP                  ^ 4      # )	a–  
Link to paper: https://huggingface.co/papers/2108.12409 - Alibi tensor is not causal as the original paper mentions, it
relies on a translation invariance of softmax for quick implementation. This implementation has been copied from
the alibi implementation of MPT source code that led to slightly different results than the Bloom alibi:
https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L292
)ÚdtypeÚdeviceg      ð?ºNNN.©ÚdimN)r   :é   Né   .)r   :NNr   .)ÚtorchÚarangeÚint32ÚviewÚmathÚceilÚlog2Úint64ÚfloatÚpowÚconcatÚsqueeze)Ú	num_headsÚsequence_lengthÚalibi_bias_maxr   ÚalibiÚnum_heads_power_of_2ÚbaseÚslopess   &&&&    Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mpt/modeling_mpt.pyÚbuild_mpt_alibi_tensorr3   *   s  € ô �LŠL˜˜_Õ,¨a´u·{±{È6ÔR×WÑWÐXYÐ[\Ð^_ÐapÓq€EØ¤§	¢	¬$¯)ª)°IÓ*>Ó ?Õ?Ðä�<Š<˜Ð/°!Õ3¼5¿;¹;ÈvÔV×\Ñ\Ó^€DØÐ$8Õ8Õ9€Dà”5—9’9˜Q Ó%Õ%€FØ�[‰[˜Ð0°!°QÓ7€FàÔ(Ü—’˜v lÕ3°V¸KÕ5HÐIÈqÔQÐRSÐU_ÐV_ÐU_ÐadÐRdÕeˆà�N€EØ�=‰=˜ÓÐó    c                   ó\   a a€ ] tR t^At oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚMptAttentionzrMulti-head self attention.
Using torch or triton attention implementation enables user to also use additive bias.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r   ÚconfigÚ	layer_idxN©r   Úint)ÚformatÚ__classdict__s   "€r2   Ú__annotate__ÚMptAttention.__annotate__F   s   ø€ ÷ #ñ #™yð #±S¸4µZñ #r4   c                óè  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        V P                  V P                  ,          V n        VP                  P                  V n        V P                  f>   ^\        P                  ! V P                  V P                  ,          4      ,          V n        VP                  P                  V n        VP                  P                  V n        \        P                  ! V P                  ^V P                  ,          RR7      V n        \        P                  ! V P                  V P                  RR7      V n        W n        R # )NF©Úbias)ÚsuperÚ__init__Úhidden_sizeÚn_headsÚmax_seq_lenÚmax_seq_lengthÚhead_dimÚattn_configÚsoftmax_scaler#   ÚsqrtÚ
attn_pdropÚattn_dropout_pÚclip_qkvr   ÚLinearÚWqkvÚout_projr:   )Úselfr9   r:   Ú	__class__s   &&&€r2   rE   ÚMptAttention.__init__F   s  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ—~‘~ˆŒØ$×0Ñ0ˆÔØ×(Ñ(¨D¯L©LÕ8ˆŒØ#×/Ñ/×=Ñ=ˆÔØ×ÑÒ%Ø!"¤T§Y¢Y¨t×/?Ñ/?À$Ç,Á,Õ/NÓ%OÕ!OˆDÔà$×0Ñ0×;Ñ;ˆÔØ×*Ñ*×3Ñ3ˆŒÜ—I’I˜d×.Ñ.°°D×4DÑ4DÕ0DÈ5ÔQˆŒ	ÜŸ	š	 $×"2Ñ"2°D×4DÑ4DÈ5ÔQˆŒØ"Žr4   c          	      óŠ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[R,          RS[ P                  R,          /# )r   Úhidden_statesÚposition_biasÚpast_key_valuesNÚattention_mask)r   ÚTensorr   )r=   r>   s   "€r2   r?   r@   V   sH   ø€ ÷ 0)ñ 0)á—|‘|ð0)ñ —|‘|ð0)ñ  �ð	0)ñ
 Ÿ™ tÕ+ñ0)r4   c                ó²  € VP                   R ,          w  rgV P                  V4      pV P                  '       d)   VP                  V P                  ) V P                  R7      pVP	                  ^^R7      w  ršpV	P                  WgV P                  V P                  4      P                  ^^4      p	V
P                  WgV P                  V P                  4      P                  ^^4      p
VP                  WgV P                  V P                  4      P                  ^^4      pVe   VP                  W«V P                  4      w  r«\        P                  ! WšP                  RR4      4      V P                  ,          pVf   TMWsP                  4       ,           pVe«   \        VP                   4      ^8w  d"   \!        R\        VP                   4       24      hV
P                   R,          p\#        ^ VP%                  ^4      V,
          4      p\#        ^ VP%                  ^4      V,
          4      pVRVR1VR13,          pWÂ,           pVe;   VP'                  V\        P(                  ! V	P*                  4      P,                  4      p\.        P0                  P3                  VP5                  4       RR7      P7                  VP*                  4      p\.        P0                  P9                  VV P:                  V P<                  R7      p\        P                  ! VV4      pVP?                  ^ ^^^4      PA                  4       PC                  WgR4      pV PE                  V4      pVV3# )	:Nr   N)ÚminÚmaxr   Nz6Expecting position_bias shape to be 3 dimensions, got r   ©ÚpÚtrainingéÿÿÿÿéþÿÿÿ)#ÚshaperR   rP   ÚclampÚchunkÚreshaperG   rJ   Ú	transposeÚupdater:   r   ÚmatmulrL   Úget_seq_lengthÚlenÚ
ValueErrorr_   ÚsizeÚmasked_fillÚfinfor   r^   r   r   Úsoftmaxr'   ÚtoÚdropoutrO   rb   ÚpermuteÚ
contiguousr"   rS   )rT   rX   rY   rZ   r[   ÚkwargsÚ
batch_sizeÚ
seq_lengthÚ	mixed_qkvÚquery_statesÚ
key_statesÚvalue_statesÚattention_scoresÚquery_lengthÚ
key_lengthÚposition_bias_query_indexÚposition_bias_key_indexÚattn_weightsÚcontext_statesÚattn_outputs   &&&&&,              r2   ÚforwardÚMptAttention.forwardV   sÔ  € ð "/×!4Ñ!4°RÕ!8Ñˆ
à—I‘I˜mÓ,ˆ	Ø�=�=ˆ=Ø!Ÿ™¨T¯]©]¨NÀÇÁ˜ÓNˆIà1:·±ÀÈ°Ó1JÑ.ˆ ,Ø#×+Ñ+¨JÀDÇLÁLÐRV×R_ÑR_Ó`×jÑjÐklÐnoÓpˆØ×'Ñ'¨
ÀÇÁÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆ
Ø#×+Ñ+¨JÀDÇLÁLÐRV×R_ÑR_Ó`×jÑjÐklÐnoÓpˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä Ÿ<š<¨×6JÑ6JÈ2ÈrÓ6RÓSÐVZ×VhÑVhÕhÐØ%4Ò%<‘zÀ*×OmÑOmÓOoÕBoˆàÒ$Ü�=×&Ñ&Ó'¨1Ô,Ü Ð#YÔZ]Ð^k×^qÑ^qÓZrÐYsÐ!tÓuÐuØ#×)Ñ)¨"Õ-ˆJä(+¨A¨}×/AÑ/AÀ!Ó/DÀ|Õ/SÓ(TÐ%Ü&)¨!¨]×-?Ñ-?ÀÓ-BÀZÕ-OÓ&PÐ#à)¨!Ð-FÑ-GÐI`ÑIaÐ*aÕbˆMà/Õ?ÐàÒ%Ø/×;Ñ;¸NÌEÏKÊKÐXd×XjÑXjÓLk×LoÑLoÓpÐô —}‘}×,Ñ,Ð-=×-CÑ-CÓ-EÈ2Ð,ÓN×QÑQÐR^×RdÑRdÓeˆÜ—}‘}×,Ñ,¨\¸T×=PÑ=PÐ[_×[hÑ[hÐ,ÓiˆäŸš l°LÓAˆØ'×/Ñ/°°1°a¸Ó;×FÑFÓH×MÑMÈjÐfhÓiˆØ—m‘m NÓ3ˆà˜LÐ(Ð(r4   )
rR   rO   rP   rJ   rF   r:   rI   rG   rS   rL   ©N)NN)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rE   r†   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rU   r>   s   @@r2   r6   r6   A   s#   ù‡ € ñ÷#õ #÷ 0)÷ 0)ò 0)r4   r6   c                   óP   a a€ ] tR t^‰t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMptMLPc                ó    <€ V ^8„  d   QhRS[ /# ©r   r9   r   )r=   r>   s   "€r2   r?   ÚMptMLP.__annotate__Š   s   ø€ ÷ <ñ <™yñ <r4   c                ó>  <€ \         SV `  4        VP                  p\        P                  ! V^V,          RR7      V n        \        P                  ! RR7      V n        \        P                  ! ^V,          VRR7      V n        VP                  P                  V n        R# )é   FrB   Únone)ÚapproximateN)rD   rE   rF   r   rQ   Úup_projÚGELUÚactÚ	down_projrK   rN   Úhidden_dropout)rT   r9   rF   rU   s   && €r2   rE   ÚMptMLP.__init__Š   sm   ø€ Ü‰ÑÔØ×(Ñ(ˆä—y’y ¨a°+­oÀEÔJˆŒÜ—7’7 vÔ.ˆŒÜŸš 1 {¥?°KÀeÔLˆŒØ$×0Ñ0×;Ñ;ˆÖr4   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   rX   ÚresidualÚreturn©r   r\   )r=   r>   s   "€r2   r?   r–   “   s.   ø€ ÷ ñ ¡U§\¡\ð ¹U¿\¹\ð ÉeÏlÉlñ r4   c                óÐ   € V P                  V P                  V4      4      pV P                  V4      p\        P                  ! W0P
                  V P                  R 7      pWB,           pV# )r`   )r�   r›   rž   ÚFrt   rŸ   rb   )rT   rX   r¢   Úintermediate_outputÚoutputs   &&&  r2   r†   ÚMptMLP.forward“   sS   € ØŸ™ §¡¨mÓ!<Ó=ˆà"Ÿn™n¨]Ó;Ðä—’Ð.×2EÑ2EÐPT×P]ÑP]Ô^ˆØÕ"ˆàˆr4   )r�   rž   rŸ   r›   ©	r‰   rŠ   r‹   rŒ   rE   r†   rŽ   r�   r�   r‘   s   @@r2   r“   r“   ‰   s   ù‡ € ÷<ó <÷÷ ð r4   r“   c                   óX   a a€ ] tR t^žt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚMptBlockc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r8   r;   )r=   r>   s   "€r2   r?   ÚMptBlock.__annotate__Ÿ   s#   ø€ ÷ @ñ @™yð @±S¸4µZñ @r4   c                óÎ  <€ \         SV `  4        VP                  p\        W1P                  R 7      V n        RV P
                  n        VP                  V n        \        W4      V n
        \        W1P                  R 7      V n        RV P                  n        \        V4      V n        VP                  P                  V n        \"        P$                  ! V P                   4      V n        R# )©ÚepsN)rD   rE   rF   r   Úlayer_norm_epsilonÚnorm_1rC   rG   r+   r6   ÚattnÚnorm_2r“   ÚffnrK   rN   Údropout_rater   ÚDropoutÚresid_attn_dropout)rT   r9   r:   rF   rU   s   &&& €r2   rE   ÚMptBlock.__init__Ÿ   s¢   ø€ Ü‰ÑÔØ×(Ñ(ˆä ×1JÑ1JÔKˆŒàˆ�‰ÔàŸ™ˆŒÜ  Ó3ˆŒ	ä ×1JÑ1JÔKˆŒàˆ�‰Ôä˜&“>ˆŒà"×.Ñ.×9Ñ9ˆÔÜ"$§*¢*¨T×->Ñ->Ó"?ˆÖr4   c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[R,          RS[RS[/# )r   rX   rY   r[   Ú
layer_pastNÚ	use_cacheÚoutput_attentions)r   r\   r   Úbool)r=   r>   s   "€r2   r?   r®   ³   sW   ø€ ÷ !$ñ !$á—|‘|ð!$ñ —|‘|ð!$ñ Ÿ™ð	!$ñ
 ˜D•Lð!$ñ ð!$ñ  ñ!$r4   c                óÔ   € V P                  V4      pTp	V P                  VVVVR 7      w  r«V P                  V
4      V	,           pV P                  V4      pTp	V P	                  W‰4      pWË3# ))rY   r[   rZ   )r³   r´   r¹   rµ   r¶   )rT   rX   rY   r[   r¼   r½   r¾   rw   Úlayernorm_outputr¢   Úattn_outputsrƒ   r¨   s   &&&&&&&,     r2   r†   ÚMptBlock.forward³   sƒ   € ð  Ÿ;™; }Ó5Ðà ˆð &*§Y¡YØØ'Ø)Ø&ð	 &/ó &
Ñ"ˆð ×/Ñ/°Ó=ÀÕHˆàŸ;™; }Ó5Ðð !ˆð —‘Ð*Ó5ˆØÐ#Ð#r4   )r´   r·   r¶   r³   rµ   r+   r¹   rˆ   )NFFrª   r‘   s   @@r2   r¬   r¬   ž   s    ù‡ € ÷@õ @÷(!$÷ !$ò !$r4   r¬   c                   ó:   a € ] tR t^×t o RtRtR.tV 3R ltRtV t	R# )ÚMptPreTrainedModelÚtransformerTr¬   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r•   r   )r=   r>   s   "€r2   r?   ÚMptPreTrainedModel.__annotate__×   s   ø‡ ‚ áÑò r4   © N)
r‰   rŠ   r‹   rŒ   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ__annotate_func__rŽ   r�   )r>   s   @r2   rÅ   rÅ   ×   s"   ø‡ € ð &ÐØ&*Ð#Ø#˜Ð÷ ƒ r4   rÅ   c                   ó€   a a€ ] tR t^ßt oV3R lV 3R lltR tR
R ltV3R lR lt]RV3R lR ll4       t	R	t
VtV ;t# )ÚMptModelc                ó    <€ V ^8„  d   QhRS[ /# r•   r   )r=   r>   s   "€r2   r?   ÚMptModel.__annotate__á   s   ø€ ÷ ñ ™yñ r4   c           
     ó   <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        \        V P                  VP                  R7      V n        RV P                   n        RV n        V P'                  4        R# u upi ))r:   r°   NF)rD   rE   rF   rG   r+   r   Ú	EmbeddingÚ
vocab_sizeÚwteÚ
ModuleListÚrangeÚn_layersr¬   Úblocksr   r²   Únorm_frC   Úgradient_checkpointingÚ	post_init)rT   r9   ÚirU   s   && €r2   rE   ÚMptModel.__init__á   s¿   ø€ Ü‰Ñ˜Ô à!×-Ñ-ˆÔØŸ™ˆŒô —<’< × 1Ñ 1°4×3CÑ3CÓDˆŒô —m’mÌEÐRX×RaÑRaÔLbÓ$cÑLbÀq¤X¨f×%BÑLbÑ$cÓdˆŒô   × 0Ñ 0°f×6OÑ6OÔPˆŒàˆ�‰Ôà&+ˆÔ#ð 	�‰Öùò %ds   Â
C;c                ó   € V P                   # rˆ   ©rÕ   )rT   s   &r2   Úget_input_embeddingsÚMptModel.get_input_embeddings÷   s   € Ø�x‰xˆr4   c                ó   € \        WW44      # rˆ   )r3   )rT   r+   r,   r-   r   s   &&&&&r2   r3   ÚMptModel.build_mpt_alibi_tensorú   s   € Ü% iÀ.ÓYÐYr4   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   Únew_embeddingsr¤   )r=   r>   s   "€r2   r?   rÑ   ý   s   ø€ ÷ "ñ "±5·<±<ñ "r4   c                ó   € Wn         R # rˆ   rà   ©rT   rç   s   &&r2   Úset_input_embeddingsÚMptModel.set_input_embeddingsý   s   € Ø!Žr4   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          R	S[R,          R
S[S[ P                  R3,          S[,          /	# )r   Ú	input_idsNrZ   r[   Úinputs_embedsr½   r¾   Úoutput_hidden_statesÚreturn_dictr£   .)r   Ú
LongTensorr   r\   r¿   Útupler   )r=   r>   s   "€r2   r?   rÑ     s·   ø€ ÷ f
ñ f
á×#Ñ# dÕ*ðf
ñ  �ðf
ñ Ÿ™ tÕ+ð	f
ñ
 ×'Ñ'¨$Õ.ðf
ñ ˜$•;ðf
ñ   $�;ðf
ñ # T�kðf
ñ ˜D•[ðf
ñ 
‰u�|‰|˜SÐ Õ	!Ñ$MÕ	Mñf
r4   c	           
     óö  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   Ve   \        R4      hVe   VP                  w  r«MVe   VP                  w  r«pM\        R4      hV P                  '       d2   V P                  '       d    V'       d   \        P                  R4       RpVf   V P                  V4      pV'       d   Vf   \        V P                   R7      pTpV'       d   RMRpV'       d   RMRpV P                  V P                  V P                   P                  VP                   R7      p\#        V P                   VVVR7      P%                  \&        P(                  4      pV P*                   FF  pV'       d
   Wý3,           pV! VVVVVVR	7      pV^ ,          pV'       g   K5  VV^,          3,           pKH  	  V P-                  V4      pV'       d
   Wý3,           pV'       g1   \.        ;QJ d    . R
 WÒWþ3 4       F  NK  	  5# ! R
 WÒWþ3 4       4      # \1        VVVVR7      # )áj  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r9   ©r   )r9   rî   r[   rZ   )r¼   r[   r½   r¾   rY   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irˆ   rÉ   )Ú.0Úvs   & r2   Ú	<genexpr>Ú#MptModel.forward.<locals>.<genexpr>^  s   é € ð Ùc�a—’Ócùs   ‚Œ
)Úlast_hidden_staterZ   rX   Ú
attentionsrÉ   )r9   r¾   rï   r½   rð   rn   re   rÛ   rb   ÚloggerÚwarning_oncerÕ   r	   r3   r+   rH   r   r   rs   r   r¿   rÙ   rÚ   rò   r   )rT   rí   rZ   r[   rî   r½   r¾   rï   rð   rw   rx   ry   Ú_rX   Úall_self_attentionsÚall_hidden_statesr.   Úcausal_maskÚblockÚoutputss   &&&&&&&&&,          r2   r†   ÚMptModel.forward   sY  € ð4 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ  ]Ò%>ÜÐcÓdÐdØÒ"Ø%.§_¡_Ñ"ˆJ˜
ØÒ&Ø(5×(;Ñ(;Ñ%ˆJ¡AäÐTÓUÐUà×&×&Ð&¨4¯=¯=¨=ßÜ×#Ñ#Øpôð "�	àÒ Ø ŸH™H YÓ/ˆMç˜Ò0Ü*°$·+±+Ô>ˆOà%ˆç$5™b¸4Ðß"6™B¸DÐð ×+Ñ+¨D¯N©N¸D¿K¹K×<SÑ<SÐ\i×\pÑ\pÐ+Óqˆä(Ø—;‘;Ø'Ø)Ø+ô	
÷
 ‰"ŒU�Z‰Z‹.ð 	ð —[”[ˆEß#Ø$5Ð8HÕ$HÐ!áØØ*Ø*Ø#Ø"3Ø#ôˆGð $ A�JˆMß Ñ Ø&9¸WÀQ½Z¸MÕ&IÒ#ñ !ð$ Ÿ™ MÓ2ˆçØ 1Ð4DÕ DÐçß”5ñ Ø)Ð<MÑcó—5ð �5ñ Ø)Ð<MÑcóó ð ô 9Ø+Ø+Ø+Ø*ô	
ð 	
r4   )rÙ   rÛ   rF   rÚ   r+   rÕ   ©é   N©NNNNNNNN)r‰   rŠ   r‹   rŒ   rE   rá   r3   rê   r   r†   rŽ   r�   r�   r‘   s   @@r2   rÏ   rÏ   ß   sF   ù‡ € ÷ó ò,ôZ÷"ð "ð ÷f
ñ f
ó ÷f
ð f
r4   rÏ   z†
    The MPT Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    )Úcustom_introc                   óx   a a€ ] tR tRt oRR/tV3R lV 3R lltV3R lR lt]RV3R lR	 ll4       tR
t	Vt
V ;t# )ÚMptForCausalLMij  zlm_head.weightztransformer.wte.weightc                ó    <€ V ^8„  d   QhRS[ /# r•   r   )r=   r>   s   "€r2   r?   ÚMptForCausalLM.__annotate__s  s   ø€ ÷ ñ ™yñ r4   c                óÊ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        V P                  4        R# ©FrB   N)
rD   rE   rÏ   rÆ   r   rQ   rF   rÔ   Úlm_headrÜ   ©rT   r9   rU   s   &&€r2   rE   ÚMptForCausalLM.__init__s  sI   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆÔÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör4   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ræ   r¤   )r=   r>   s   "€r2   r?   r  {  s   ø€ ÷ &ñ &±E·L±Lñ &r4   c                ó   € Wn         R # rˆ   )r  ré   s   &&r2   Úset_output_embeddingsÚ$MptForCausalLM.set_output_embeddings{  s   € Ø%Žr4   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P                  ,          RS[S[ P                  ,          S[,          /# )r   rí   NrZ   r[   rî   Úlabelsr½   r¾   rï   rð   Úlogits_to_keepr£   )r   rñ   r   r\   r¿   r<   rò   r   )r=   r>   s   "€r2   r?   r    sË   ø€ ÷ ?
ñ ?
á×#Ñ# dÕ*ð?
ñ  �ð?
ñ Ÿ™ tÕ+ð	?
ñ
 —|‘| dÕ*ð?
ñ —‘˜tÕ#ð?
ñ ˜$•;ð?
ñ   $�;ð?
ñ # T�kð?
ñ ˜D•[ð?
ñ ™eŸl™lÕ*ð?
ñ 
‰u�|‰|Õ	Ñ@Õ	@ñ?
r4   c                ó  € V	e   T	MV P                   P                  p	V P                  VVVVVVVV	R7      pV^ ,          p\        V
\        4      '       d   \        V
) R4      MT
pV P                  VRVR3,          4      pRpVe-   V P                  ! R	RVRVRV P                   P                  /VB pV	'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  VP                  R7      # )
aô  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
    `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
    are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
N©rZ   r[   rî   r½   r¾   rï   rð   r   Úlogitsr  rÔ   ºr   NN©Úlossr  rZ   rX   rü   rÉ   )r9   rð   rÆ   Ú
isinstancer<   Úslicer  Úloss_functionrÔ   r   rZ   rX   rü   )rT   rí   rZ   r[   rî   r  r½   r¾   rï   rð   r  rw   Útransformer_outputsrX   Úslice_indicesr  r  r¨   s   &&&&&&&&&&&,      r2   r†   ÚMptForCausalLM.forward~  s  € ð@ &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÕ.ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDçØ�YÐ!4°RÕ!8Õ8ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä0ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r4   )r  rÆ   )
NNNNNNNNNé    )r‰   rŠ   r‹   rŒ   Ú_tied_weights_keysrE   r  r   r†   rŽ   r�   r�   r‘   s   @@r2   r  r  j  sE   ù‡ € ð +Ð,DÐEÐ÷ó ÷&ð &ð ÷?
ñ ?
ó ÷?
ð ?
r4   r  aÒ  
    The MPT Model transformer with a sequence classification head on top (linear layer).

    [`MptForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-1) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   óp   a a€ ] tR tRt oV3R lV 3R lltV3R lR lt]R	V3R lR ll4       tRtVt	V ;t
# )
ÚMptForSequenceClassificationiÁ  c                ó    <€ V ^8„  d   QhRS[ /# r•   r   )r=   r>   s   "€r2   r?   Ú)MptForSequenceClassification.__annotate__Ð  s   ø€ ÷ ñ ™yñ r4   c                óì   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# r  )
rD   rE   Ú
num_labelsrÏ   rÆ   r   rQ   rF   ÚscorerÜ   r  s   &&€r2   rE   Ú%MptForSequenceClassification.__init__Ð  sV   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ# FÓ+ˆÔÜ—Y’Y˜v×1Ñ1°6×3DÑ3DÈ5ÔQˆŒ
ð 	�‰Ör4   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ræ   r¤   )r=   r>   s   "€r2   r?   r+  Ù  s   ø€ ÷ $ñ $±E·L±Lñ $r4   c                ó   € Wn         R # rˆ   )r.  ré   s   &&r2   r  Ú2MptForSequenceClassification.set_output_embeddingsÙ  s   € Ø#Ž
r4   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P                  ,          S[,          /
# ©r   rí   NrZ   r[   rî   r  r½   r¾   rï   rð   r£   )r   rñ   r   r\   r¿   rò   r   )r=   r>   s   "€r2   r?   r+  Ý  sÄ   ø€ ÷ e
ñ e
á×#Ñ# dÕ*ðe
ñ  �ðe
ñ Ÿ™ tÕ+ð	e
ñ
 —|‘| dÕ*ðe
ñ —‘˜tÕ#ðe
ñ ˜$•;ðe
ñ   $�;ðe
ñ # T�kðe
ñ ˜D•[ðe
ñ 
‰u�|‰|Õ	Ñ?Õ	?ñe
r4   c
                ó
  € V	e   T	MV P                   P                  p	V P                  VVVVVVVV	R7      pV^ ,          pV P                  V4      pVe   VP                  ^ ,          pMVP                  ^ ,          pV P                   P
                  f   V^8w  d   \        R4      hV P                   P
                  f   RpMÍVeœ   WP                   P
                  8g  P                  VP                  \        P                  4      p\        P                  ! VP                  R,          VP                  \        P                  R7      pVV,          P                  R4      pM.Rp\        P                  V P                  P                    R24       V\        P                  ! WíP                  R7      V3,          pRpVEeq   V P                   P"                  f–   V P$                  ^8X  d   RV P                   n        MrV P$                  ^8”  dQ   VP&                  \        P(                  8X  g    VP&                  \        P*                  8X  d   RV P                   n        MR	V P                   n        V P                   P"                  R8X  dL   \-        4       pV P$                  ^8X  d'   V! VP/                  4       VP/                  4       4      pMgV! VV4      pM]V P                   P"                  R8X  d   \1        4       pV! VV4      pM.V P                   P"                  R	8X  d   \3        4       pV! VV4      pV	'       g"   V3VR
,          ,           pVe   V3V,           # T# \5        VVVP6                  VP8                  VP:                  R7      # )áÎ  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nr  z=Cannot handle batch sizes > 1 if no padding token is defined.)r   r   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rõ   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr  r  rc   )r9   rð   rÆ   r.  re   Úpad_token_idrn   rs   r   r   r!   r    Úargmaxrý   rþ   rU   r‰   Úproblem_typer-  r   Úlongr<   r   r*   r   r   r   rZ   rX   rü   )rT   rí   rZ   r[   rî   r  r½   r¾   rï   rð   rw   r#  rX   r  rx   Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsr  Úloss_fctr¨   s   &&&&&&&&&&,           r2   r†   Ú$MptForSequenceClassification.forwardÜ  sç  € ð> &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÕ.ˆØ—‘˜MÓ*ˆàÒ Ø"Ÿ™¨Õ+‰Jà&×,Ñ,¨QÕ/ˆJà�;‰;×#Ñ#Ò+°
¸a´ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ò+Ø!#ÑØÒ"à%¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸLšL¨¯©¸Õ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Õ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¿}¹}ÔMÐOaÐaÕbˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù ¨vÓ6‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù ¨vÓ6�ßØ#Ð%Ð(;¸BÕ(?Õ?ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r4   )r-  r.  rÆ   ©	NNNNNNNNN)r‰   rŠ   r‹   rŒ   rE   r  r   r†   rŽ   r�   r�   r‘   s   @@r2   r)  r)  Á  s;   ù‡ € ÷ó ÷$ð $ð ÷e
ñ e
ó ÷e
ð e
r4   r)  c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚMptForTokenClassificationiE  c                ó    <€ V ^8„  d   QhRS[ /# r•   r   )r=   r>   s   "€r2   r?   Ú&MptForTokenClassification.__annotate__G  s   ø€ ÷ ñ ™yñ r4   c                óÖ  <€ \         SV `  V4       VP                  V n        \        V4      V n        \        VR 4      '       d   VP                  e   VP                  pM/\        VR4      '       d   VP                  e   VP                  pMRp\        P                  ! V4      V n
        \        P                  ! VP                  VP                  4      V n        V P                  4        R# )Úclassifier_dropoutNrŸ   gš™™™™™¹?)rD   rE   r-  rÏ   rÆ   ÚhasattrrJ  rŸ   r   r¸   rt   rQ   rF   Ú
classifierrÜ   )rT   r9   rJ  rU   s   && €r2   rE   Ú"MptForTokenClassification.__init__G  sµ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä# FÓ+ˆÔÜ�6Ð/×0Ò0°V×5NÑ5NÒ5ZØ!'×!:Ñ!:ÑÜ�VÐ-×.Ò.°6×3HÑ3HÒ3TØ!'×!6Ñ!6Ñà!$ÐÜ—z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör4   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P                  ,          S[,          /
# r4  )r   rñ   r   r\   r¿   rò   r   )r=   r>   s   "€r2   r?   rH  Y  sÄ   ø€ ÷ B
ñ B
á×#Ñ# dÕ*ðB
ñ  �ðB
ñ Ÿ™ tÕ+ð	B
ñ
 —|‘| dÕ*ðB
ñ —‘˜tÕ#ðB
ñ ˜$•;ðB
ñ   $�;ðB
ñ # T�kðB
ñ ˜D•[ðB
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñB
r4   c
                óN  € V	e   T	MV P                   P                  p	V P                  VVVVVVVV	R7      pV^ ,          pV P                  V4      pV P	                  V4      pRpVeu   VP                  VP                  4      pVP                  w  pp\        4       pV! VP                  VV,          V P                  4      VP                  VV,          4      4      pV	'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  R7      # )r6  Nr  ºr   NN)r  r  rX   rü   )r9   rð   rÆ   rt   rL  rs   r   re   r   r"   r-  r   rX   rü   )rT   rí   rZ   r[   rî   r  r½   r¾   rï   rð   Údeprecated_argumentsr#  rX   r  r  rx   ry   rB  r¨   s   &&&&&&&&&&,        r2   r†   Ú!MptForTokenClassification.forwardX  s)  € ð> &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÕ.ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÒà—Y‘Y˜vŸ}™}Ó-ˆFØ%+§\¡\Ñ"ˆJ˜
Ü'Ó)ˆHÙØ—‘˜J¨Õ3°T·_±_ÓEÀvÇ{Á{ÐS]Ð`jÕSjÓGkóˆD÷ Ø�YÐ!4°RÕ!8Õ8ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä$ØØØ-×;Ñ;Ø*×5Ñ5ô	
ð 	
r4   )rL  rt   r-  rÆ   rD  ©
r‰   rŠ   r‹   rŒ   rE   r   r†   rŽ   r�   r�   r‘   s   @@r2   rF  rF  E  s1   ù‡ € ÷ó ð" ÷B
ñ B
ó ÷B
ð B
r4   rF  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMptForQuestionAnsweringiž  c                ó²   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  ^4      V n        V P                  4        R# )r   N)	rD   rE   rÏ   rÆ   r   rQ   rF   Ú
qa_outputsrÜ   r  s   &&€r2   rE   Ú MptForQuestionAnswering.__init__   sA   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆÔÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör4   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# )r   rí   Nr[   rî   Ústart_positionsÚend_positionsr¾   rï   rð   r£   )r   rñ   ÚFloatTensorr¿   rò   r   )r=   r>   s   "€r2   r?   Ú$MptForQuestionAnswering.__annotate__©  s¹   ø€ ÷ F
ñ F
á×#Ñ# dÕ*ðF
ñ ×)Ñ)¨DÕ0ðF
ñ ×(Ñ(¨4Õ/ð	F
ñ
 ×)Ñ)¨DÕ0ðF
ñ ×'Ñ'¨$Õ.ðF
ñ   $�;ðF
ñ # T�kðF
ñ ˜D•[ðF
ñ 
Ñ-Õ	-ñF
r4   c	           	     ón  € Ve   TMV P                   P                  pV P                  VVVVVVR7      p
V
^ ,          pV P                  V4      pVP	                  ^RR7      w  rÞVP                  R4      P                  4       pVP                  R4      P                  4       pRpVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! WÔ4      pV! Wå4      pVV,           ^,          pV'       g"   WÞ3V
R,          ,           pVe   V3V,           # T# \        VVVV
P                  V
P                  R7      # )rô   N)r[   rî   r¾   rï   rð   r   )Úignore_indexrP  )r  Ústart_logitsÚ
end_logitsrX   rü   rc   )r9   rð   rÆ   rW  Úsplitr*   rv   rm   ro   rf   r   r   rX   rü   )rT   rí   r[   rî   rZ  r[  r¾   rï   rð   rw   r  Úsequence_outputr  r`  ra  Ú
total_lossÚignored_indexrB  Ú
start_lossÚend_lossr¨   s   &&&&&&&&&,           r2   r†   ÚMptForQuestionAnswering.forward¨  s±  € ð4 &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà×"Ñ"ØØ)Ø'Ø/Ø!5Ø#ð #ó 
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJçØ"Ð/°'¸"µ+Õ=ˆFØ/9Ò/E�Z�M FÕ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r4   )rW  rÆ   r  rS  r‘   s   @@r2   rU  rU  ž  s,   ù‡ € õð ÷F
ñ F
ó ÷F
ð F
r4   rU  r  )r  rÏ   rÅ   r)  rF  rU  )0r�   r#   r   r   Útorch.nnr   r   r   r   r   r¦   Úcache_utilsr   r	   Ú
generationr
   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_mptr   Ú
get_loggerr‰   rý   r3   ÚModuler6   r“   r¬   rÅ   rÏ   r  r)  rF  rU  Ú__all__rÉ   r4   r2   Ú<module>ru     s]  ðñ ã ã Ý ß LÓ LÝ $ç .Ý )Ý /Ý 9÷õ õ .ß ,Ý (ð 
×	Ò	˜HÓ	%€ôô.E)�2—9‘9ô E)ôPˆR�Y‰Yô ô*6$Ð)ô 6$ðr ô%˜ó %ó ð%ð ôG
Ð!ó G
ó ðG
ñT ðôôN
Ð'¨ó N
óðN
ñb ðôôs
Ð#5ó s
óðs
ðl ôU
Ð 2ó U
ó ðU
ðp ôP
Ð0ó P
ó ðP
òf‚r4   