+
    œ°³jpu  ã                   ó\  € R t ^ RIHt ^ RIt^ RIHt ^ RIHtHtHt ^RI	H
t
 ^RIHtHt ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHtHtHtHt ^RIHtHt ^RIHt ^RI H!t!H"t"H#t#H$t$ ^RI%H&t& ^RI'H(t( ^RI)H*t* ]$PV                  ! ],4      t- ! R R]P\                  4      t/R(R R llt0 ! R R]Pb                  4      t2 ! R R]4      t3]" ! R R]4      4       t4 ! R R]44      t5]" ! R R]44      4       t6 ! R  R!]4]4      t7]"! R"R#7       ! R$ R%]44      4       t8]" ! R& R']44      4       t9. R)Ot:R# )*zPyTorch OPT model.)ÚCallableN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úcapture_outputs©Ú	OPTConfigc                   ó^   a a€ ] tR t^-t oRtV3R lV 3R lltRV3R lV 3R llltRtVtV ;t	# )ÚOPTLearnedPositionalEmbeddingzF
This module learns positional embeddings up to a fixed maximum size.
c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )é   Únum_embeddingsÚembedding_dim)Úint)ÚformatÚ__classdict__s   "€Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/opt/modeling_opt.pyÚ__annotate__Ú*OPTLearnedPositionalEmbedding.__annotate__2   s   ø€ ÷ Fñ F¡sð F¹3ñ Fó    c                óV   <€ ^V n         \        SV `	  WP                   ,           V4       R# ©r    N)ÚoffsetÚsuperÚ__init__)Úselfr!   r"   Ú	__class__s   &&&€r&   r.   Ú&OPTLearnedPositionalEmbedding.__init__2   s"   ø€ ð ˆŒÜ‰Ñ˜¯+©+Õ5°}ÖEr)   c                ób   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  R,          /# )r    Úattention_maskÚpast_key_values_lengthÚposition_idsN)ÚtorchÚ
LongTensorr#   )r$   r%   s   "€r&   r'   r(   8   s:   ø€ ÷ ;ñ ;á×(Ñ(ð;ñ !$ð;ñ ×&Ñ&¨Õ-ñ	;r)   c                óÎ   <€ VfC   \         P                  ! V^R7      pW1,          ^,
          P                  4       pVRVR13,          p\        SV `  W0P
                  ,           4      # )z3`input_ids_shape` is expected to be [bsz x seqlen].N©ÚdimºNNN)r6   ÚcumsumÚlongr-   Úforwardr,   )r/   r3   r4   r5   r0   s   &&&&€r&   r>   Ú%OPTLearnedPositionalEmbedding.forward8   sZ   ø€ ð ÒÜ Ÿ<š<¨¸AÔ>ˆLØ(Õ9¸AÕ=×CÑCÓEˆLà'¨Ð+AÑ+BÐ(BÕCˆLä‰w‰˜|¯k©kÕ9Ó:Ð:r)   )r,   )é    N©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r.   r>   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r0   r%   s   @@r&   r   r   -   s%   ù‡ € ñ÷Fó F÷;÷ ;õ ;r)   r   c                óÞ   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        /# )	r    ÚmoduleÚqueryÚkeyÚvaluer3   NÚscalingÚdropout)r   ÚModuler6   ÚTensorÚfloat)r$   s   "r&   r'   r'   J   sg   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ñ%r)   c                 óÎ  € \         P                  ! WP                  RR4      4      V,          pVe	   W„,           p\        P                  P                  VR\         P                  R7      P                  VP                  4      p\        P                  P                  W†V P                  R7      p\         P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )é   )r:   Údtype©ÚpÚtrainingéÿÿÿÿéþÿÿÿ)r6   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxÚfloat32ÚtorW   rQ   rZ   Ú
contiguous)
rL   rM   rN   rO   r3   rP   rQ   ÚkwargsÚattn_weightsÚattn_outputs
   &&&&&&&,  r&   Úeager_attention_forwardrg   J   s°   € ô —<’< §}¡}°R¸Ó'<Ó=ÀÕG€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r)   c                   ó\   a a€ ] tR t^at oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚOPTAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r    ÚconfigÚ	layer_idxN©r   r#   )r$   r%   s   "€r&   r'   ÚOPTAttention.__annotate__d   s'   ø€ ÷ !Yñ !Yáð!Yñ ˜•:ñ!Yr)   c                ó,  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n	        W n
        Vf.   \        P                  RV P                  P                   R24       V P                  V P                  ,          V n        RV n        V P                  V P                  ,          V P                  8w  d'   \#        RV P                   RV P                   R24      hV P                  R,          V n        \&        P(                  ! V P                  V P                  V P                  R7      V n        \&        P(                  ! V P                  V P                  V P                  R7      V n        \&        P(                  ! V P                  V P                  V P                  R7      V n        \&        P(                  ! V P                  V P                  V P                  R7      V n        R # )	NzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).©Úbiasg      à¿)r-   r.   rl   Úhidden_sizeÚ	embed_dimÚnum_attention_headsÚ	num_headsÚattention_dropoutrQ   Úenable_biasrm   ÚloggerÚwarning_oncer0   rB   Úhead_dimÚ	is_causalÚ
ValueErrorrP   r   ÚLinearÚk_projÚv_projÚq_projÚout_proj)r/   rl   rm   rd   r0   s   &&&,€r&   r.   ÚOPTAttention.__init__d   s{  ø€ ô 	‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØ×/Ñ/ˆŒØ!×-Ñ-ˆÔØ"ŒØÒÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð Ÿ™¨$¯.©.Õ8ˆŒØˆŒà�M‰M˜DŸN™NÕ*¨t¯~©~Ô=ÜØMÈdÏnÉnÐM]Ø$ T§^¡^Ð$4°Bð8óð ð —}‘} dÕ*ˆŒä—i’i §¡°·±ÀT×EUÑEUÔVˆŒÜ—i’i §¡°·±ÀT×EUÑEUÔVˆŒÜ—i’i §¡°·±ÀT×EUÑEUÔVˆŒÜŸ	š	 $§.¡.°$·.±.Àt×GWÑGWÔXˆŽr)   c                óÚ   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          RS[RS[S[ P                  S[ P                  R,          S[R,          3,          /# )r    Úhidden_statesÚpast_key_valuesNr3   Úoutput_attentionsÚreturn)r6   rS   r   ÚboolÚtuple)r$   r%   s   "€r&   r'   ro   ‡   sk   ø€ ÷ .)ñ .)á—|‘|ð.)ñ  �ð.)ñ Ÿ™ tÕ+ð	.)ñ
  ð.)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¸$µ,Ð>Õ	?ñ.)r)   c                ól  € VP                  4       w  rgpV P                  V4      V P                  ,          p	V	P                  VRV P                  V P
                  4      P                  ^^4      p	V P                  V4      p
V P                  V4      pV
P                  VRV P                  V P
                  4      P                  ^^4      p
VP                  VRV P                  V P
                  4      P                  ^^4      pVe   VP                  W«V P                  4      w  r«\        P                  ! V P                  P                  \        4      pT! V V	V
VV3RV P                   '       g   RMV P"                  RR/VB w  rÞVP%                  WgR4      P'                  4       pV P)                  V4      pWÞ3# )z#Input shape: Batch x Time x ChannelrQ   ç        rP   g      ð?r[   )Úsizer�   rP   Úviewrv   r{   r^   r   r€   Úupdaterm   r   Úget_interfacerl   Ú_attn_implementationrg   rZ   rQ   Úreshaperc   r‚   )r/   r…   r†   r3   r‡   rd   ÚbszÚtgt_lenÚ_Úquery_statesÚ
key_statesÚvalue_statesÚattention_interfacerf   re   s   &&&&&,         r&   r>   ÚOPTAttention.forward‡   s‘  € ð (×,Ñ,Ó.‰ˆ�að —{‘{ =Ó1°D·L±LÕ@ˆØ#×(Ñ(¨¨b°$·.±.À$Ç-Á-ÓP×ZÑZÐ[\Ð^_Ó`ˆà—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆØ—_‘_ S¨"¨d¯n©n¸d¿m¹mÓL×VÑVÐWXÐZ[Ó\ˆ
Ø#×(Ñ(¨¨b°$·.±.À$Ç-Á-ÓP×ZÑZÐ[\Ð^_Ó`ˆàÒ&à'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð ð	%
ð ñ	%
Ñ!ˆð "×)Ñ)¨#¸Ó;×FÑFÓHˆØ—m‘m KÓ0ˆàÐ(Ð(r)   )rl   rQ   rt   rx   r{   r|   r   rm   rv   r‚   r�   rP   r€   ©N)NNFrA   rJ   s   @@r&   ri   ri   a   s$   ù‡ € ÙG÷!Yõ !Y÷F.)÷ .)ò .)r)   ri   c                   óX   a a€ ] tR t^¸t oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚOPTDecoderLayerc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# rk   rn   )r$   r%   s   "€r&   r'   ÚOPTDecoderLayer.__annotate__¹   s#   ø€ ÷ vñ v™yð v±S¸4µZñ vr)   c                ó˜  <€ \         SV `  4        VP                  V n        \	        WR 7      V n        VP                  V n        VP                  V n        \        VP                  ,          V n
        \        P                  ! V P                  VP                  R7      V n        \        P                  ! V P                  VP                   VP"                  R7      V n        \        P                  ! VP                   V P                  VP"                  R7      V n        \        P                  ! V P                  VP                  R7      V n        R# ))rl   rm   ©Úelementwise_affinerq   N)r-   r.   rs   rt   ri   Ú	self_attnÚdo_layer_norm_beforerQ   r   Úactivation_functionÚactivation_fnr   Ú	LayerNormÚlayer_norm_elementwise_affineÚself_attn_layer_normr~   Úffn_dimrx   Úfc1Úfc2Úfinal_layer_norm)r/   rl   rm   r0   s   &&&€r&   r.   ÚOPTDecoderLayer.__init__¹   s×   ø€ Ü‰ÑÔØ×+Ñ+ˆŒä%¨VÔIˆŒà$*×$?Ñ$?ˆÔ!Ø—~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔä$&§L¢LØ�N‰N¨v×/SÑ/Sô%
ˆÔ!ô —9’9˜TŸ^™^¨V¯^©^À&×BTÑBTÔUˆŒÜ—9’9˜VŸ^™^¨T¯^©^À&×BTÑBTÔUˆŒÜ "§¢¨T¯^©^ÐPV×PtÑPtÔ uˆÖr)   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[R,          RS[ P                  R,          RS[S[,          RS[ P                  /# )	r    r…   r3   Nr†   Ú	use_cacher5   rd   rˆ   )r6   rS   r   r‰   r7   r   r   )r$   r%   s   "€r&   r'   rŸ   Ê   s{   ø€ ÷ 3ñ 3á—|‘|ð3ñ Ÿ™ tÕ+ð3ñ  �ð	3ñ
 ˜$•;ð3ñ ×&Ñ&¨Õ-ð3ñ Ñ-Õ.ð3ñ 
�‰ñ3r)   c                ó0  € TpV P                   '       d   V P                  V4      pV P                  ! RR VRVRVRV/VB w  r\        P                  P                  WP
                  V P                  R7      pWq,           pV P                   '       g   V P                  V4      pVP                  p	VP                  RVP                  R4      4      pTpV P                   '       d   V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      p\        P                  P                  WP
                  V P                  R7      pWq,           P                  V	4      pV P                   '       g   V P                  V4      pV# )r…   r†   r5   r3   rX   © r[   )r¤   r©   r£   r   r_   rQ   rZ   Úshaper’   r�   r­   r«   r¦   r¬   rŽ   )
r/   r…   r3   r†   r°   r5   rd   Úresidualr•   Úhidden_states_shapes
   &&&&&&,   r&   r>   ÚOPTDecoderLayer.forwardÊ   sŠ  € ð !ˆð ×$×$Ð$Ø ×5Ñ5°mÓDˆMð  Ÿ>š>ñ 
Ø'ð
à+ð
ð &ð
ð *ð	
ð
 ñ
Ñˆô Ÿ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆð ×(×(Ð(Ø ×5Ñ5°mÓDˆMð ,×1Ñ1ÐØ%×-Ñ-¨b°-×2DÑ2DÀRÓ2HÓIˆØ ˆð ×$×$Ð$Ø ×1Ñ1°-Ó@ˆMàŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆàŸ™ Ó/ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-Ódˆà!Õ1×7Ñ7Ð8KÓLˆð ×(×(Ð(Ø ×1Ñ1°-Ó@ˆMàÐr)   )	r¦   r¤   rQ   rt   r«   r¬   r­   r£   r©   r›   )NNFN)	rB   rC   rD   rE   r.   r>   rG   rH   rI   rJ   s   @@r&   r�   r�   ¸   s    ù‡ € ÷võ v÷"3÷ 3ò 3r)   r�   c                   óZ   a € ] tR tRt o RtRtR.tRtRtRt	Rt
RtR]R]/tV 3R ltRtV tR	# )
ÚOPTPreTrainedModelé   ÚmodelTr�   r…   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©r    rl   r   )r$   r%   s   "€r&   r'   ÚOPTPreTrainedModel.__annotate__   s   ø‡ ‚ áÑò r)   r²   N)rB   rC   rD   rE   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphr�   ri   Ú_can_record_outputsÚ__annotate_func__rG   rH   )r%   s   @r&   r¸   r¸      sR   ø‡ € ð  ÐØ&*Ð#Ø*Ð+ÐØ"&ÐØÐØ€NØÐØ!Ðà˜Ø�lðÐ÷ ƒ r)   r¸   c                   óv   a a€ ] tR tRt oRtV3R lV 3R llt]]]RV3R lR ll4       4       4       t	Rt
VtV ;t# )	Ú
OPTDecoderi  z‡
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`OPTDecoderLayer`]

Args:
    config: OPTConfig
c                ó    <€ V ^8„  d   QhRS[ /# r½   r   )r$   r%   s   "€r&   r'   ÚOPTDecoder.__annotate__  s   ø€ ÷ #ñ #™yñ #r)   c           
     óf  <€ \         SV `  V4       VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                  ! VP                  VP                  V P
                  4      V n        \        VP                  VP                  4      V n        VP                  VP                  8w  d4   \        P                   ! VP                  VP                  R R7      V n        MRV n        VP                  VP                  8w  d4   \        P                   ! VP                  VP                  R R7      V n        MRV n        VP&                  '       dE   VP(                  '       g3   \        P*                  ! VP                  VP,                  R7      V n        MRV n        \        P0                  ! \3        VP4                  4       Uu. uF  p\7        WR7      NK  	  up4      V n        R V n        V P=                  4        R# u upi )Frq   Nr¡   )rm   )r-   r.   rQ   Ú	layerdropÚpad_token_idÚpadding_idxÚmax_position_embeddingsÚmax_target_positionsÚ
vocab_sizer   Ú	EmbeddingÚword_embed_proj_dimÚembed_tokensr   rs   Úembed_positionsr~   Úproject_outÚ
project_inr¤   Ú_remove_final_layer_normr§   r¨   r­   Ú
ModuleListÚrangeÚnum_hidden_layersr�   ÚlayersÚgradient_checkpointingÚ	post_init)r/   rl   Úir0   s   && €r&   r.   ÚOPTDecoder.__init__  sŸ  ø€ Ü‰Ñ˜Ô Ø—~‘~ˆŒØ×)Ñ)ˆŒØ!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<VÑ<VÐX\×XhÑXhÓiˆÔÜ<¸V×=[Ñ=[Ð]c×]oÑ]oÓpˆÔà×%Ñ%¨×);Ñ);Ô;Ü!Ÿyšy¨×);Ñ);¸V×=WÑ=WÐ^cÔdˆDÕà#ˆDÔà×%Ñ%¨×);Ñ);Ô;Ü Ÿiši¨×(BÑ(BÀF×DVÑDVÐ]bÔcˆD�Oà"ˆDŒOð
 ×&×&Ð&¨v×/N×/NÐ/NÜ$&§L¢LØ×"Ñ"°v×7[Ñ7[ô%ˆDÕ!ð %)ˆDÔ!ä—m’mÔSXÐY_×YqÑYqÔSrÓ$sÑSrÈa¤_°V×%IÑSrÑ$sÓtˆŒà&+ˆÔ#à�‰Öùò	 %ts   Ç4H.c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          R	S[/# ©
r    Ú	input_idsNr3   r†   Úinputs_embedsr°   r5   rd   rˆ   ©	r6   r7   rS   r   ÚFloatTensorr‰   r   r   r   )r$   r%   s   "€r&   r'   rÌ   A  s�   ø€ ÷ K
ñ K
á×#Ñ# dÕ*ðK
ñ Ÿ™ tÕ+ðK
ñ  �ð	K
ñ
 ×(Ñ(¨4Õ/ðK
ñ ˜$•;ðK
ñ ×&Ñ&¨Õ-ðK
ñ Ñ+Õ,ðK
ñ 
!ñK
r)   c                ó’  € VR J VR J,          '       d   \        R4      hVe$   VP                  RVP                  R,          4      pVf   V P                  V4      pV'       d   Vf   \	        V P
                  R7      pVe   VP                  4       M^ pVfM   W„P                  ^,          ,           p	\        P                  ! VP                  ^ ,          W”P                  R7      pVfC   \        P                  ! V^R7      pWb,          ^,
          P                  4       pVRVR 13,          p\        V P
                  VVVR7      p
V P                  W(VR7      pV P                  e   V P                  V4      pWKP                  VP                  4      ,           p\!        V P"                  4       FP  w  rÞV P$                  '       d)   \        P&                  ! . 4      pWðP(                  8  d   K?  V! V3RV
R	VR
VRV/VB pKR  	  V P*                  e   V P+                  V4      pV P,                  e   V P-                  V4      p\/        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)rl   ©Údevicer9   r;   )rl   ræ   r3   r†   )r5   r3   r5   r†   r°   )Úlast_hidden_stater†   r[   )r}   rŽ   r³   rÖ   r	   rl   Úget_seq_lengthr6   Úonesrë   r<   r=   r   r×   rÙ   rb   Ú	enumeraterÞ   rZ   ÚrandrÎ   r­   rØ   r   )r/   rå   r3   r†   ræ   r°   r5   rd   Úpast_seen_tokensÚ
seq_lengthÚcausal_maskÚ
pos_embedsr…   ÚidxÚdecoder_layerÚdropout_probabilitys   &&&&&&&,        r&   r>   ÚOPTDecoder.forward>  s)  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø!Ÿ™ r¨9¯?©?¸2Õ+>Ó?ˆIàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOà?NÒ?Z˜?×9Ñ9Ô;Ð`aÐàÒ!Ø)×,?Ñ,?ÀÕ,BÕBˆJÜ"ŸZšZ¨×(;Ñ(;¸AÕ(>À
×SgÑSgÔhˆNð ÒÜ Ÿ<š<¨¸AÔ>ˆLØ(Õ9¸AÕ=×CÑCÓEˆLà'¨Ð+;Ñ+<Ð(<Õ=ˆLä(Ø—;‘;Ø'Ø)Ø+ô	
ˆð ×)Ñ)¨.ÐYeÐ)Ófˆ
à�?‰?Ò&Ø ŸO™O¨MÓ:ˆMà%¯©°m×6JÑ6JÓ(KÕKˆô #,¨D¯K©KÖ"8ÑˆCà�}�}ˆ}Ü&+§j¢j°£nÐ#Ø&¯©Ô7Ùá)Øñà*ðð *ðð !0ð	ð
 $ðð ñŠMñ #9ð  × Ñ Ò,Ø ×1Ñ1°-Ó@ˆMà×ÑÒ'Ø ×,Ñ,¨]Ó;ˆMä&Ø+Ø+ô
ð 	
r)   )rQ   r×   rÖ   r­   rß   rÎ   rÞ   rÒ   rÐ   rÙ   rØ   rÓ   ©NNNNNN)rB   rC   rD   rE   rF   r.   r   r   r   r>   rG   rH   rI   rJ   s   @@r&   rÊ   rÊ     sG   ù‡ € ñ÷#ó #ðJ  ØØ÷K
ñ K
ó ó ó  ÷K
ð K
r)   rÊ   c                   ót   a a€ ] tR tRt oV3R lV 3R lltR tR t]]R	V3R lR ll4       4       t	Rt
VtV ;t# )
ÚOPTModeli�  c                ó    <€ V ^8„  d   QhRS[ /# r½   r   )r$   r%   s   "€r&   r'   ÚOPTModel.__annotate__‘  s   ø€ ÷ ñ ™yñ r)   c                óf   <€ \         SV `  V4       \        V4      V n        V P	                  4        R # r›   )r-   r.   rÊ   Údecoderrà   ©r/   rl   r0   s   &&€r&   r.   ÚOPTModel.__init__‘  s&   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒà�‰Ör)   c                ó.   € V P                   P                  # r›   ©rÿ   rÖ   ©r/   s   &r&   Úget_input_embeddingsÚOPTModel.get_input_embeddings—  s   € Ø�|‰|×(Ñ(Ð(r)   c                ó&   € WP                   n        R # r›   r  ©r/   rO   s   &&r&   Úset_input_embeddingsÚOPTModel.set_input_embeddingsš  s   € Ø$)�‰Ö!r)   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          R	S[/# rä   rç   )r$   r%   s   "€r&   r'   rý   Ÿ  s“   ø€ ÷ 
ñ 
á×#Ñ# dÕ*ð
ñ Ÿ™ tÕ+ð
ñ  �ð	
ñ
 ×(Ñ(¨4Õ/ð
ñ ˜$•;ð
ñ ×&Ñ&¨Õ-ð
ñ Ñ+Õ,ð
ñ 
!ñ
r)   c                ó¬   € V P                   ! RR VRVRVRVRVRV/VB p\        VP                  VP                  VP                  VP
                  R7      # )rå   r3   r5   r†   ræ   r°   )rì   r†   r…   r»   r²   )rÿ   r   rì   r†   r…   r»   )	r/   rå   r3   r†   ræ   r°   r5   rd   Údecoder_outputss	   &&&&&&&, r&   r>   ÚOPTModel.forward�  s‡   € ð 48·<²<ñ 4
Øð4
à)ð4
ð &ð4
ð ,ð	4
ð
 (ð4
ð  ð4
ð ñ4
ˆô 'Ø-×?Ñ?Ø+×;Ñ;Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r)   )rÿ   rù   )rB   rC   rD   rE   r.   r  r	  r   r   r>   rG   rH   rI   rJ   s   @@r&   rû   rû   �  s?   ù‡ € ÷ó ò)ò*ð Ø÷
ñ 
ó ó ÷
ð 
r)   rû   c                   óp   a a€ ] tR tRt oRR/tV 3R ltR tR t]]	R
V3R lR ll4       4       t
R	tVtV ;t# )ÚOPTForCausalLMi»  zlm_head.weightz!model.decoder.embed_tokens.weightc                óÊ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        V P                  4        R# ©Frq   N)
r-   r.   rû   rº   r   r~   rÕ   rÓ   Úlm_headrà   r   s   &&€r&   r.   ÚOPTForCausalLM.__init__¾  sK   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ
ô —y’y ×!;Ñ!;¸V×=NÑ=NÐUZÔ[ˆŒð 	�‰Ör)   c                óB   € V P                   P                  P                  # r›   ©rº   rÿ   rÖ   r  s   &r&   r  Ú#OPTForCausalLM.get_input_embeddingsÈ  ó   € Ø�z‰z×!Ñ!×.Ñ.Ð.r)   c                ó:   € WP                   P                  n        R # r›   r  r  s   &&r&   r	  Ú#OPTForCausalLM.set_input_embeddingsË  ó   € Ø*/�
‰
×ÑÖ'r)   c                ó`  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[S[ P                  ,          R
S[S[,          RS[	S[
,          /
# )r    rå   Nr3   r†   ræ   Úlabelsr°   r5   Úlogits_to_keeprd   rˆ   )r6   r7   rS   r   rè   r‰   r#   r   r   rŠ   r   )r$   r%   s   "€r&   r'   ÚOPTForCausalLM.__annotate__Ð  sÁ   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ Ÿ™ tÕ+ð<
ñ  �ð	<
ñ
 ×(Ñ(¨4Õ/ð<
ñ × Ñ  4Õ'ð<
ñ ˜$•;ð<
ñ ×&Ñ&¨Õ-ð<
ñ ™eŸl™lÕ*ð<
ñ Ñ+Õ,ð<
ñ 
Ñ'Õ	'ñ<
r)   c	                óÆ  € V P                   P                  ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      P                  4       pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )aî  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, OPTForCausalLM

>>> model = OPTForCausalLM.from_pretrained("facebook/opt-350m")
>>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious. I'm just a little bit of a weirdo."
```rå   r3   r5   r†   ræ   r°   Nr;   Úlogitsr  rÓ   ©Úlossr!  r†   r…   r»   r²   )rº   rÿ   rì   Ú
isinstancer#   Úslicer  rc   Úloss_functionrl   rÓ   r   r†   r…   r»   )r/   rå   r3   r†   ræ   r  r°   r5   r  rd   Úoutputsr…   Úslice_indicesr!  r#  s   &&&&&&&&&,     r&   r>   ÚOPTForCausalLM.forwardÎ  s  € ðJ ,0¯:©:×+=Ò+=ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓA×LÑLÓNˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r)   )r  rº   )NNNNNNNr@   )rB   rC   rD   rE   Ú_tied_weights_keysr.   r  r	  r   r   r>   rG   rH   rI   rJ   s   @@r&   r  r  »  sF   ù‡ € Ø*Ð,OÐPÐõò/ò0ð Ø÷<
ñ <
ó ó ÷<
ð <
r)   r  aÒ  
    The OPT Model transformer with a sequence classification head on top (linear layer).

    [`OPTForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )Úcustom_introc                   ót   a a€ ] tR tRt oV3R lV 3R llt]]R	V3R lR ll4       4       tR tR t	Rt
VtV ;t# )
ÚOPTForSequenceClassificationi  c                ó    <€ V ^8„  d   QhRS[ /# r½   r   )r$   r%   s   "€r&   r'   Ú)OPTForSequenceClassification.__annotate__  s   ø€ ÷ ñ ™yñ r)   c                óì   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  V P                  R R7      V n        V P                  4        R# r  )
r-   r.   Ú
num_labelsrû   rº   r   r~   rÕ   Úscorerà   r   s   &&€r&   r.   Ú%OPTForSequenceClassification.__init__  sT   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ˜fÓ%ˆŒ
Ü—Y’Y˜v×9Ñ9¸4¿?¹?ÐQVÔWˆŒ
ð 	�‰Ör)   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[S[,          R
S[S[,          /	# )r    rå   Nr3   r†   ræ   r  r°   r5   rd   rˆ   )	r6   r7   rè   r   r‰   r   r   rŠ   r   )r$   r%   s   "€r&   r'   r/  )  s»   ø€ ÷ Q
ñ Q
á×#Ñ# dÕ*ðQ
ñ ×)Ñ)¨DÕ0ðQ
ñ  �ð	Q
ñ
 ×(Ñ(¨4Õ/ðQ
ñ × Ñ  4Õ'ðQ
ñ ˜$•;ðQ
ñ ×&Ñ&¨Õ-ðQ
ñ Ñ+Õ,ðQ
ñ 
Ñ1Õ	1ñQ
r)   c                óè  € V P                   ! V3RVRVRVRVRV/VB p	V	P                  p
V P                  V
4      pVe   VP                  R,          w  rÍMVP                  R,          w  rÍV P                  P
                  f   V^8w  d   \        R4      hV P                  P
                  f   RpMÍVeœ   WP                  P
                  8g  P                  VP                  \        P                  4      p\        P                  ! VP                  R,          VP                  \        P                  R	7      pVV,          P                  R4      pM.Rp\        P                  V P                  P                    R
24       V\        P                  ! WËP                  R7      V3,          pRpVEeš   V P                  P"                  f–   V P$                  ^8X  d   RV P                  n        MrV P$                  ^8”  dQ   VP&                  \        P(                  8X  g    VP&                  \        P*                  8X  d   RV P                  n        MRV P                  n        V P                  P"                  R8X  dL   \-        4       pV P$                  ^8X  d'   V! VP/                  4       VP/                  4       4      pM�V! VV4      pM†V P                  P"                  R8X  d>   \1        4       pV! VP3                  RV P$                  4      VP3                  R4      4      pM.V P                  P"                  R8X  d   \5        4       pV! VV4      p\7        VVV	P8                  V	P:                  V	P<                  R7      # )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r†   r3   r5   ræ   r°   N:Nr    Nz=Cannot handle batch sizes > 1 if no padding token is defined.)rë   rW   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rê   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr"  r[   )rº   rì   r2  r³   rl   rÏ   r}   rb   rë   r6   Úint32ÚarangeÚargmaxry   rz   r0   rB   Úproblem_typer1  rW   r=   r#   r   Úsqueezer   rŽ   r   r   r†   r…   r»   )r/   rå   r3   r†   ræ   r  r°   r5   rd   Útransformer_outputsr…   r!  Ú
batch_sizeÚsequence_lengthÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsr#  Úloss_fcts   &&&&&&&&,           r&   r>   Ú$OPTForSequenceClassification.forward'  så  € ð& 8<·z²zØñ8
à+ð8
ð *ð8
ð &ð	8
ð
 (ð8
ð  ð8
ð ñ8
Ðð ,×=Ñ=ˆØ—‘˜MÓ*ˆàÒ Ø*3¯/©/¸"Õ*=Ñ'ˆJ˜à*7×*=Ñ*=¸bÕ*AÑ'ˆJà�;‰;×#Ñ#Ò+°
¸a´ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ò+Ø!#ÑØÒ"à%¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸLšL¨¯©¸Õ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Õ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¿}¹}ÔMÐOaÐaÕbˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù × 2Ñ 2°2°t·±Ó GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù ¨vÓ6�ä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r)   c                óB   € V P                   P                  P                  # r›   r  r  s   &r&   r  Ú1OPTForSequenceClassification.get_input_embeddings|  r  r)   c                ó:   € WP                   P                  n        R # r›   r  r  s   &&r&   r	  Ú1OPTForSequenceClassification.set_input_embeddings  r  r)   )rº   r1  r2  )NNNNNNN©rB   rC   rD   rE   r.   r   r   r>   r  r	  rG   rH   rI   rJ   s   @@r&   r-  r-    sC   ù‡ € ÷ó ð Ø÷Q
ñ Q
ó ó ðQ
òf/÷0ò 0r)   r-  c                   ót   a a€ ] tR tRt oV3R lV 3R llt]]R	V3R lR ll4       4       tR tR t	Rt
VtV ;t# )
ÚOPTForQuestionAnsweringiƒ  c                ó    <€ V ^8„  d   QhRS[ /# r½   r   )r$   r%   s   "€r&   r'   Ú$OPTForQuestionAnswering.__annotate__…  s   ø€ ÷ ñ ™yñ r)   c                ó²   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  ^4      V n        V P                  4        R# r+   )	r-   r.   rû   rº   r   r~   rÕ   Ú
qa_outputsrà   r   s   &&€r&   r.   Ú OPTForQuestionAnswering.__init__…  s@   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ
ÜŸ)š) F×$>Ñ$>ÀÓBˆŒð 	�‰Ör)   c                ó^  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[S[,          RS[S[,          /
# )r    rå   Nr3   r†   ræ   Ústart_positionsÚend_positionsr°   r5   rd   rˆ   )	r6   r7   rè   r   r‰   r   r   rŠ   r   )r$   r%   s   "€r&   r'   rO  �  sÑ   ø€ ÷ Q
ñ Q
á×#Ñ# dÕ*ðQ
ñ ×)Ñ)¨DÕ0ðQ
ñ  �ð	Q
ñ
 ×(Ñ(¨4Õ/ðQ
ñ ×)Ñ)¨DÕ0ðQ
ñ ×'Ñ'¨$Õ.ðQ
ñ ˜$•;ðQ
ñ ×&Ñ&¨Õ-ðQ
ñ Ñ+Õ,ðQ
ñ 
Ñ-Õ	-ñQ
r)   c	                óZ  € V P                   ! V3RVRVRVRVRV/V	B p
V
P                  pV P                  V4      pVP                  ^R
R7      w  rÞVP	                  R
4      P                  4       pVP	                  R
4      P                  4       pRpVeö   Veò   \        VP                  4       4      ^8”  d   VP	                  R
4      p\        VP                  4       4      ^8”  d   VP	                  R
4      pVP                  ^4      pVP                  ^ V4      P                  VP                  4      pVP                  ^ V4      P                  VP                  4      p\        VR7      pV! WÕ4      pV! Wæ4      pVV,           ^,          p\        VVVV
P                  V
P                  R	7      # )a  
Example:

```python
>>> from transformers import AutoTokenizer, OPTForQuestionAnswering
>>> import torch

>>> torch.manual_seed(4)  # doctest: +IGNORE_RESULT
>>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m")

>>> # note: we are loading a OPTForQuestionAnswering from the hub here,
>>> # so the head will be randomly initialized, hence the predictions will be random
>>> model = OPTForQuestionAnswering.from_pretrained("facebook/opt-350m")

>>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"

>>> inputs = tokenizer(question, text, return_tensors="pt")
>>> with torch.no_grad():
...     outputs = model(**inputs)

>>> answer_start_index = outputs.start_logits.argmax()
>>> answer_end_index = outputs.end_logits.argmax()

>>> answer_offset = len(tokenizer(question)[0])

>>> predict_answer_tokens = inputs.input_ids[
...     0, answer_offset + answer_start_index : answer_offset + answer_end_index + 1
... ]
>>> predicted = tokenizer.decode(predict_answer_tokens)
>>> predicted
' a nice puppet'
```r†   r3   r5   ræ   r°   r9   N)Úignore_index)r#  Ústart_logitsÚ
end_logitsr…   r»   r[   )rº   rì   rQ  Úsplitr=  rc   Úlenr�   Úclamprb   rë   r   r   r…   r»   )r/   rå   r3   r†   ræ   rT  rU  r°   r5   rd   r>  r…   r!  rX  rY  Ú
total_lossÚignored_indexrE  Ú
start_lossÚend_losss   &&&&&&&&&,          r&   r>   ÚOPTForQuestionAnswering.forward�  s¶  € ð\ 8<·z²zØñ8
à+ð8
ð *ð8
ð &ð	8
ð
 (ð8
ð  ð8
ð ñ8
Ðð ,×=Ñ=ˆà—‘ Ó/ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+DÜ�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�Ø(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓE×HÑHÈÏÉÓWˆOØ)×/Ñ/°°=ÓA×DÑDÀVÇ]Á]ÓSˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJä+ØØ%Ø!Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r)   c                óB   € V P                   P                  P                  # r›   r  r  s   &r&   r  Ú,OPTForQuestionAnswering.get_input_embeddingsâ  r  r)   c                ó:   € WP                   P                  n        R # r›   r  r  s   &&r&   r	  Ú,OPTForQuestionAnswering.set_input_embeddingså  r  r)   )rº   rQ  )NNNNNNNNrK  rJ   s   @@r&   rM  rM  ƒ  sC   ù‡ € ÷ó ð Ø÷Q
ñ Q
ó ó ðQ
òf/÷0ò 0r)   rM  )rŒ   )r  rû   r¸   r-  rM  );rF   Úcollections.abcr   r6   r   Útorch.nnr   r   r   Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_optr   Ú
get_loggerrB   ry   rÔ   r   rg   rR   ri   r�   r¸   rÊ   rû   r  r-  rM  Ú__all__r²   r)   r&   Ú<module>rw     sB  ðñ å $ã Ý ß AÑ Aå !ß .Ý )Ý /Ý BÝ 9÷ó ÷ GÝ &ß RÓ RÝ 7Ý 5Ý (ð 
×	Ò	˜HÓ	%€ô; B§L¡Lô ;÷:%ô.T)�2—9‘9ô T)ônEÐ0ô EðP ô˜ó ó ðô {
Ð#ô {
ð| ô(
Ð!ó (
ó ð(
ôVQ
Ð'¨ô Q
ñh ðôôc0Ð#5ó c0óðc0ðL ôb0Ð0ó b0ó ðb0òJ‚r)   