+
    œ°³jU�  ã                   óœ  € R t ^ RIt^ RIt^ RIHt ^ RIHtHtHt ^RIH	t
 ^RIHt ^RIHtHt ^RIHt ^RIHt ^R	IHtHt ^R
IHt ^RIHtHtHtHt ^RIHt ^RI H!t!H"t" ^RI#H$t$ ]! 4       '       d   ^RIH%t% ]"PL                  ! ]'4      t(R R lt)R t*R R lt+R R lt, ! R R]PZ                  4      t. ! R R].4      t/R].R]//t0 ! R R]PZ                  4      t1 ! R R ]4      t2]! ! R! R"]4      4       t3]! ! R# R$]34      4       t4]!! R%R&7       ! R' R(]3]4      4       t5]!! R)R&7       ! R* R+]34      4       t6]! ! R, R-]34      4       t7. R.Ot8R# )/zPyTorch GPT-J model.N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)Ú!flash_attn_supports_top_left_maskÚis_flash_attn_available)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPast)ÚPreTrainedModel)Úauto_docstringÚlogging©Ú
GPTJConfig)Ú_flash_attention_forwardc                óP   € V ^8„  d   QhR\         R\         R\        P                  /# )é   Únum_posÚdimÚreturn)ÚintÚtorchÚTensor)Úformats   "Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gptj/modeling_gptj.pyÚ__annotate__r#   /   s*   € ÷ Pñ P¬ð P´3ð P¼5¿<¹<ñ Pó    c           	      ó°  € R R\         P                  ! ^ V^\         P                  R7      V,          ,          ,          p\         P                  ! R\         P                  ! V \         P                  R7      P	                  4       V4      P	                  4       p\         P
                  ! \         P                  ! V4      \         P                  ! V4      3^R7      # )g      ð?i'  )Údtypezi , j -> i j©r   )r   ÚarangeÚint64ÚeinsumÚfloatÚcatÚsinÚcos)r   r   Úinv_freqÚsinusoid_inps   &&  r"   Úcreate_sinusoidal_positionsr1   /   s…   € Ø�e¤§¢¨Q°°Q¼e¿k¹kÔ JÈSÕ PÕQÕR€HÜ—<’< ´·²¸WÌEÏKÉKÔ0X×0^Ñ0^Ó0`ÐbjÓk×qÑqÓs€LÜ�9Š9”e—i’i Ó-¬u¯yªy¸Ó/FÐGÈQÔOÐOr$   c                 ó|   € V P                  VP                  4      P                  VP                  ^ ,          ^^4      # ©é    )ÚtoÚdeviceÚrepeatÚshape)Úembed_positionsÚposition_idss   &&r"   Úget_embed_positionsr;   5   s5   € Ø×Ñ˜l×1Ñ1Ó2×9Ñ9¸,×:LÑ:LÈQÕ:OÐQRÐTUÓVÐVr$   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r   Úxr   ©r   r    )r!   s   "r"   r#   r#   9   s"   € ÷ ñ œŸ™ð ¬¯©ñ r$   c                 ó~   € V R,          pV R,          p\         P                  ! V) V3RR7      p V P                  R4      # )ºNNNr'   )r@   r@   r@   :NNr   )r@   r@   r@   :é   Nr   éÿÿÿÿéþÿÿÿ)r   ÚstackÚflatten)r=   Úx1Úx2s   &  r"   Úrotate_every_tworH   9   s9   € Ø	
ˆ<�€BØ	
ˆ=Õ	€BÜ�Š�b�S˜"�I 2Ô&€AØ�9‰9�R‹=Ðr$   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# )r   Útensorr-   r.   r   r>   )r!   s   "r"   r#   r#   @   s;   € ÷ =ñ =¤§¡ð =´E·L±Lð =ÄuÇ|Á|ð =ÔX]×XdÑXdñ =r$   c                 ó¼   € \         P                  ! VR,          ^^4      p\         P                  ! VR,          ^^4      pW,          \        V 4      V,          ,           # )r@   )r@   r@   Nr@   )r   Úrepeat_interleaverH   )rJ   r-   r.   s   &&&r"   Úapply_rotary_pos_embrM   @   sL   € Ü
×
!Ò
! # mÕ"4°a¸Ó
;€CÜ
×
!Ò
! # mÕ"4°a¸Ó
;€CØ�LÔ-¨fÓ5¸Õ;Õ<Ð<r$   c                   óh   a a€ ] tR t^Ft oR	V 3R lltR tR tR	R ltR tR
V3R lR llt	Rt
VtV ;t# )ÚGPTJAttentionc                óJ  <€ \         SV `  4        Wn        VP                  V n        \
        P                  ! VP                  4      V n        \
        P                  ! VP                  4      V n
        R V n        W n        Vf.   \        P                  RV P                  P                    R24       VP"                  V n        VP&                  V n        V P$                  V P&                  ,          V n        V P(                  V P&                  ,          V P$                  8w  d'   \+        RV P$                   RV P&                   R24      h\,        P.                  ! V P(                  4      V n        \
        P2                  ! V P$                  V P$                  RR7      V n        \
        P2                  ! V P$                  V P$                  RR7      V n        \
        P2                  ! V P$                  V P$                  RR7      V n        \
        P2                  ! V P$                  V P$                  RR7      V n        VP<                  V n        V P<                  ;'       g    V P$                  V n        \
        P@                  ! \C        V P                  V P>                  4      RR	7      V n"        R# )
TNzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).F©Úbias)Ú
persistent)#ÚsuperÚ__init__ÚconfigÚmax_position_embeddingsÚmax_positionsr   ÚDropoutÚ
attn_pdropÚattn_dropoutÚresid_pdropÚresid_dropoutÚ	is_causalÚ	layer_idxÚloggerÚwarning_onceÚ	__class__Ú__name__Úhidden_sizeÚ	embed_dimÚnum_attention_headsÚhead_dimÚ
ValueErrorÚmathÚsqrtÚ
scale_attnÚLinearÚk_projÚv_projÚq_projÚout_projÚ
rotary_dimÚpos_embd_dimÚBufferr1   r9   )ÚselfrV   r_   rb   s   &&&€r"   rU   ÚGPTJAttention.__init__G   sä  ø€ Ü‰ÑÔØŒØ#×;Ñ;ˆÔäŸJšJ v×'8Ñ'8Ó9ˆÔÜŸZšZ¨×(:Ñ(:Ó;ˆÔàˆŒØ"ŒØÒÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð  ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ ØŸ™¨$×*BÑ*BÕBˆŒØ�=‰=˜4×3Ñ3Õ3°t·~±~ÔEÜØWÐX\×XfÑXfÐWgð h+Ø+/×+CÑ+CÐ*DÀBðHóð ô Ÿ)š) D§M¡MÓ2ˆŒä—i’i §¡°·±ÀUÔKˆŒÜ—i’i §¡°·±ÀUÔKˆŒÜ—i’i §¡°·±ÀUÔKˆŒÜŸ	š	 $§.¡.°$·.±.ÀuÔMˆŒØ ×+Ñ+ˆŒØ ŸO™O×=Ð=¨t¯~©~ˆÔÜ!ŸyšyÜ'¨×(:Ñ(:¸D×<MÑ<MÓNÐ[`ô 
ˆÖr$   c                ój  € VP                  4       RR W#3,           pVP                  V4      pV'       d   V# \        VP                  4      ^8X  d   VP	                  ^ ^^^^4      # \        VP                  4      ^8X  d   VP	                  ^ ^^^4      # \        R\        VP                  4       24      h)z?
Splits hidden dim into attn_head_size and num_attention_heads
Nú3Input tensor rank should be one of [4, 5], but is: rB   )ÚsizeÚviewÚlenr8   Úpermuterh   )rt   rJ   rf   Úattn_head_sizeÚrotaryÚ	new_shapes   &&&&& r"   Ú_split_headsÚGPTJAttention._split_headsl   s¤   € ð —K‘K“M # 2Ð&Ð*=Ð)NÕNˆ	Ø—‘˜YÓ'ˆßØˆMÜˆv�|‰|Ó Ô!Ø—>‘> ! Q¨¨1¨aÓ0Ð0Ü�—‘Ó !Ô#Ø—>‘> ! Q¨¨1Ó-Ð-äÐRÔSVÐW]×WcÑWcÓSdÐReÐfÓgÐgr$   c                óž  € \        VP                  4      ^8X  d%   VP                  ^ ^^^^4      P                  4       pM^\        VP                  4      ^8X  d$   VP                  ^ ^^^4      P                  4       pM!\	        R\        VP                  4       24      hVP                  4       RR W#,          3,           pVP                  V4      # )zB
Merges attn_head_size dim and num_attn_heads dim into hidden dim
rw   NrC   )rz   r8   r{   Ú
contiguousrh   rx   ry   )rt   rJ   rf   r|   r~   s   &&&& r"   Ú_merge_headsÚGPTJAttention._merge_heads{   s°   € ô ˆv�|‰|Ó Ô!Ø—^‘^ A q¨!¨Q°Ó2×=Ñ=Ó?‰FÜ�—‘Ó !Ô#Ø—^‘^ A q¨!¨QÓ/×:Ñ:Ó<‰FäÐRÔSVÐW]×WcÑWcÓSdÐReÐfÓgÐgØ—K‘K“M # 2Ð&Ð*=Õ*NÐ)PÕPˆ	Ø�{‰{˜9Ó%Ð%r$   c                óÒ  € VP                  \        P                  4      pVP                  \        P                  4      p\        P                  ! WP	                  RR4      4      pWPP
                  ,          pVe	   WT,           p\        P                  P                  VRR7      pVP                  VP                  4      pV P                  V4      p\        P                  ! WS4      pWe3# )rA   r'   rB   rC   )r5   r   Úfloat32ÚmatmulÚ	transposerk   r   Ú
functionalÚsoftmaxr&   r[   )rt   ÚqueryÚkeyÚvalueÚattention_maskÚattn_weightsÚattn_outputs   &&&&&  r"   Ú_attnÚGPTJAttention._attnˆ   s­   € ð —‘œŸ™Ó'ˆØ�f‰f”U—]‘]Ó#ˆä—|’| E¯=©=¸¸RÓ+@ÓAˆØ#§o¡oÕ5ˆàÒ%Ø'Õ8ˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆØ#—‘ u§{¡{Ó3ˆØ×(Ñ(¨Ó6ˆä—l’l <Ó7ˆàÐ(Ð(r$   c                óÚ   € V P                   pVP                  VP                  8w  d"   VP                  VP                  4      pW n         VP                  VP                  ^ ,          ^^4      # r3   )r9   r6   r5   r7   r8   )rt   r:   r9   s   && r"   Ú_get_embed_positionsÚ"GPTJAttention._get_embed_positions¡   s]   € Ø×.Ñ.ˆØ×!Ñ! \×%8Ñ%8Ô8Ø-×0Ñ0°×1DÑ1DÓEˆOØ#2Ô Ø×%Ñ% l×&8Ñ&8¸Õ&;¸QÀÓBÐBr$   c                ó¬  <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[S[ P                  ,          3,          S[S[ P                  S[S[ P                  ,          S[S[ P                  R	3,          3,          ,          R,          /# ©
r   Úhidden_statesÚ
layer_pastNrŽ   r:   Ú	use_cacheÚoutput_attentionsr   .©r   ÚFloatTensorr   Ú
LongTensorÚboolÚtupler    )r!   Ú__classdict__s   "€r"   r#   ÚGPTJAttention.__annotate__¨   sÈ   ø€ ÷ 9)ñ 9)á×(Ñ(ð9)ñ ˜D•Lð9)ñ ×)Ñ)¨DÕ0ð	9)ñ
 ×&Ñ&¨Õ-ð9)ñ ˜$•;ð9)ñ   $�;ð9)ñ 	‰e�l‰l™E¡%§,¡,Õ/Ð/Õ0Ù
‘—‘™e¡E§L¡LÕ1±5¹¿¹ÀsÐ9JÕ3KÐKÕ
Lõ	Mà
õ	ñ9)r$   c                ó¦  € V P                  V4      pV P                  V4      p	V P                  V4      p
V P                  W€P                  V P
                  R 4      pV P                  W�P                  V P
                  R 4      p	V P                  W P                  V P
                  R4      p
V P                  V4      pVP                  R4      P                  ^^VP                  R,          4      p\        P                  ! V^V4      P                  V	P                  4      p\        P                  ! WÝP                  R,          ^,          RR7      w  rïV P                  e²   V	RRRRV P                  13,          pV	RRRV P                  R13,          pVRRRRV P                  13,          pVRRRV P                  R13,          p\!        VWï4      p\!        VWï4      p\        P"                  ! VV.RR7      p	\        P"                  ! VV.RR7      pM\!        WžV4      p	\!        WŽV4      pV	P%                  ^ ^^^4      p	VP%                  ^ ^^^4      pVe   VP'                  WšV P(                  4      w  ršV P+                  W‰W£4      w  ppV P-                  VV P                  V P
                  4      pV P/                  V4      pV P1                  V4      pVV3# )TFr'   Nr@   rB   )ro   rm   rn   r   rf   rg   r”   Ú	unsqueezer7   r8   r   Úgatherr5   r&   Úsplitrq   rM   r,   r{   Úupdater_   r‘   rƒ   rp   r]   )rt   r˜   r™   rŽ   r:   rš   r›   Úkwargsr‹   rŒ   r�   r9   Úrepeated_position_idsÚsincosr-   r.   Úk_rotÚk_passÚq_rotÚq_passr�   r�   s   &&&&&&&,              r"   ÚforwardÚGPTJAttention.forward¨   s|  € ð —‘˜MÓ*ˆØ�k‰k˜-Ó(ˆØ—‘˜MÓ*ˆà×!Ñ! %×)AÑ)AÀ4Ç=Á=ÐRVÓWˆØ×Ñ ×%=Ñ%=¸t¿}¹}ÈdÓSˆØ×!Ñ! %×)AÑ)AÀ4Ç=Á=ÐRWÓXˆà×3Ñ3°LÓAˆà ,× 6Ñ 6°rÓ :× AÑ AÀ!ÀQÈ×H]ÑH]Ð^`ÕHaÓ bÐÜ—’˜o¨qÐ2GÓH×KÑKÈCÏIÉIÓVˆÜ—;’;˜v§|¡|°BÕ'7¸1Õ'<À"ÔE‰ˆà�?‰?Ò&Ø˜˜1˜aÐ!2 4§?¡?Ð!2Ð2Õ3ˆEØ˜˜A˜q $§/¡/Ñ"3Ð3Õ4ˆFà˜!˜Q Ð#4 T§_¡_Ð#4Ð4Õ5ˆEØ˜1˜a  D§O¡OÑ$5Ð5Õ6ˆFä(¨°Ó9ˆEÜ(¨°Ó9ˆEä—)’)˜U F˜O°Ô4ˆCÜ—I’I˜u f˜o°2Ô6‰Eä& s°Ó5ˆCÜ(¨°SÓ9ˆEà�k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆàÒ!Ø#×*Ñ*¨3°t·~±~ÓF‰JˆCð %)§J¡J¨u¸5Ó$QÑ!ˆ�\à×'Ñ'¨°T×5MÑ5MÈtÏ}É}Ó]ˆØ—m‘m KÓ0ˆØ×(Ñ(¨Ó5ˆà˜LÐ(Ð(r$   )r[   rV   re   r9   rg   r^   rm   r_   rX   rf   rp   rr   ro   r]   rq   rk   rn   ©N©NNNFF)rc   Ú
__module__Ú__qualname__Ú__firstlineno__rU   r   rƒ   r‘   r”   r¯   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rb   r¡   s   @@r"   rO   rO   F   s0   ù‡ € ÷#
òJhò&ô)ò2C÷9)÷ 9)ò 9)r$   rO   c                   óL   a a€ ] tR t^ät oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚGPTJFlashAttention2a4  
GPTJ flash attention module. This module inherits from `GPTJAttention` as the weights of the module stays
untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
flash attention and deal with padding tokens in case the input contains any of them.
c                óF   <€ \         SV `  ! V/ VB  \        4       V n        R # r±   )rT   rU   r   Ú_flash_attn_uses_top_left_mask)rt   Úargsr¨   rb   s   &*,€r"   rU   ÚGPTJFlashAttention2.__init__ë   s#   ø€ Ü‰Ò˜$Ð) &Ò)ô
 /PÓ.QˆÖ+r$   c                ó¬  <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[S[ P                  ,          3,          S[S[ P                  S[S[ P                  ,          S[S[ P                  R	3,          3,          ,          R,          /# r—   rœ   )r!   r¡   s   "€r"   r#   Ú GPTJFlashAttention2.__annotate__ó   sÑ   ø€ ÷ o)ñ o)á×(Ñ(ðo)ñ ˜D•Lðo)ñ ×)Ñ)¨DÕ0ð	o)ñ
 ×&Ñ&¨Õ-ðo)ñ ˜$•;ðo)ñ   $�;ðo)ñ 	‰e�l‰l™E¡%§,¡,Õ/Ð/Õ0Ù
‘—‘™e¡E§L¡LÕ1±5¹¿¹ÀsÐ9JÕ3KÐKÕ
Lõ	Mà
õ	ño)r$   c                óÊ	  € V P                  V4      pV P                  V4      p	V P                  V4      p
V P                  W€P                  V P
                  R 4      pV P                  W�P                  V P
                  R 4      p	V P                  W P                  V P
                  R4      p
V P                  V4      pVP                  R4      P                  ^^VP                  R,          4      p\        P                  ! V^V4      P                  V	P                  4      p\        P                  ! WÝP                  R,          ^,          RR7      w  rïV P                  e²   V	RRRRV P                  13,          pV	RRRV P                  R13,          pVRRRRV P                  13,          pVRRRV P                  R13,          p\!        VWï4      p\!        VWï4      p\        P"                  ! VV.RR7      p	\        P"                  ! VV.RR7      pM\!        WžV4      p	\!        WŽV4      pV	P%                  ^ ^^^4      p	VP%                  ^ ^^^4      pVe   VP'                  WšV P(                  4      w  ršV	P%                  ^ ^^^4      P+                  4       p	VP%                  ^ ^^^4      P+                  4       pV
P%                  ^ ^^^4      P+                  4       p
VP                  pVP,                  P.                  R8w  d   VP,                  P.                  MRpV\        P0                  8X  dÓ   \        P2                  ! V4      '       d   \        P4                  ! V4      pMS\7        V P8                  R4      '       d   V P8                  P                  pM V P                   P:                  P                  p\<        P?                  RV R	24       VP                  V4      pV	P                  V4      p	V
P                  V4      p
V P@                  '       d   V P8                  PB                  MR
pVP                  ^,          p\E        VV	V
VVVV PF                  V PH                  R7      pVPK                  VP                  ^ ,          VP                  ^,          VP                  ^,          VP                  ^,          ,          4      pV PM                  V4      pV PO                  V4      pVV3# )TFr'   Nr@   ÚmpsÚcpuÚ_is_quantizedz¾The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in Ú.g        )Údropoutr^   Úuse_top_left_maskrB   )(ro   rm   rn   r   rf   rg   r”   r¤   r7   r8   r   r¥   r5   r&   r¦   rq   rM   r,   r{   r§   r_   r‚   r6   Útyper†   Úis_autocast_enabledÚget_autocast_dtypeÚhasattrrV   Úweightr`   ra   ÚtrainingrZ   r   r^   r½   Úreshaperp   r]   )rt   r˜   r™   rŽ   r:   rš   r›   r¨   r‹   rŒ   r�   r9   r©   rª   r-   r.   r«   r¬   r­   r®   Úinput_dtypeÚdevice_typeÚtarget_dtypeÚattention_dropoutÚquery_lengthr�   r�   s   &&&&&&&,                   r"   r¯   ÚGPTJFlashAttention2.forwardó   s!  € ð —‘˜MÓ*ˆØ�k‰k˜-Ó(ˆØ—‘˜MÓ*ˆà×!Ñ! %×)AÑ)AÀ4Ç=Á=ÐRVÓWˆØ×Ñ ×%=Ñ%=¸t¿}¹}ÈdÓSˆØ×!Ñ! %×)AÑ)AÀ4Ç=Á=ÐRWÓXˆà×3Ñ3°LÓAˆà ,× 6Ñ 6°rÓ :× AÑ AÀ!ÀQÈ×H]ÑH]Ð^`ÕHaÓ bÐÜ—’˜o¨qÐ2GÓH×KÑKÈCÏIÉIÓVˆÜ—;’;˜v§|¡|°BÕ'7¸1Õ'<À"ÔE‰ˆà�?‰?Ò&Ø˜˜1˜aÐ!2 4§?¡?Ð!2Ð2Õ3ˆEØ˜˜A˜q $§/¡/Ñ"3Ð3Õ4ˆFà˜!˜Q Ð#4 T§_¡_Ð#4Ð4Õ5ˆEØ˜1˜a  D§O¡OÑ$5Ð5Õ6ˆFä(¨°Ó9ˆEÜ(¨°Ó9ˆEä—)’)˜U F˜O°Ô4ˆCÜ—I’I˜u f˜o°2Ô6‰Eä& s°Ó5ˆCÜ(¨°SÓ9ˆEð
 �k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆð Ò!Ø#×*Ñ*¨3°t·~±~ÓF‰JˆCð �k‰k˜!˜Q  1Ó%×0Ñ0Ó2ˆØ—‘˜a  A qÓ)×4Ñ4Ó6ˆØ—‘˜a  A qÓ)×4Ñ4Ó6ˆð —k‘kˆØ+0¯<©<×+<Ñ+<ÀÔ+E�e—l‘l×'Ò'È5ˆØœ%Ÿ-™-Ô'Ü×(Ò(¨×5Ò5Ü$×7Ò7¸ÓD‘ä˜Ÿ™ o×6Ò6Ø#Ÿ{™{×0Ñ0‘à#Ÿ{™{×1Ñ1×7Ñ7�ä×Ñðà �> ð$ôð —H‘H˜\Ó*ˆEØ—&‘&˜Ó&ˆCØ—H‘H˜\Ó*ˆEà6:·m·m°m˜DŸK™K×2Ò2ÈÐà—{‘{ 1•~ˆô 0ØØØØØØ%Ø—n‘nØ"×AÑAô	
ˆð #×*Ñ*Ø×Ñ˜qÕ! <×#5Ñ#5°aÕ#8¸,×:LÑ:LÈQÕ:OÐR^×RdÑRdÐefÕRgÕ:gó
ˆð —m‘m KÓ0ˆØ×(Ñ(¨Ó5ˆØ˜LÐ(Ð(r$   )r½   r²   )
rc   r³   r´   rµ   Ú__doc__rU   r¯   r¶   r·   r¸   r¹   s   @@r"   r»   r»   ä   s"   ù‡ € ñõR÷o)÷ o)ò o)r$   r»   ÚeagerÚflash_attention_2c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚGPTJMLPik  c                ó*  <€ \         SV `  4        VP                  p\        P                  ! W14      V n        \        P                  ! W4      V n        \        VP                  ,          V n	        \        P                  ! VP                  4      V n        R # r±   )rT   rU   Ún_embdr   rl   Úfc_inÚfc_outr   Úactivation_functionÚactrY   r\   rÇ   )rt   Úintermediate_sizerV   re   rb   s   &&& €r"   rU   ÚGPTJMLP.__init__l  s`   ø€ Ü‰ÑÔØ—M‘Mˆ	ä—Y’Y˜yÓ<ˆŒ
Ü—i’iÐ 1Ó=ˆŒä˜&×4Ñ4Õ5ˆŒÜ—z’z &×"4Ñ"4Ó5ˆŽr$   c                ó\   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  /# )r   r˜   Nr   )r   r�   )r!   r¡   s   "€r"   r#   ÚGPTJMLP.__annotate__v  s,   ø€ ÷ ñ ¡U×%6Ñ%6¸Õ%=ð Á%×BSÑBSñ r$   c                óŽ   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV# r±   )rÝ   rà   rÞ   rÇ   )rt   r˜   s   &&r"   r¯   ÚGPTJMLP.forwardv  s@   € ØŸ
™
 =Ó1ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆØŸ™ ]Ó3ˆØÐr$   )rà   rÇ   rÝ   rÞ   ©	rc   r³   r´   rµ   rU   r¯   r¶   r·   r¸   r¹   s   @@r"   rÚ   rÚ   k  s   ù‡ € õ6÷÷ ð r$   rÚ   c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )Ú	GPTJBlocki~  c                óD  <€ \         SV `  4        VP                  e   VP                  M^VP                  ,          p\        P
                  ! VP                  VP                  R7      V n        \        VP                  ,          ! W4      V n
        \        W14      V n        R # )N©Úeps)rT   rU   Ún_innerrÜ   r   Ú	LayerNormÚlayer_norm_epsilonÚln_1ÚGPTJ_ATTENTION_CLASSESÚ_attn_implementationÚattnrÚ   Úmlp)rt   rV   r_   Ú	inner_dimrb   s   &&& €r"   rU   ÚGPTJBlock.__init__  sj   ø€ Ü‰ÑÔØ&,§n¡nÒ&@�F—N’NÀaÈ&Ï-É-ÕFWˆ	Ü—L’L §¡°F×4MÑ4MÔNˆŒ	Ü*¨6×+FÑ+FÖGÈÓZˆŒ	Ü˜9Ó-ˆŽr$   c                óh  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  ,          S[S[ P                  S[S[ P                  R	3,          3,          ,          R,          /# )
r   r˜   Nr™   rŽ   r:   rš   r›   r   .rœ   )r!   r¡   s   "€r"   r#   ÚGPTJBlock.__annotate__†  s°   ø€ ÷ +ñ +á×(Ñ(¨4Õ/ð+ñ ˜D•Lð+ñ ×)Ñ)¨DÕ0ð	+ñ
 ×&Ñ&¨Õ-ð+ñ ˜$•;ð+ñ   $�;ð+ñ 
‰u�|‰|Õ	™u¡U§\¡\±5¹×9JÑ9JÈCÐ9OÕ3PÐ%PÕQÕ	QÐTXÕ	Xñ+r$   c           	     ó    € TpV P                  V4      pV P                  VVVVVVR 7      w  ršV P                  V4      pW›,           V,           pW3# ))r˜   r™   rŽ   r:   rš   r›   )rð   ró   rô   )rt   r˜   r™   rŽ   r:   rš   r›   r¨   ÚresidualÚattn_outputsr�   Úfeed_forward_hidden_statess   &&&&&&&,    r"   r¯   ÚGPTJBlock.forward†  sf   € ð !ˆØŸ	™	 -Ó0ˆØ%)§Y¡YØ'Ø!Ø)Ø%ØØ/ð &/ó &
Ñ"ˆð &*§X¡X¨mÓ%<Ð"Ø$ÕAÀHÕLˆàÐ*Ð*r$   )ró   rð   rô   r±   r²   rç   r¹   s   @@r"   ré   ré   ~  s   ù‡ € ÷.÷+÷ +ò +r$   ré   c                   óZ   a a€ ] tR tRt oRtRtR.tR.tRtRt	V 3R lt
V3R ltRtVtV ;t# )	ÚGPTJPreTrainedModeli   ÚtransformerTré   Úpast_key_valuesc                óÔ   <€ \         SV `  V4       \        V\        4      '       dB   \        P
                  ! VP                  \        VP                  VP                  4      4       R # R # r±   )
rT   Ú_init_weightsÚ
isinstancerO   ÚinitÚcopy_r9   r1   rX   rr   )rt   Úmodulerb   s   &&€r"   r  Ú!GPTJPreTrainedModel._init_weightsª  sK   ø€ Ü‰Ñ˜fÔ%Ü�fœm×,Ò,Ü�JŠJ�v×-Ñ-Ô/JÈ6×K_ÑK_Ðag×atÑatÓ/uÖvñ -r$   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )r   rV   r   )r!   r¡   s   "€r"   r#   Ú GPTJPreTrainedModel.__annotate__   s   ø‡ ‚ áÑò r$   © )rc   r³   r´   rµ   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_can_compile_fullgraphr  Ú__annotate_func__r¶   r·   r¸   r¹   s   @@r"   rÿ   rÿ      s=   ù‡ € ð &ÐØ&*Ð#Ø$˜ÐØ#4Ð"5ÐØÐØ!Ðõw÷ … r$   rÿ   c                   ó^   a a€ ] tR tRt oV 3R ltR tR t]RV3R lR ll4       tRt	Vt
V ;t# )	Ú	GPTJModeli°  c           
     ó>  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  V P                  4      V n        \
        P                  ! VP                  4      V n
        \
        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        \
        P                   ! V P                  VP"                  R7      V n        RV n        V P)                  4        R# u upi ))r_   rë   FN)rT   rU   rÜ   re   Ú
vocab_sizer   Ú	EmbeddingÚwterY   Ú
embd_pdropÚdropÚ
ModuleListÚrangeÚn_layerré   Úhrî   rï   Úln_fÚgradient_checkpointingÚ	post_init)rt   rV   Úirb   s   && €r"   rU   ÚGPTJModel.__init__²  s¿   ø€ Ü‰Ñ˜Ô àŸ™ˆŒØ ×+Ñ+ˆŒÜ—<’< × 1Ñ 1°4·>±>ÓBˆŒÜ—J’J˜v×0Ñ0Ó1ˆŒ	Ü—’ÌÈfÏnÉnÔH]Ó^ÑH]À1¤	¨&× >ÑH]Ñ^Ó_ˆŒÜ—L’L §¡°V×5NÑ5NÔOˆŒ	à&+ˆÔ#ð 	�‰Öùò  _s   Â/Dc                ó   € V P                   # r±   ©r  )rt   s   &r"   Úget_input_embeddingsÚGPTJModel.get_input_embeddingsÁ  s   € Ø�x‰xˆr$   c                ó   € Wn         R # r±   r%  )rt   Únew_embeddingss   &&r"   Úset_input_embeddingsÚGPTJModel.set_input_embeddingsÄ  s   € Ø!Žr$   c                ó\  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[R,          RS[S[,          /# )r   Ú	input_idsNr  rŽ   Útoken_type_idsr:   Úinputs_embedsrš   r›   Úoutput_hidden_statesÚreturn_dictr   )r   rž   r   r�   rŸ   r    r   )r!   r¡   s   "€r"   r#   ÚGPTJModel.__annotate__È  s×   ø€ ÷ g
ñ g
á×#Ñ# dÕ*ðg
ñ  �ðg
ñ ×)Ñ)¨DÕ0ð	g
ñ
 ×(Ñ(¨4Õ/ðg
ñ ×&Ñ&¨Õ-ðg
ñ ×(Ñ(¨4Õ/ðg
ñ ˜$•;ðg
ñ   $�;ðg
ñ # T�kðg
ñ ˜D•[ðg
ñ 
Ñ(Õ	(ñg
r$   c           
     ó®  € Ve   TMV P                   P                  pV	e   T	MV P                   P                  p	Ve   TMV P                   P                  pV
e   T
MV P                   P                  p
VRJ VRJ,          '       d   \        R4      hV P                  '       d2   V P                  '       d    V'       d   \        P                  R4       RpVf   V P                  V4      pV'       d   Vf   \        V P                   R7      pVP                  ^,          pVfP   Ve   VP                  4       M^ p\        P                  ! WÆP                   R7      V,           pVP#                  ^ 4      p\%        V P                   VVVVR7      pTpVe-   VP'                  RV4      pV P                  V4      pVV,           pV P)                  V4      pRWÏP+                  R4      3pV'       d   RMRpV	'       d   RMRp\-        V P.                  4       FJ  w  ppV	'       d   VV3,           pV! VVVVVVR7      pV^ ,          pV'       g   K9  VV^,          3,           pKL  	  V P1                  V4      pVP'                  V4      pV	'       d   VV3,           pV
'       g3   \2        ;QJ d    . R	 WòVV3 4       F  NK  	  5# ! R	 WòVV3 4       4      # \5        VVVVR
7      # )áu  
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
    model's internal embedding lookup matrix.
Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rV   ©r6   )rV   r/  rŽ   r  r:   )r™   rŽ   r:   rš   r›   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5ir±   r  )Ú.0Úvs   & r"   Ú	<genexpr>Ú$GPTJModel.forward.<locals>.<genexpr>&  s   é € ð Ùc�a—’Ócùs   ‚Œ
)Úlast_hidden_stater  r˜   Ú
attentionsrB   r  )rV   r›   r0  rš   r1  rh   r   rÎ   r`   ra   r  r	   r8   Úget_seq_lengthr   r(   r6   r¤   r   ry   r  rx   Ú	enumerater  r  r    r   )rt   r-  r  rŽ   r.  r:   r/  rš   r›   r0  r1  r¨   Ú
seq_lengthÚpast_key_values_lengthÚcausal_maskr˜   Útoken_type_embedsÚoutput_shapeÚall_self_attentionsÚall_hidden_statesr"  ÚblockÚoutputss   &&&&&&&&&&&,           r"   r¯   ÚGPTJModel.forwardÇ  s¿  € ð* 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆà˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZà×&×&Ð&¨4¯=¯=¨=ßÜ×#Ñ#Øpôð "�	àÒ Ø ŸH™H YÓ/ˆMç˜Ò0Ü*°$·+±+Ô>ˆOà"×(Ñ(¨Õ+ˆ
ØÒØIXÒId _×%CÑ%CÔ%EÐjkÐ"Ü Ÿ<š<¨
×;OÑ;OÔPÐSiÕiˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆàÒ%Ø+×0Ñ0°°ZÓ@ˆNØ $§¡¨Ó 8ÐØ)Ð,=Õ=ˆMàŸ	™	 -Ó0ˆØ˜J×(:Ñ(:¸2Ó(>Ð?ˆç$5™b¸4Ðß"6™B¸DÐÜ! $§&¡&Ö)‰HˆAˆuß#Ø$5¸Ð8HÕ$HÐ!áØØ*Ø*Ø)Ø#Ø"3ôˆGð $ A�JˆMß Ñ Ø&9¸WÀQ½Z¸MÕ&IÒ#ñ *ð" Ÿ	™	 -Ó0ˆà%×*Ñ*¨<Ó8ˆçØ 1°]Ð4DÕ DÐçß”5ñ Ø)Ð<MÐObÑcó—5ð �5ñ Ø)Ð<MÐObÑcóó ð ô 'Ø+Ø+Ø+Ø*ô	
ð 	
r$   )r  re   r   r  r  r  r  ©
NNNNNNNNNN)rc   r³   r´   rµ   rU   r&  r*  r   r¯   r¶   r·   r¸   r¹   s   @@r"   r  r  °  s6   ù‡ € õòò"ð ÷g
ñ g
ó ÷g
ð g
r$   r  zK
    The GPT-J Model transformer with a language modeling head on top.
    )Úcustom_introc                   óZ   a a€ ] tR tRt oRR/tV 3R lt]RV3R lR ll4       tRtVt	V ;t
# )	ÚGPTJForCausalLMi2  zlm_head.weightztransformer.wte.weightc                óÆ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  4      V n        V P                  4        R # r±   )
rT   rU   r  r   r   rl   rÜ   r  Úlm_headr!  ©rt   rV   rb   s   &&€r"   rU   ÚGPTJForCausalLM.__init__:  sE   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆÔÜ—y’y §¡°×0AÑ0AÓBˆŒð 	�‰Ör$   c                ó®  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[R,          RS[R,          RS[S[ P                  ,          RS[S[,          /# )r   r-  Nr  rŽ   r.  r:   r/  Úlabelsrš   r›   r0  r1  Úlogits_to_keepr   )	r   rž   r   r�   rŸ   r   r    r    r   )r!   r¡   s   "€r"   r#   ÚGPTJForCausalLM.__annotate__C  sò   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ  �ð<
ñ ×)Ñ)¨DÕ0ð	<
ñ
 ×(Ñ(¨4Õ/ð<
ñ ×&Ñ&¨Õ-ð<
ñ ×(Ñ(¨4Õ/ð<
ñ × Ñ  4Õ'ð<
ñ ˜$•;ð<
ñ   $�;ð<
ñ # T�kð<
ñ ˜D•[ð<
ñ ™eŸl™lÕ*ð<
ñ 
Ñ'Õ	'ñ<
r$   c                ó  € Ve   TMV P                   P                  pV P                  VVVVVVVV	V
VR7
      pV^ ,          p\        V\        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! R	RVRVRV P                   P                  /VB pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  VP                  R7      # )
aÿ  
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
    `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
    are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
N©	r  rŽ   r.  r:   r/  rš   r›   r0  r1  r@   ÚlogitsrR  r  ºrA   NN©ÚlossrW  r  r˜   r<  r  )rV   r1  r   r  r   ÚslicerN  Úloss_functionr  r   r  r˜   r<  )rt   r-  r  rŽ   r.  r:   r/  rR  rš   r›   r0  r1  rS  r¨   Útransformer_outputsr˜   Úslice_indicesrW  rZ  Úoutputs   &&&&&&&&&&&&&,      r"   r¯   ÚGPTJForCausalLM.forwardB  s$  € ð6 &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø)Ø%Ø'ØØ/Ø!5Ø#ð /ó 
Ðð ,¨AÕ.ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDçØ�YÐ!4°RÕ!8Õ8ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä%ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r$   )rN  r   )NNNNNNNNNNNr4   )rc   r³   r´   rµ   Ú_tied_weights_keysrU   r   r¯   r¶   r·   r¸   r¹   s   @@r"   rL  rL  2  s6   ù‡ € ð +Ð,DÐEÐõð ÷<
ñ <
ó ÷<
ð <
r$   rL  aã  
    The GPT-J Model transformer with a sequence classification head on top (linear layer).

    [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT, GPT-2, GPT-Neo) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚGPTJForSequenceClassificationi‚  c                óì   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  V P                  R R7      V n        V P                  4        R# )FrQ   N)
rT   rU   Ú
num_labelsr  r   r   rl   rÜ   Úscorer!  rO  s   &&€r"   rU   Ú&GPTJForSequenceClassification.__init__‘  sR   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ$ VÓ,ˆÔÜ—Y’Y˜vŸ}™}¨d¯o©oÀEÔJˆŒ
ð 	�‰Ör$   c                ó„  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[R,          RS[R,          RS[S[,          /# )r   r-  Nr  rŽ   r.  r:   r/  rR  rš   r›   r0  r1  r   )r   rž   r   r�   rŸ   r    r   )r!   r¡   s   "€r"   r#   Ú*GPTJForSequenceClassification.__annotate__›  sí   ø€ ÷ b
ñ b
á×#Ñ# dÕ*ðb
ñ  �ðb
ñ ×)Ñ)¨DÕ0ð	b
ñ
 ×(Ñ(¨4Õ/ðb
ñ ×&Ñ&¨Õ-ðb
ñ ×(Ñ(¨4Õ/ðb
ñ × Ñ  4Õ'ðb
ñ ˜$•;ðb
ñ   $�;ðb
ñ # T�kðb
ñ ˜D•[ðb
ñ 
Ñ1Õ	1ñb
r$   c                ó˜  € Ve   TMV P                   P                  pV P                  VVVVVVVV	V
VR7
      pV^ ,          pV P                  V4      pVe   VP                  ^ ,          pMVP                  ^ ,          pV P                   P
                  f   V^8w  d   \        R4      hV P                   P
                  f   RpMÍVeœ   WP                   P
                  8g  P                  VP                  \        P                  4      p\        P                  ! VP                  R,          VP                  \        P                  R7      pVV,          P                  R4      pM.Rp\        P                  V P                  P                    R24       V\        P                  ! VVP                  R7      V3,          pRpVEeµ   VP                  VP                  4      pV P                   P"                  f–   V P$                  ^8X  d   RV P                   n        MrV P$                  ^8”  dQ   VP&                  \        P(                  8X  g    VP&                  \        P*                  8X  d   RV P                   n        MR	V P                   n        V P                   P"                  R8X  dL   \-        4       pV P$                  ^8X  d'   V! VP/                  4       VP/                  4       4      pM�V! VV4      pM†V P                   P"                  R8X  d>   \1        4       pV! VP3                  RV P$                  4      VP3                  R4      4      pM.V P                   P"                  R	8X  d   \5        4       pV! VV4      pV'       g"   V3VR
,          ,           pVe   V3V,           # T# \7        VVVP8                  VP:                  VP<                  R7      # )aÙ  
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
NrV  z=Cannot handle batch sizes > 1 if no padding token is defined.)r6   r&   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r5  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrX  rY  rB   )rV   r1  r   rf  r8   Úpad_token_idrh   r5   r6   r   Úint32r(   Úargmaxr`   ra   rb   rc   Úproblem_typere  r&   Úlongr   r   Úsqueezer   ry   r   r   r  r˜   r<  )rt   r-  r  rŽ   r.  r:   r/  rR  rš   r›   r0  r1  r¨   r]  r˜   rW  Ú
batch_sizeÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsrZ  Úloss_fctr_  s   &&&&&&&&&&&&,           r"   r¯   Ú%GPTJForSequenceClassification.forwardš  s  € ð4 &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø)Ø%Ø'ØØ/Ø!5Ø#ð /ó 
Ðð ,¨AÕ.ˆØ—‘˜MÓ*ˆàÒ Ø"Ÿ™¨Õ+‰Jà&×,Ñ,¨QÕ/ˆJà�;‰;×#Ñ#Ò+°
¸a´ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ò+Ø!#ÑØÒ"à%¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸLšL¨¯©¸Õ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Õ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¸v¿}¹}ÔMÐOaÐaÕbˆàˆØÓØ—Y‘Y˜}×3Ñ3Ó4ˆFØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù × 2Ñ 2°2°t·±Ó GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù ¨vÓ6�ßØ#Ð%Ð(;¸BÕ(?Õ?ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r$   )re  rf  r   )NNNNNNNNNNN©
rc   r³   r´   rµ   rU   r   r¯   r¶   r·   r¸   r¹   s   @@r"   rc  rc  ‚  s,   ù‡ € õð ÷b
ñ b
ó ÷b
ð b
r$   rc  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚGPTJForQuestionAnsweringi   c                óè   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # r±   )
rT   rU   re  r  r   r   rl   rd   Ú
qa_outputsr!  rO  s   &&€r"   rU   Ú!GPTJForQuestionAnswering.__init__  sT   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ$ VÓ,ˆÔÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör$   c                ó„  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[R,          RS[S[,          /# )r   r-  NrŽ   r.  r:   r/  Ústart_positionsÚend_positionsr›   r0  r1  r   )r   rž   r�   rŸ   r    r   )r!   r¡   s   "€r"   r#   Ú%GPTJForQuestionAnswering.__annotate__  så   ø€ ÷ C
ñ C
á×#Ñ# dÕ*ðC
ñ ×)Ñ)¨DÕ0ðC
ñ ×(Ñ(¨4Õ/ð	C
ñ
 ×&Ñ&¨Õ-ðC
ñ ×(Ñ(¨4Õ/ðC
ñ ×)Ñ)¨DÕ0ðC
ñ ×'Ñ'¨$Õ.ðC
ñ   $�;ðC
ñ # T�kðC
ñ ˜D•[ðC
ñ 
Ñ-Õ	-ñC
r$   c                óÜ  € V
e   T
MV P                   P                  p
V P                  VVVVVVV	V
R7      pV^ ,          pV P                  V4      pVP	                  ^RR7      w  ppVP                  R4      P                  4       pVP                  R4      P                  4       pRpVe÷   Veó   \        VP                  4       4      ^8”  d+   VP                  R4      P                  VP                  4      p\        VP                  4       4      ^8”  d+   VP                  R4      P                  VP                  4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! Wö4      pV! VV4      pVV,           ^,          pV
'       g#   VV3VR,          ,           pVe   V3V,           # T# \        VVVVP                  VP                  R7      # )r4  N)rŽ   r.  r:   r/  r›   r0  r1  r'   )Úignore_index:r   NN)rZ  Ústart_logitsÚ
end_logitsr˜   r<  rB   )rV   r1  r   r  r¦   rs  r‚   rz   rx   r5   r6   Úclampr   r   r˜   r<  )rt   r-  rŽ   r.  r:   r/  r‚  rƒ  r›   r0  r1  r¨   rG  Úsequence_outputrW  r‡  rˆ  Ú
total_lossÚignored_indexry  Ú
start_lossÚend_lossr_  s   &&&&&&&&&&&,           r"   r¯   Ú GPTJForQuestionAnswering.forward  sß  € ð* &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà×"Ñ"ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð #ó 	
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=×"@Ñ"@À×ATÑATÓ"U�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9× <Ñ <¸Z×=NÑ=NÓ O�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÕ/°1Õ4ˆJçØ" JÐ/°'¸"µ+Õ=ˆFØ/9Ò/E�Z�M FÕ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r$   )re  r  r   rI  r{  r¹   s   @@r"   r}  r}     s,   ù‡ € õð ÷C
ñ C
ó ÷C
ð C
r$   r}  )rL  r}  rc  r  rÿ   )9rÖ   ri   r   r   Útorch.nnr   r   r   Ú r   r  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_gptjr   r   Ú
get_loggerrc   r`   r1   r;   rH   rM   ÚModulerO   r»   rñ   rÚ   ré   rÿ   r  rL  rc  r}  Ú__all__r  r$   r"   Ú<module>rŸ     s‰  ðñ ã ã Ý ß AÑ Aå &Ý !ß .Ý )Ý /ß hÝ 9÷ó õ .ß ,Ý *ñ ×ÒÝJð 
×	Ò	˜HÓ	%€õPòWõõ=ô[)�B—I‘Iô [)ô|~)˜-ô ~)ðD ˆ]ØÐ,ðÐ ôˆb�i‰iô ô&+Ð*ô +ðD ôw˜/ó wó ðwð ô~
Ð#ó ~
ó ð~
ñB ðôô
H
Ð)¨?ó H
óð
H
ñV ðôôm
Ð$7ó m
óðm
ð` ôN
Ð2ó N
ó ðN
òb‚r$   