+
    œ°³j0}  ã                   ód  € R t ^ RIt^ RIHt ^ RIt^ RIHt ^RIHt ^RI	H
t
 ^RIHt ^RIHt ^RIHtHtHtHtHtHtHt ^R	IHt ]P2                  ! ]4      tRsR
 t ! R R]P<                  P>                  4      t R#R lt!R#R lt" ! R R]PF                  4      t$ ! R R]PF                  4      t% ! R R]4      t&] ! R R]4      4       t']! RR7      ] ! R R]4      4       4       t(]! RR7      ] ! R R]4      4       4       t)] ! R R]'4      4       t*]! R R7       ! R! R"]']
4      4       t+. R$Ot,R# )%zPyTorch RWKV model.N)Ú	dataclass)Únn)Úinitialization)ÚGenerationMixin)ÚGradientCheckpointingLayer)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚis_bitsandbytes_availableÚis_kernels_availableÚis_ninja_availableÚis_torch_cuda_availableÚlogging©Ú
RwkvConfigc                 ór   € \        4       '       g   \        R 4      h^RIHp V! R^R7      sV \        n        R# )zFkernels is not installed, please install it with `pip install kernels`)Ú
get_kernelzkernels-community/rwkv)ÚversionN)r   ÚImportErrorÚintegrations.hub_kernelsr   Úrwkv_cuda_kernelÚmax_seq_length)Úcontext_lengthr   s   & Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/rwkv/modeling_rwkv.pyÚload_wkv_cuda_kernelr   -   s1   € ä×!Ò!ÜÐbÓcÐcå6á!Ð":ÀAÔFÐØ&4ÔÖ#ó    c                   óH   a € ] tR t^8t o ]RR l4       t]RR l4       tRtV tR# )ÚRwkvLinearAttentionNc                ó4  € VP                  4       w  rxp	V\        P                  8”  d!   \        R V R\        P                   R24      hWy,          \	        V	^ 4      ,          ^ 8w  d    \        RV RV	 R\	        V	^ 4       R24      hVP
                  V n        VP                  P                  R8w  gR   VP                  P                  R8w  g7   VP                  P                  R8w  g   VP                  P                  R8w  d   \        R4      h\        P                  ! VP                  4       P                  4       4      ) pVP
                  \        P                  8X  d1   VP                  4       pVP                  4       pVP                  4       pVP                  4       pVP                  4       pVP                  4       p\        P                  ! V\        P                  R	7      p
V'       g   Veì   VfY   \        P                   ! VV	^\        P"                  VP                  \        P                  R
7      pVR;;,          R,          uu&   MD\        P$                  ! V Uu. uF  q»P'                  ^4      NK  	  up^R7      P                  4       pVP
                  \        P(                  8X  d   \        P*                  pM\        P,                  pV! WW4W¥4       MIVP
                  \        P(                  8X  d   \        P.                  M\        P0                  pV! WW4V
4       V P3                  WW4V
4       Ve7   \        P4                  ! V^^R7       Uu. uF  q»P7                  ^4      NK  	  ppV
P9                  V P                  4      V3# u upi u upi )zCannot process a batch with z+ tokens at the same time, use a maximum of z with this model.zThe product of batch size (z) and hidden size (z") needs to be a round multiple of Ú.ÚcudazUCalling the CUDA kernel for wkv attention requires all tensors to be on CUDA devices.©Úmemory_format)ÚdtypeÚdevicer"   ç±¡*ÓÎÒG)Údim)ºNNNr'   é   )Úsizer   r   Ú
ValueErrorÚminr#   Úinput_dtyper$   ÚtypeÚtorchÚexpÚfloatÚ
contiguousÚfloat16Ú
empty_likeÚcontiguous_formatÚzerosÚfloat32ÚcatÚ	unsqueezeÚbfloat16Úforward_with_state_bf16Úforward_with_stateÚforward_bf16ÚforwardÚsave_for_backwardÚchunkÚsqueezeÚto)ÚctxÚ
time_decayÚ
time_firstÚkeyÚvalueÚstateÚreturn_stateÚ
batch_sizeÚseq_lenÚhidden_sizeÚoutputÚsÚforward_funcs   &&&&&&&      r   r=   ÚRwkvLinearAttention.forward9   s×  € à+.¯8©8«:Ñ(ˆ
˜[ØÔ%×4Ñ4Ô4ÜØ.¨w¨iÐ7bÜ#×2Ñ2Ð3Ð3DðFóð ð Õ#¤c¨+°rÓ&:Õ:¸aÔ?ÜØ-¨j¨\Ð9LÈ[ÈMð ZÜ" ;°Ó3Ð4°Að7óð ð
 Ÿ)™)ˆŒð ×Ñ×"Ñ" fÔ,Ø× Ñ ×%Ñ%¨Ô/Ø�z‰z�‰ &Ô(Ø�|‰|× Ñ  FÔ*äÐtÓuÐuä—i’i 
× 0Ñ 0Ó 2× =Ñ =Ó ?Ó@Ð@ˆ
Ø�9‰9œŸ™Ô%Ø#×)Ñ)Ó+ˆJØ—)‘)“+ˆCØ—K‘K“MˆEØ×*Ñ*Ó,ˆ
Ø�n‰nÓˆØ× Ñ Ó"ˆä×!Ò! #´U×5LÑ5LÔMˆß˜5Ò,ØŠ}ÜŸšØØØÜŸ-™-ØŸ:™:Ü"'×"9Ñ"9ô�ð �g— $Õ&”äŸ	š	¹5Ó"A¹5°a§;¡;¨q¦>¹5Ñ"AÀqÔI×TÑTÓV�Ø�y‰yœEŸN™NÔ*Ü/×GÑG‘ä/×BÑB�Ù˜°¸VÕKà<?¿I¹IÌÏÉÔ<WÔ+×8Ò8Ô]m×]uÑ]uˆLÙ˜°¸VÔDà×Ñ˜j°cÀ&ÔIàÒÜ+0¯;ª;°u¸aÀQÕ+GÓHÑ+G a—Y‘Y˜q–\Ñ+GˆEÐHà�y‰y˜Ÿ™Ó)¨5Ð0Ð0ùò #Bùò Is   É"NÍNc                óL  € V P                   pV P                  w  rErgp\        P                  ! T\        P                  V\        P
                  8X  d   \        P
                  M\        P                  R 7      p	\        P                  ! V\        P                  R7      p
\        P                  ! V\        P                  R7      p\        P                  ! V\        P                  R7      pV\        P                  8X  d   VP                  4       pV\        P
                  8X  d   \        P                  M\        P                  pV! VVVVVVP                  4       V	V
VV4
       V	P                  V4      V
P                  V4      VP                  V4      VP                  V4      RR3# ))r"   r#   r!   N)r,   Úsaved_tensorsr.   r3   r4   r9   r6   r2   r0   r   Úbackward_bf16Úbackwardr1   rA   )rB   Úg_outputÚg_stater,   rC   rD   rE   rF   rL   Úg_time_decayÚg_time_firstÚg_keyÚg_valueÚbackward_funcs   &&&           r   rS   ÚRwkvLinearAttention.backwardx   sF  € ð —o‘oˆà58×5FÑ5FÑ2ˆ
 ¨Fä×'Ò'ØÜ×1Ñ1Ø$/´5·>±>Ô$A”%—.’.ÄuÇ}Á}ô
ˆô
 ×'Ò'¨
Ä%×BYÑBYÔZˆÜ× Ò  ´E×4KÑ4KÔLˆÜ×"Ò" 5¼×8OÑ8OÔPˆàœ%Ÿ-™-Ô'Ø—~‘~Ó'ˆHØ:EÌÏÉÔ:WÔ(×6Ò6Ô]m×]vÑ]vˆÙØØØØØØ×ÑÓ!ØØØØô	
ð �O‰O˜KÓ(Ø�O‰O˜KÓ(Ø�H‰H�[Ó!Ø�J‰J�{Ó#ØØð
ð 	
r   © ©NF©N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr=   rS   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   r   r   8   s,   ø‡ € Øó<1ó ð<1ð| ó%
ó ö%
r   r   c                 ó¨  € VP                  4       w  rgp\        P                  ! V4      pVf�   \        P                  ! VR,          \        P                  R7      p	\        P                  ! VR,          \        P                  R7      p
\        P                  ! VR,          \        P                  R7      R,
          pMVw  ršp\        P                  ! V 4      ) p \        V4       EFX  pVRV3,          P                  4       pVRV3,          p\        P                  ! W½V,           4      p\        P                  ! W¿,
          4      p\        P                  ! WÑ,           V,
          4      pVV	,          VV,          ,           pVV
,          V,           pVV,          P                  VP                  4      VRV3&   \        P                  ! W°,           V4      p\        P                  ! W°,           V,
          4      p\        P                  ! VV,
          4      pVV	,          VV,          ,           p	VV
,          V,           p
TpEK[  	  V'       g   Ve   WšV.pW„3# )Nr'   )r#   r%   ©r'   é    )
r)   r.   Ú
zeros_liker6   r/   Úranger0   ÚmaximumrA   r#   )rC   rD   rE   rF   rG   rH   Ú_Ú
seq_lengthrL   Ú	num_stateÚ	den_stateÚ	max_stateÚcurrent_indexÚcurrent_keyÚcurrent_valueÚmax_for_outputÚe1Úe2Ú	numeratorÚdenominatorÚmax_for_states   &&&&&&               r   Úrwkv_linear_attention_cpur|   ¢   sÏ  € ð —x‘x“zÑ€A�1Ü×Ò˜cÓ"€Fà‚}Ü×$Ò$ S¨¥Y´e·m±mÔDˆ	Ü×$Ò$ S¨¥Y´e·m±mÔDˆ	Ü×$Ò$ S¨¥Y´e·m±mÔDÀtÕK‰	à*/Ñ'ˆ	˜iô
 —)’)˜JÓ'Ð'€Jä˜z×*ˆØ˜!˜]Ð*Õ+×1Ñ1Ó3ˆØ˜a Ð.Õ/ˆô Ÿš yÀ
Õ2JÓKˆÜ�YŠY�yÕ1Ó2ˆÜ�YŠY�{Õ/°.Õ@ÓAˆØ˜•N R¨-Õ%7Õ7ˆ	Ø˜9•n rÕ)ˆØ$-°Õ$;×#?Ñ#?ÀÇÁÓ#Mˆˆq�-ÐÑ ô Ÿš iÕ&<¸kÓJˆÜ�YŠY�yÕ-°Õ=Ó>ˆÜ�YŠY�{ ]Õ2Ó3ˆØ˜•N R¨-Õ%7Õ7ˆ	Ø˜•N RÕ'ˆ	Ø!‹	ñ% +÷( �uÒ(Ø yÐ1ˆàˆ=Ðr   c           	      ó  € \         ;QJ d    R  WW#3 4       F  '       g   K   RM	  RM! R  WW#3 4       4      pVP                  ^4      ^8H  p\        e   V'       g	   V'       d   \        WW#WER7      # \        P                  WW#WE4      # )c              3   óR   "  € T F  qP                   P                  R 8g  x € K  	  R# 5i)r    N)r$   r-   )Ú.0Úts   & r   Ú	<genexpr>Ú(rwkv_linear_attention.<locals>.<genexpr>Ï   s   é € ÐXÑ3W¨a—(‘(—-‘- 6Ö)Ó3Wùs   ‚%'TF©rG   rH   )Úanyr)   r   r|   r   Úapply)rC   rD   rE   rF   rG   rH   Úno_cudaÚ	one_tokens   &&&&&&  r   Úrwkv_linear_attentionrˆ   Î   so   € ß‹cÑX°JÈCÑ3WÓX�c�cŠcÑX°JÈCÑ3WÓXÓX€Gð —‘˜“˜qÑ €IÜÒ§7¯iÜ(¨ÀÐSXÔtÐtä"×(Ñ(¨ÀÈUÓaÐar   c                   óJ   a a€ ] tR t^Ùt oRV 3R lltRR ltRR ltRtVtV ;t	# )ÚRwkvSelfAttentionc                ó¼  <€ \         SV `  4        Wn        \        R J;'       d    \        P                  VP
                  8H  p\        4       '       d/   \        4       '       d   V'       g    \        VP
                  4       W n        VP                  pVP                  e   VP                  MTpWPn        \        P                   ! \"        P$                  ! V4      4      V n        \        P                   ! \"        P$                  ! V4      4      V n        \        P                   ! \"        P$                  ! ^^V4      4      V n        \        P                   ! \"        P$                  ! ^^V4      4      V n        \        P                   ! \"        P$                  ! ^^V4      4      V n        \        P0                  ! R4      V n        \        P4                  ! WERR7      V n        \        P4                  ! WERR7      V n        \        P4                  ! WERR7      V n        \        P4                  ! WTRR7      V n        R #   \         d    \        P                  R4        ELÙi ; i)Nz9Could not load the custom CUDA kernel for RWKV attention.F©Úbias©rj   rj   é   éÿÿÿÿ)ÚsuperÚ__init__Úconfigr   r   r   r   r   r   Ú	ExceptionÚloggerÚinfoÚlayer_idrK   Úattention_hidden_sizer   Ú	Parameterr.   ÚemptyrC   rD   Útime_mix_keyÚtime_mix_valueÚtime_mix_receptanceÚ	ZeroPad2dÚ
time_shiftÚLinearrE   rF   Ú
receptancerL   )Úselfr“   r—   Úkernel_loadedrK   r˜   Ú	__class__s   &&&   €r   r’   ÚRwkvSelfAttention.__init__Ú   s�  ø€ Ü‰ÑÔØŒÜ(°Ð4×qÐqÔ9I×9XÑ9XÐ\b×\qÑ\qÑ9qˆÜ×ÒÔ$;×$=Ò$=ÇmðYÜ$ V×%:Ñ%:Ô;ð !ŒØ×(Ñ(ˆà,2×,HÑ,HÒ,TˆF×(Ò(ÐZeð 	ð &;Ô"äŸ,š,¤u§{¢{Ð3HÓ'IÓJˆŒÜŸ,š,¤u§{¢{Ð3HÓ'IÓJˆŒäŸLšL¬¯ª°Q¸¸;Ó)GÓHˆÔÜ Ÿlšl¬5¯;ª;°q¸!¸[Ó+IÓJˆÔÜ#%§<¢<´·²¸A¸qÀ+Ó0NÓ#OˆÔ äŸ,š, }Ó5ˆŒÜ—9’9˜[ÀeÔLˆŒÜ—Y’Y˜{ÈÔNˆŒ
ÜŸ)š) KÈUÔSˆŒÜ—i’iÐ 5ÈÔOˆŽøô) ô YÜ—‘ÐW×XðYús   Á,H7 È7 IÉIc                óà  € VP                  ^4      ^8X  d#   Ve   V^,          RRV P                  3,          pM5V P                  V4      pVe!   V^,          RRV P                  3,          VR&   WP                  ,          V^V P                  ,
          ,          ,           pWP                  ,          V^V P                  ,
          ,          ,           pWP
                  ,          V^V P
                  ,
          ,          ,           pV P                  V4      pV P                  V4      p\        P                  ! V P                  V4      4      pVe!   VR,          V^,          RRV P                  3&   WdWR3# ©r�   r'   ri   )r'   r�   )r)   r—   rŸ   r›   rœ   r�   rE   rF   r.   Úsigmoidr¡   )r¢   ÚhiddenrG   ÚshiftedrE   rF   r¡   s   &&&    r   Úextract_key_valueÚ#RwkvSelfAttention.extract_key_valueø   s*  € à�;‰;�q‹>˜QÔ 5Ò#4Ø˜A•h˜q ! T§]¡]Ð2Õ3‰Gà—o‘o fÓ-ˆGØÒ Ø % a¥¨¨A¨t¯}©}Ð)<Õ =�˜‘Ø×(Ñ(Õ(¨7°a¸$×:KÑ:KÕ6KÕ+LÕLˆØ×,Ñ,Õ,¨w¸!¸d×>QÑ>QÕ:QÕ/RÕRˆØ×6Ñ6Õ6¸ÀAÈ×H`ÑH`ÕD`Õ9aÕaˆ
à�h‰h�s‹mˆØ—
‘
˜5Ó!ˆÜ—]’] 4§?¡?°:Ó#>Ó?ˆ
ØÒØ,2°5­MˆE�!�H�Q˜˜4Ÿ=™=Ð(Ñ)Ø Ð,Ð,r   c           	     ó  a € S P                  WR 7      w  rErbVeA   \        ;QJ d    . V 3R lVR,           4       F  NK  	  5M! V 3R lVR,           4       4      MRp\        S P                  S P                  VVVVR7      w  r‡Vea   V^ ,          V^,          RRS P
                  3&   V^,          V^,          RRS P
                  3&   V^,          V^,          RRS P
                  3&   S P                  WH,          4      V3# )©rG   Nc              3   óN   <"  € T F  qR R SP                   3,          x € K  	  R# 5i)r'   N©r—   )r   rM   r¢   s   & €r   r�   Ú,RwkvSelfAttention.forward.<locals>.<genexpr>  s"   øé € ÐF¹I°q˜a  D§M¡MÐ1×2Ò2»Iùs   ƒ"%:r(   NNrƒ   r'   )r«   Útuplerˆ   rC   rD   r—   rL   )	r¢   r©   rG   Ú	use_cacher¡   rE   rF   Úlayer_stateÚrwkvs	   f&&&     r   r=   ÚRwkvSelfAttention.forward  sæ   ø€ Ø(,×(>Ñ(>¸vÐ(>Ó(SÑ%ˆ
˜ØJOÒJ[—e”eÔF¸EÀ"¾IÓF—e‘eÔF¸EÀ"¾IÓFÔFÐaeˆÜ1Ø�O‰OØ�O‰OØØØØ"ô
Ñˆð Ò"Ø,7¸­NˆE�!�H�Q˜˜4Ÿ=™=Ð(Ñ)Ø,7¸­NˆE�!�H�Q˜˜4Ÿ=™=Ð(Ñ)Ø,7¸­NˆE�!�H�Q˜˜4Ÿ=™=Ð(Ñ)à�{‰{˜:Õ,Ó-¨uÐ4Ð4r   )r˜   r“   rE   r—   rL   r¡   rC   rD   r›   r�   rœ   rŸ   rF   ©rj   r^   r]   )
r_   r`   ra   rb   r’   r«   r=   rd   re   Ú__classcell__©r¤   rg   s   @@r   rŠ   rŠ   Ù   s   ù‡ € ÷Pô<-÷&5ô 5r   rŠ   c                   ó@   a a€ ] tR tRt oRV 3R lltRR ltRtVtV ;t# )ÚRwkvFeedForwardi  c                óV  <€ \         SV `  4        Wn        W n        VP                  pVP
                  e   VP
                  M^VP                  ,          p\        P                  ! R4      V n        \        P                  ! \        P                  ! ^^V4      4      V n        \        P                  ! \        P                  ! ^^V4      4      V n        \        P                  ! W4RR7      V n        \        P                  ! W3RR7      V n        \        P                  ! WCRR7      V n        R # )NFrŒ   rŽ   )r‘   r’   r“   r—   rK   Úintermediate_sizer   rž   rŸ   r™   r.   rš   r›   r�   r    rE   r¡   rF   )r¢   r“   r—   rK   r½   r¤   s   &&&  €r   r’   ÚRwkvFeedForward.__init__   sÐ   ø€ Ü‰ÑÔØŒØ ŒØ×(Ñ(ˆà(.×(@Ñ(@Ò(LˆF×$Ò$ÐRSÐV\×VhÑVhÕRhð 	ô Ÿ,š, }Ó5ˆŒÜŸLšL¬¯ª°Q¸¸;Ó)GÓHˆÔÜ#%§<¢<´·²¸A¸qÀ+Ó0NÓ#OˆÔ ä—9’9˜[À%ÔHˆŒÜŸ)š) KÀ5ÔIˆŒÜ—Y’YÐ0ÀEÔJˆŽ
r   c                óÚ  € VP                  ^4      ^8X  d#   Ve   V^ ,          RRV P                  3,          pM5V P                  V4      pVe!   V^ ,          RRV P                  3,          VR&   WP                  ,          V^V P                  ,
          ,          ,           pWP                  ,          V^V P                  ,
          ,          ,           p\
        P                  ! \
        P                  ! V P                  V4      4      4      pV P                  V4      p\
        P                  ! V P                  V4      4      pVe!   VR,          V^ ,          RRV P                  3&   WV,          V3# r§   )r)   r—   rŸ   r›   r�   r.   ÚsquareÚrelurE   rF   r¨   r¡   )r¢   r©   rG   rª   rE   r¡   rF   s   &&&    r   r=   ÚRwkvFeedForward.forward1  s  € Ø�;‰;�q‹>˜QÔ 5Ò#4Ø˜A•h˜q ! T§]¡]Ð2Õ3‰Gà—o‘o fÓ-ˆGØÒ Ø % a¥¨¨A¨t¯}©}Ð)<Õ =�˜‘Ø×(Ñ(Õ(¨7°a¸$×:KÑ:KÕ6KÕ+LÕLˆØ×6Ñ6Õ6¸ÀAÈ×H`ÑH`ÕD`Õ9aÕaˆ
ä�lŠlœ5Ÿ:š: d§h¡h¨s£mÓ4Ó5ˆØ—
‘
˜3“ˆÜ—]’] 4§?¡?°:Ó#>Ó?ˆ
àÒØ,2°5­MˆE�!�H�Q˜˜4Ÿ=™=Ð(Ñ)àÕ! 5Ð(Ð(r   )r“   rE   r—   r¡   r›   r�   rŸ   rF   r·   r^   ©	r_   r`   ra   rb   r’   r=   rd   re   r¸   r¹   s   @@r   r»   r»     s   ù‡ € ÷K÷")ô )r   r»   c                   ó<   a a€ ] tR tRt oV 3R ltRR ltRtVtV ;t# )Ú	RwkvBlockiE  c                ó°  <€ \         SV `  4        Wn        W n        V^ 8X  d2   \        P
                  ! VP                  VP                  R7      V n        \        P
                  ! VP                  VP                  R7      V n	        \        P
                  ! VP                  VP                  R7      V n
        \        W4      V n        \        W4      V n        R# )rj   )ÚepsN)r‘   r’   r“   r—   r   Ú	LayerNormrK   Úlayer_norm_epsilonÚpre_lnÚln1Úln2rŠ   Ú	attentionr»   Úfeed_forward)r¢   r“   r—   r¤   s   &&&€r   r’   ÚRwkvBlock.__init__F  s’   ø€ Ü‰ÑÔØŒØ Œà�qŒ=ÜŸ,š, v×'9Ñ'9¸v×?XÑ?XÔYˆDŒKä—<’< × 2Ñ 2¸×8QÑ8QÔRˆŒÜ—<’< × 2Ñ 2¸×8QÑ8QÔRˆŒä*¨6Ó<ˆŒÜ+¨FÓ=ˆÖr   c                ó8  € V P                   ^ 8X  d   V P                  V4      pV P                  V P                  V4      W#R7      w  rRW,           pV P	                  V P                  V4      VR7      w  rbW,           pW3pV'       d   Wu3,          pV# VR,          pV# )rj   )rG   r³   r®   r^   )r—   rÊ   rÍ   rË   rÎ   rÌ   )r¢   r©   rG   r³   Úoutput_attentionsrÍ   rÎ   Úoutputss   &&&&&   r   r=   ÚRwkvBlock.forwardT  s—   € Ø�=‰=˜AÔØ—[‘[ Ó(ˆFàŸ>™>¨$¯(©(°6Ó*:À%˜>Ó]Ñˆ	ØÕ#ˆà"×/Ñ/°·±¸Ó0@ÈÐ/ÓNÑˆØÕ&ˆà�/ˆßØ�|Õ#ˆGð ˆð �wÕˆGàˆr   )rÍ   r“   rÎ   r—   rË   rÌ   rÊ   )NFFrÃ   r¹   s   @@r   rÅ   rÅ   E  s   ù‡ € õ>÷ô r   rÅ   c                   óŒ   a a€ ] tR tRt oRtR.tRR.tRtRt]	P                  ! 4       V3R lV 3R ll4       tV3R	 ltR
tVtV ;t# )ÚRwkvPreTrainedModelig  rµ   rÅ   rC   rD   Tc                ó4   <€ V ^8„  d   QhRS[ P                  /# )r(   Úmodule)r   ÚModule)Úformatrg   s   "€r   Ú__annotate__Ú RwkvPreTrainedModel.__annotate__q  s   ø€ ÷ G7ñ G7¡B§I¡Iñ G7r   c           	     ó:  <€ \         SV `  V4       \        V\        4      '       Ed   VP                  pVP
                  P                  pVP
                  P                  pVP                  pW#^,
          ,          pRW#,          ,
          p\        P                  ! \        V4       Uu. uF  qˆV,          NK  	  upVP                  P                  VP                  P                  R7      p	V	R
,          p	\        V4       U
u. uF5  p
R^W¥^,
          ,          RRV,          ,           ,          ,          ,           NK7  	  pp
\        P                  ! W±P                  P                  VP                  P                  R7      p\        P                  ! \        V4       Uu. uF  qˆ^,           ^,          ^,
          NK  	  upVP                   P                  VP                   P                  R7      R,          p\"        P$                  ! VP                  V4       \"        P$                  ! VP                   \        P&                  ! VP                   \(        P*                  ! R4      ,          V,           4      4       \"        P$                  ! VP                  \        P,                  ! W—4      4       \"        P$                  ! VP.                  \        P,                  ! W—4      RV,          ,           4       \"        P$                  ! VP0                  \        P,                  ! V	RV,          4      4       R# \        V\2        4      '       Ed   VP                  pVP
                  P                  pVP
                  P                  pRW#,          ,
          p\        P                  ! \        V4       Uu. uF  qˆV,          NK  	  upVP                  P                  VP                  P                  R7      p	V	R
,          p	\"        P$                  ! VP                  \        P,                  ! W—4      4       \"        P$                  ! VP0                  \        P,                  ! W—4      4       R# \        V\4        P6                  4      '       dû   VP8                  P:                  pRpRpVP<                  e!   \"        P>                  ! VP<                  4       V^ ,          V^,          8”  d,   \(        P@                  ! V^ ,          V^,          ,          4      pV^ ,          V P
                  PB                  8X  d%   V^,          V P
                  P                  8X  d   RpWï,          p\"        PD                  ! VP8                  VR7       R# \        V\4        PF                  4      '       dp   VP8                  P:                  pR	\(        P@                  ! \I        V^ ,          V^,          4      4      ,          p\"        PD                  ! VP8                  VR7       R# R# u upi u up
i u upi u upi )zInitialize the weights.g      ð?)r#   r$   Ngffffffæ?gÍÌÌÌÌÌô?g      à?g333333Ó?)Úgaing-Cëâ6?)NNr'   éûÿÿÿ)%r‘   Ú_init_weightsÚ
isinstancerŠ   r—   r“   Únum_hidden_layersrK   r˜   r.   Útensorrl   r›   r#   r$   rC   rD   ÚinitÚcopy_Ú	ones_likeÚmathÚlogÚpowrœ   r�   r»   r   r    ÚweightÚshaper�   Úzeros_ÚsqrtÚ
vocab_sizeÚorthogonal_Ú	EmbeddingÚmax)r¢   r×   r—   rá   rK   r˜   Úratio_0_to_1Úratio_1_to_almost0ÚiÚtime_weightÚhÚdecay_speedÚzigzagrê   rÝ   Úscaler¤   s   &&              €r   rß   Ú!RwkvPreTrainedModel._init_weightsp  sO  ø€ ô 	‰Ñ˜fÔ%Ü�fÔ/×0Ó0Ø—‘ˆHØ &§¡× ?Ñ ?ÐØ Ÿ-™-×3Ñ3ˆKØ$*×$@Ñ$@Ð!à#¸1Õ'<Õ=ˆLØ!$¨Õ(DÕ!EÐäŸ,š,Ü*/°Ô*<Ó=Ñ*< Q�[—�Ñ*<Ñ=Ø×)Ñ)×/Ñ/Ø×*Ñ*×1Ñ1ôˆKð
 & mÕ4ˆKô Ð4Ô5óá5�Að �Q˜!°qÕ8Õ9¸sÀSÈ<ÕEWÕ?WÕXÕX×XÐXÙ5ð ð ô  Ÿ,š, {×:KÑ:K×:QÑ:QÐZ`×ZkÑZk×ZrÑZrÔsˆKä—’Ü.3Ð4IÔ.JÓKÑ.J¨˜!•e˜q•[ 1—_�_Ñ.JÑKØ ×+Ñ+×1Ñ1Ø!×,Ñ,×3Ñ3ôð
 õð ô �JŠJ�v×(Ñ(¨+Ô6Ü�JŠJ�v×(Ñ(¬%¯/ª/¸&×:KÑ:KÌdÏhÊhÐWZËmÕ:[Ð^dÕ:dÓ*eÔfä�JŠJ�v×*Ñ*¬E¯IªI°kÓ,VÔWÜ�JŠJ�v×,Ñ,¬e¯iªi¸Ó.XÐ[^ÐamÕ[mÕ.mÔnÜ�JŠJ�v×1Ñ1´5·9²9¸[È#ÐPbÕJbÓ3cÖdÜ˜¤×0Ó0Ø—‘ˆHØ &§¡× ?Ñ ?ÐØ Ÿ-™-×3Ñ3ˆKà!$¨Õ(DÕ!EÐäŸ,š,Ü*/°Ô*<Ó=Ñ*< Q�[—�Ñ*<Ñ=Ø×)Ñ)×/Ñ/Ø×*Ñ*×1Ñ1ôˆKð
 & mÕ4ˆKä�JŠJ�v×*Ñ*¬E¯IªI°kÓ,VÔWÜ�JŠJ�v×1Ñ1´5·9²9¸[Ó3]Ö^Ü˜¤§	¡	×*Ò*Ø—M‘M×'Ñ'ˆEØˆDØˆEØ�{‰{Ò&Ü—’˜FŸK™KÔ(Ø�Q�x˜% �(Ô"Ü—y’y  q¥¨E°!­HÕ!4Ó5�Ø�Q�x˜4Ÿ;™;×1Ñ1Ô1°e¸AµhÀ$Ç+Á+×BYÑBYÔ6YØ�à�MˆDÜ×Ò˜VŸ]™]°×6Ü˜¤§¡×-Ò-Ø—M‘M×'Ñ'ˆEØœ$Ÿ)š)¤C¨¨a­°%¸µ(Ó$;Ó<Õ<ˆDÜ×Ò˜VŸ]™]°×6ñ .ùòo >ùòùò Lùò* >s   Â'X	Ä ;XÆXÎ&Xc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )r(   r“   r   )rÙ   rg   s   "€r   rÚ   rÛ   g  s   ø‡ ‚ áÑò r   r\   )r_   r`   ra   rb   Úbase_model_prefixÚ_no_split_modulesÚ_keep_in_fp32_modulesÚsupports_gradient_checkpointingÚ_is_statefulr.   Úno_gradrß   Ú__annotate_func__rd   re   r¸   r¹   s   @@r   rÕ   rÕ   g  sR   ù‡ € ð ÐØ$˜ÐØ)¨<Ð8ÐØ&*Ð#Ø€Là
‡]‚]ƒ_÷G7ò G7ó ðG7÷ … r   rÕ   z+
    Class for the RWKV model outputs.
    )Úcustom_introc                   ó@   a € ] tR tRt o RtRtRtRtRtV 3R lt	Rt
V tR# )Ú
RwkvOutputi»  zû
state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`):
    The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
    avoid providing the old `input_ids`.
Nc                ó
  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   # )r(   NÚlast_hidden_staterG   .Úhidden_statesÚ
attentions©r.   ÚFloatTensorÚlistr²   )rÙ   rg   s   "€r   rÚ   ÚRwkvOutput.__annotate__»  s~   ø‡ ‚ ñ ×(Ñ(¨4Õ/Ñ6ñ ñ ‘×!Ñ!Õ" TÕ)Ñ0ñ ñ ™×*Ñ*¨CÐ/Õ0°4Õ7Ñ>ñ ñ  ‘e×'Ñ'¨Ð,Õ-°Õ4Ñ;ò! r   r\   )r_   r`   ra   rb   Ú__doc__r  rG   r  r  r  rd   re   rf   s   @r   r  r  »  s(   ø‡ € ñð 37ÐØ,0€EØ:>€MØ7;€J÷! ƒ r   r  zK
    Base class for causal language model (or autoregressive) outputs.
    c                   óD   a € ] tR tRt o RtRtRtRtRtRt	V 3R lt
RtV tR# )ÚRwkvCausalLMOutputiÎ  aP  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`):
    The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
    avoid providing the old `input_ids`.
Nc                ó8  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   # )r(   NÚlossÚlogitsrG   .r  r  r	  )rÙ   rg   s   "€r   rÚ   ÚRwkvCausalLMOutput.__annotate__Î  s–   ø‡ ‚ ñ" ×
Ñ
˜dÕ
"Ñ)ñ# ñ$ ×Ñ Õ$Ñ+ñ% ñ& ‘×!Ñ!Õ" TÕ)Ñ0ñ' ñ( ™×*Ñ*¨CÐ/Õ0°4Õ7Ñ>ñ) ñ* ‘e×'Ñ'¨Ð,Õ-°Õ4Ñ;ò+ r   r\   )r_   r`   ra   rb   r  r  r  rG   r  r  r  rd   re   rf   s   @r   r  r  Î  s,   ø‡ € ñð &*€DØ'+€FØ,0€EØ:>€MØ7;€J÷+ ƒ r   r  c                   ój   a a€ ] tR tRt oV 3R ltR tR t]R
V3R lR ll4       tR t	R t
R	tVtV ;t# )Ú	RwkvModeliæ  c           
     ó¦  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        \        P                  ! VP
                  4      V n        RV n        RV n        V P!                  4        R# u upi )r°   FN)r‘   r’   r   rï   rí   rK   Ú
embeddingsÚ
ModuleListrl   rá   rÅ   ÚblocksrÈ   Úln_outÚlayers_are_rescaledÚgradient_checkpointingÚ	post_init)r¢   r“   Úidxr¤   s   && €r   r’   ÚRwkvModel.__init__è  s™   ø€ Ü‰Ñ˜Ô äŸ,š, v×'8Ñ'8¸&×:LÑ:LÓMˆŒÜ—m’mÔPUÐV\×VnÑVnÔPoÓ$pÑPoÈ¤Y¨v×%DÑPoÑ$pÓqˆŒÜ—l’l 6×#5Ñ#5Ó6ˆŒà#(ˆÔ à&+ˆÔ#ð 	�‰Öùò %qs   Á(Cc                ó   € V P                   # r^   ©r  ©r¢   s   &r   Úget_input_embeddingsÚRwkvModel.get_input_embeddingsö  s   € Ø�‰Ðr   c                ó   € Wn         R # r^   r!  ©r¢   Únew_embeddingss   &&r   Úset_input_embeddingsÚRwkvModel.set_input_embeddingsù  s   € Ø(Žr   c                ó0  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# )r(   Ú	input_idsNÚattention_maskÚinputs_embedsrG   r³   rÑ   Úoutput_hidden_statesÚreturn_dictÚreturn)r.   Ú
LongTensorr
  r  Úboolr²   r  )rÙ   rg   s   "€r   rÚ   ÚRwkvModel.__annotate__ý  s¶   ø€ ÷ h
ñ h
á×#Ñ# dÕ*ðh
ñ ×(Ñ(¨4Õ/ðh
ñ ×(Ñ(¨4Õ/ð	h
ñ
 ‘E×%Ñ%Õ&¨Õ-ðh
ñ ˜$•;ðh
ñ   $�;ðh
ñ # T�kðh
ñ ˜D•[ðh
ñ 
‘Õ	ñh
r   c	           
     ón  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TM)V P                  '       g   V P                   P                  MRpVe   TMV P                   P
                  pVe   \        P                  R4       V P                  V P                  8X  d   V P                  4        Ve   Ve   \        R4      hVf   Vf   \        R4      hVf   V P                  V4      pV'       d±   Vf­   VP                  ^ 4      V P                   P                  V P                   P                  3p
\        ^4       Uu. uFF  p\         P"                  ! T
RV^8:  d   VP$                  M\         P&                  RVP(                  / NKH  	  ppV^;;,          R,          uu&   V P*                  '       d2   V P                  '       d    V'       d   \        P                  R	4       RpTpV'       d   RMRpV'       d   RMRp\-        V P.                  4       F—  w  ppV! WÄWVR
7      w  rÄpV P                  '       dN   V P                   P0                  ^ 8”  d3   V^,           V P                   P0                  ,          ^ 8X  d
   V^,          pV'       d
   Wì3,           pV'       g   K�  VV3,           pK™  	  V P3                  V4      pV'       d
   Wì3,           pV'       g1   \4        ;QJ d    . R WÄWí3 4       F  NK  	  5# ! R WÄWí3 4       4      # \7        VVVVR7      # u upi )a(  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
    `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
    sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*):
    If passed along, the model uses the previous state in all the blocks (which will give the output for the
    `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
use_cache (`bool`, *optional*):
    If set to `True`, the last state is returned and can be used to quickly generate the next logits.
NFz<`attention_mask` was passed, but it is unused in this model.zDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsr#   r$   gêŒ 9Y>)FzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...)rG   r³   rÑ   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5ir^   r\   )r   Úxs   & r   r�   Ú$RwkvModel.forward.<locals>.<genexpr>^  s   é € ÐtÑ$b˜qŸšÓ$bùs   ‚Œ
)r  rG   r  r  r\   )r“   rÑ   r.  Útrainingr³   r/  r•   Úwarning_oncer  Ú_rescale_layersr*   r  r)   rK   rá   rl   r.   r5   r#   r6   r$   r  Ú	enumerater  Úrescale_everyr  r²   r  )r¢   r+  r,  r-  rG   r³   rÑ   r.  r/  Úkwargsrê   ró   r  Úall_self_attentionsÚall_hidden_statesr  Úblockr  s   &&&&&&&&&,        r   r=   ÚRwkvModel.forwardü  sÖ  € ð@ 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘IÐZ^×Zg×ZgÐZg¸T¿[¹[×=RÒ=RÐmrˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ%Ü×ÑÐ ^Ô_à�=‰=˜D×4Ñ4Ô4Ø× Ñ Ô"àÒ  ]Ò%>ÜÐcÓdÐdØÒ =Ò#8ÜÐTÓUÐUàÒ Ø ŸO™O¨IÓ6ˆMç˜šØ"×'Ñ'¨Ó*¨D¯K©K×,CÑ,CÀTÇ[Á[×EbÑEbÐcˆEô
 ˜qœó	ñ "�Aô —’ØðØ9:¸a¼ -×"5Ò"5ÄUÇ]Á]ðØ[h×[oÑ[oôñ "ð	 ð ð �!�H˜Õ‹Hà×&×&Ð&¨4¯=¯=¨=ßÜ×#Ñ#Øpôð "�	à%ˆç$5™b¸4Ðß"6™B¸DÐÜ# D§K¡KÖ0‰JˆC�Ù/4Ø°iô0Ñ,ˆM *ð
 ×(×(Ð(Ø—K‘K×-Ñ-°Ô1Ø˜1•W §¡× 9Ñ 9Õ9¸QÔ>à -°Õ 1�ç#Ø$5Ð8HÕ$HÐ!ç Ñ Ø&9¸Z¸MÕ&IÒ#ñ! 1ð$ Ÿ™ MÓ2ˆçØ 1Ð4DÕ DÐçß”5Ñt ]Ð;LÑ$bÓt—5Ðt�5Ñt ]Ð;LÑ$bÓtÓtÐtäØ+ØØ+Ø*ô	
ð 	
ùò[s   ÅAL2c           
     ó@  € V P                   V P                  '       * 8X  d   R # V P                  P                  ^ 8”  Ed7   \        P
                  ! 4       ;_uu_ 4        \        V P                  4       EFö  w  rV P                  '       dµ   VP                  P                  P                  P                  ^\        WP                  P                  ,          4      ,          4       VP                  P                  P                  P                  ^\        WP                  P                  ,          4      ,          4       KÌ  \        VP                  P                  P                  R4      '       dÊ   VP                  P                  P                  P                   P#                  ^\        WP                  P                  ,          4      ,          4       VP                  P                  P                  P                   P#                  ^\        WP                  P                  ,          4      ,          4       EKÅ  \        VP                  P                  P                  R4      '       dP   V P%                  VP                  P                  V4       V P%                  VP                  P                  V4       EKD  VP                  P                  P                  P#                  ^\        WP                  P                  ,          4      ,          4       VP                  P                  P                  P#                  ^\        WP                  P                  ,          4      ,          4       EKù  	  R R R 4       V P                  '       * V n         R #   + '       g   i     L(; i)NÚSCBÚquant_state)r  r8  r“   r<  r.   r   r;  r  rÍ   rL   ré   Úmul_ÚintrÎ   rF   ÚhasattrrC  Údiv_Ú _bnb_4bit_dequantize_and_rescale)r¢   Úblock_idr@  s   &  r   r:  ÚRwkvModel._rescale_layersg  sC  € à×#Ñ#¨D¯M©MÔ(9Ô:ÙØ�;‰;×$Ñ$ qÕ(Ü—’—•Ü'0°·±×'=‘O�HØ—}—}�}ØŸ™×.Ñ.×5Ñ5×:Ñ:¸1ÄÀH×P[ÑP[×PiÑPiÕDiÓ@jÕ;jÔkØ×*Ñ*×0Ñ0×7Ñ7×<Ñ<¸QÄ#Àh×R]ÑR]×RkÑRkÕFkÓBlÕ=lÖmô # 5§?¡?×#9Ñ#9×#@Ñ#@À%×HÒHØ!ŸO™O×2Ñ2×9Ñ9×=Ñ=×BÑBÀ1ÌÈH×XcÑXc×XqÑXqÕLqÓHrÕCrÔsØ!×.Ñ.×4Ñ4×;Ñ;×?Ñ?×DÑDÀQÌ#Èh×ZeÑZe×ZsÑZsÕNsÓJtÕEt×uÜ$ U§_¡_×%;Ñ%;×%BÑ%BÀM×RÒRØ ×AÑAÀ%Ç/Á/×BXÑBXÐZbÔcØ ×AÑAÀ%×BTÑBT×BZÑBZÐ\d×eà!ŸO™O×2Ñ2×9Ñ9×>Ñ>¸qÄCÈ×T_ÑT_×TmÑTmÕHmÓDnÕ?nÔoØ!×.Ñ.×4Ñ4×;Ñ;×@Ñ@ÀÄcÈ(×VaÑVa×VoÑVoÕJoÓFpÕAp×qñ (>÷ !ð" (,§}¡}Ô#4ˆÖ ÷# !—ús   ÁLNÎN	c                óæ  € \        4       '       g   \        R4      h^ RIpVP                  P	                  VP
                  P                  VP
                  P                  4      pVP                  ^\        W P                  P                  ,          4      ,          4       VP                  P                  VP                  R4      RR7      P                  VP                  4      p\!        VRV4       R# )zƒ
Perform the dequantization and rescaling of the weights of a given layer. After that operation the layer will
be quantized again.
z/Please install bitsandbytes to use this method.NÚcpuF)Úrequires_gradré   )r
   r   ÚbitsandbytesÚ
functionalÚdequantize_4bitré   ÚdatarD  rH  rF  r“   r<  r   Ú
Params4bitrA   r$   Úsetattr)r¢   Útarget_layerrJ  ÚbnbÚdequant_weightsÚquant_weights   &&&   r   rI  Ú*RwkvModel._bnb_4bit_dequantize_and_rescale  sµ   € ô
 )×*Ò*ÜÐOÓPÐPÛ"àŸ.™.×8Ñ8¸×9LÑ9L×9QÑ9QÐS_×SfÑSf×SrÑSrÓsˆà×Ñ˜Q¤# h·+±+×2KÑ2KÕ&KÓ"LÕLÔMð —v‘v×(Ñ(¨×);Ñ);¸EÓ)BÐRWÐ(ÓX×[Ñ[Ð\k×\rÑ\rÓsˆÜ�˜h¨Ö5r   )r  r  r  r  r  )NNNNNNNN)r_   r`   ra   rb   r’   r#  r(  r	   r=   r:  rI  rd   re   r¸   r¹   s   @@r   r  r  æ  s@   ù‡ € õòò)ð ÷h
ñ h
ó ðh
òT5÷06ò 6r   r  z‡
    The RWKV Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                   óf   a a€ ] tR tRt oRR/tV 3R ltR tR t]R
V3R lR ll4       t	R	t
VtV ;t# )ÚRwkvForCausalLMi•  zhead.weightzrwkv.embeddings.weightc                óÊ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        V P                  4        R# )FrŒ   N)
r‘   r’   r  rµ   r   r    rK   rí   Úheadr  )r¢   r“   r¤   s   &&€r   r’   ÚRwkvForCausalLM.__init__ž  sH   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ	Ü—I’I˜f×0Ñ0°&×2CÑ2CÈ%ÔPˆŒ	ð 	�‰Ör   c                ó   € V P                   # r^   ©r]  r"  s   &r   Úget_output_embeddingsÚ%RwkvForCausalLM.get_output_embeddings¦  s   € Ø�y‰yÐr   c                ó   € Wn         R # r^   r`  r&  s   &&r   Úset_output_embeddingsÚ%RwkvForCausalLM.set_output_embeddings©  s   € Ø"Ž	r   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P                  ,          RS[S[,          /# )r(   r+  Nr,  r-  rG   Úlabelsr³   rÑ   r.  r/  Úlogits_to_keepr0  )	r.   r1  r
  r  r2  rF  ÚTensorr²   r  )rÙ   rg   s   "€r   rÚ   ÚRwkvForCausalLM.__annotate__­  sá   ø€ ÷ D
ñ D
á×#Ñ# dÕ*ðD
ñ ×(Ñ(¨4Õ/ðD
ñ ×(Ñ(¨4Õ/ð	D
ñ
 ‘E×%Ñ%Õ&¨Õ-ðD
ñ × Ñ  4Õ'ðD
ñ ˜$•;ðD
ñ   $�;ðD
ñ # T�kðD
ñ ˜D•[ðD
ñ ™eŸl™lÕ*ðD
ñ 
Ñ#Õ	#ñD
r   c           
     ó  € V	e   T	MV P                   P                  p	V P                  VVVVVVV	R7      pV^ ,          p\        V
\        4      '       d   \        V
) R4      MT
pV P                  VRVR3,          4      pRpVe-   V P                  ! R	RVRVRV P                   P                  /VB pV	'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  VP                  R7      # )
a²  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
    `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
    sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*):
    If passed along, the model uses the previous state in all the blocks (which will give the output for the
    `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
    `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
    are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
use_cache (`bool`, *optional*):
    If set to `True`, the last state is returned and can be used to quickly generate the next logits.
N)r-  rG   r³   rÑ   r.  r/  r'   r  rg  rí   :r�   NN)r  r  rG   r  r  r\   )r“   r/  rµ   rà   rF  Úslicer]  Úloss_functionrí   r  rG   r  r  )r¢   r+  r,  r-  rG   rg  r³   rÑ   r.  r/  rh  r=  Úrwkv_outputsr  Úslice_indicesr  r  rL   s   &&&&&&&&&&&,      r   r=   ÚRwkvForCausalLM.forward¬  s  € ðL &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—y‘yØØ'ØØØ/Ø!5Ø#ð !ó 
ˆð % Q�ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜=¨¨M¸1Ð)<Õ=Ó>ˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDçØ�Y ¨bÕ!1Õ1ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä!ØØØ×$Ñ$Ø&×4Ñ4Ø#×.Ñ.ô
ð 	
r   )r]  rµ   )
NNNNNNNNNrj   )r_   r`   ra   rb   Ú_tied_weights_keysr’   ra  rd  r	   r=   rd   re   r¸   r¹   s   @@r   r[  r[  •  sD   ù‡ € ð (Ð)AÐBÐõòò#ð ÷D
ñ D
ó ÷D
ð D
r   r[  r]   )r[  r  rÕ   )-r  ræ   Údataclassesr   r.   r   Ú r   rã   Ú
generationr   Úmodeling_layersr   Úmodeling_utilsr   Úutilsr   r	   r
   r   r   r   r   Úconfiguration_rwkvr   Ú
get_loggerr_   r•   r   r   ÚautogradÚFunctionr   r|   rˆ   rØ   rŠ   r»   rÅ   rÕ   r  r  r  r[  Ú__all__r\   r   r   Ú<module>r}     s}  ðñ ã Ý !ã Ý å &Ý )Ý 9Ý -÷÷ ñ õ +ð 
×	Ò	˜HÓ	%€ð Ð ò5ôg
˜%Ÿ.™.×1Ñ1ô g
ôT)ôXbôC5˜Ÿ	™	ô C5ôL#)�b—i‘iô #)ôLÐ*ô ðD ôP7˜/ó P7ó ðP7ñf ðôð
 ô
<�ó 
<ó óð
<ñ ðôð
 ô<˜ó <ó óð<ð$ ôk6Ð#ó k6ó ðk6ñ\ ðôôV
Ð)¨?ó V
óðV
òr B‚r   