+
    œ°³jâQ  ã                   ó~  € ^ RI t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIH	t	H
t
 ^RIHt ^RIHt ^RIHtHtHt ^RIHtHt ^R	IHtHt ^R
IHtHt ^RIHt ^RIHtH t H!t! ^RI"H#t# ^RI$H%t%H&t& ^RI'H(t( ^RI)H*t*  ! R R]PV                  4      t, ! R R]PV                  4      t- ! R R]PV                  4      t.R R lt/R+R R llt0R t1R,R lt2 ! R R]PV                  4      t3 ! R R ]4      t4]  ! R! R"]4      4       t5]  ! R# R$]54      4       t6]  ! R% R&]5]4      4       t7 ! R' R(]]54      t8 ! R) R*]]54      t9. R-Ot:R# ).é    N)ÚCallable)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚHeliumConfigc                   óB   a a€ ] tR t^/t oRV 3R lltR tR tRtVtV ;t	# )ÚHeliumRMSNormc                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R # ©N)ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s   &&&€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/helium/modeling_helium.pyr!   ÚHeliumRMSNorm.__init__0   s-   ø€ Ü‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Öó    c                ót  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  P                  \        P                  4      V,          P                  V4      # )é   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor$   Úfloat32ÚpowÚmeanÚrsqrtr'   r&   )r(   Úhidden_statesÚinput_dtypeÚvariances   &&  r,   ÚforwardÚHeliumRMSNorm.forward5   s„   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ—‘—‘œuŸ}™}Ó-°Õ=×AÑAÀ+ÓNÐNr.   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler&   Úshaper'   )r(   s   &r,   Ú
extra_reprÚHeliumRMSNorm.extra_repr<   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr.   )r'   r&   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r!   r<   rA   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r+   Ú__classdict__s   @@r,   r   r   /   s   ù‡ € ÷$ò
O÷Jò Jr.   r   c                   óÎ   a a€ ] tR t^@t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚHeliumRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r0   Úconfigr   )ÚformatrK   s   "€r,   Ú__annotate__Ú"HeliumRotaryEmbedding.__annotate__B   s   ø€ ÷ Oñ O™|ñ Or.   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r    r!   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrR   Úrope_parametersrW   Úcompute_default_rope_parametersr   Úattention_scalingr"   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r(   rR   rN   Úrope_init_fnra   r+   s   &&&  €r,   r!   ÚHeliumRotaryEmbedding.__init__A   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr.   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r0   rR   Úreturn)r   r?   r$   ÚTensorÚfloat)rS   rK   s   "€r,   rT   rU   T   s.   ø€ ÷ 5ñ 5±ð 5ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ 5r.   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©r3   )r]   Úgetattrr)   Únum_attention_headsr$   Úarangeri   r4   )rR   rN   ÚkwargsÚbaseÚdimÚattention_factorra   s   &&,    r,   r^   Ú5HeliumRotaryEmbedding.compute_default_rope_parametersR   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r.   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)r3   rN   ÚmpsÚcpuF)Údevice_typeÚenabled©rs   rm   )NºNNNNr2   )r|   Nr|   )ra   Úexpandr@   r4   r$   ri   rN   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr_   Úsinr3   )
r(   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedry   ÚfreqsÚembrƒ   r„   s
   &&&       r,   r<   ÚHeliumRotaryEmbedding.forwardf   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r_   rR   ra   r[   rc   r\   rW   r   )rC   rD   rE   rF   r   r!   Ústaticmethodr^   r$   Úno_gradr   r<   rG   rH   rI   rJ   s   @@r,   rM   rM   @   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r.   rM   c                   ó8   a a€ ] tR t^yt oV 3R ltR tRtVtV ;t# )Ú	HeliumMLPc                ó  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  VP                  R 7      V n        \
        P                  ! V P                  V P                  VP                  R 7      V n	        \
        P                  ! V P                  V P                  VP                  R 7      V n
        \        VP                  ,          V n        R# )©ÚbiasN)r    r!   rR   r)   Úintermediate_sizer"   ÚLinearÚmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r(   rR   r+   s   &&€r,   r!   ÚHeliumMLP.__init__z   s¶   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r   )r˜   rš   r–   r—   )r(   r…   r˜   s   && r,   r<   ÚHeliumMLP.forward„   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr.   )rš   rR   r˜   r–   r)   r“   r—   ©	rC   rD   rE   rF   r!   r<   rG   rH   rI   rJ   s   @@r,   r�   r�   y   s   ù‡ € õ0÷ò r.   r�   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r0   r9   Ún_reprg   )r$   rh   Úint)rS   s   "r,   rT   rT   ‰   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur.   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r|   r|   Nr|   r|   )r@   r}   Úreshape)r9   r¡   ÚbatchÚnum_key_value_headsÚslenrl   s   &&    r,   Ú	repeat_kvr¨   ‰   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr.   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r0   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrq   )r"   ÚModuler$   rh   ri   r   r   )rS   s   "r,   rT   rT   •   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r.   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r0   )rs   r3   )ÚpÚtrainingr2   )r¨   Únum_key_value_groupsr$   Úmatmulr�   r"   Ú
functionalÚsoftmaxr5   r4   r3   r°   r´   Ú
contiguous)rª   r«   r¬   r­   r®   r¯   r°   rq   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r,   Úeager_attention_forwardr¾   •   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r.   c                óz   € V R,          pV R,          p\         P                  ! V) V3RR7      P                  R4      # )z*Rotates half the hidden dims of the input.r{   ).:r   Nr0   ).:é   Nr0   r2   éþÿÿÿ)r$   ÚstackÚflatten)r…   Úx1Úx2s   &  r,   Úrotate_halfrÆ   ®   s6   € à	
ˆ9�€BØ	
ˆ9�€BÜ�;Š;˜˜˜R�y bÔ)×1Ñ1°"Ó5Ð5r.   c                ó   € VP                  V4      pVP                  V4      pVRRVP                  R,          ^,          13,          P                  ^RR7      pVRRVP                  R,          ^,          13,          P                  ^RR7      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nr{   r2   )Ú	unsqueezer@   Úrepeat_interleaverÆ   )ÚqÚkrƒ   r„   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r,   Úapply_rotary_pos_embrÏ   µ   s¹   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
:Ñ
:¸1À"Ð
:Ó
E€CØ
ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
:Ñ
:¸1À"Ð
:Ó
E€Cà�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GàÐÐr.   c                   ó\   a a€ ] tR t^Ôt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚHeliumAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r0   rR   Ú	layer_idxN©r   r¢   )rS   rK   s   "€r,   rT   ÚHeliumAttention.__annotate__×   s#   ø€ ÷ Tñ T™|ð T¹¸d½
ñ Tr.   c                ó”  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        ^\        P                  ! V P                  4      ,          V n        VP                  V n        RV n        \        P                   ! VP
                  VP                  V P                  ,          VP"                  R7      V n        \        P                   ! VP
                  VP                  V P                  ,          VP"                  R7      V n        \        P                   ! VP
                  VP                  V P                  ,          VP"                  R7      V n        \        P                   ! VP
                  VP
                  RR7      V n        R# )rl   Tr‘   FN)r    r!   rR   rÔ   rn   r)   ro   rl   r¦   rµ   ÚmathÚsqrtr¯   Úattention_dropoutÚ	is_causalr"   r”   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©r(   rR   rÔ   r+   s   &&&€r,   r!   ÚHeliumAttention.__init__×   s?  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Øœ4Ÿ9š9 T§]¡]Ó3Õ3ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’i × 2Ñ 2°F×4FÑ4FÈUÔSˆŽr.   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )r0   r9   Úposition_embeddingsNr®   Úpast_key_valuesrq   rg   )r$   rh   r?   r   r   r   )rS   rK   s   "€r,   rT   rÖ   ì   s‡   ø€ ÷ &)ñ &)á—|‘|ð&)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð&)ñ Ÿ™ tÕ+ð	&)ñ
  �ð&)ñ Ñ+Õ,ð&)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ&)r.   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pWï3# )Nr°   ç        r¯   r2   )r@   rl   rÝ   Úviewr�   rÞ   rß   rÏ   ÚupdaterÔ   r   Úget_interfacerR   Ú_attn_implementationr¾   r´   rÚ   r¯   r¤   r¹   rà   )r(   r9   rä   r®   rå   rq   Úinput_shapeÚhidden_shapeÚquery_statesrº   r»   rƒ   r„   Úattention_interfacer½   r¼   s   &&&&&,          r,   r<   ÚHeliumAttention.forwardì   s‡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r.   )rÚ   rR   rl   rÛ   rÞ   rÔ   rµ   rà   rÝ   r¯   rß   r   )NNN)
rC   rD   rE   rF   Ú__doc__r!   r<   rG   rH   rI   rJ   s   @@r,   rÑ   rÑ   Ô   s#   ù‡ € ÙG÷Tõ T÷*&)÷ &)ò &)r.   rÑ   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚHeliumDecoderLayeri  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# rÓ   rÕ   )rS   rK   s   "€r,   rT   ÚHeliumDecoderLayer.__annotate__  s#   ø€ ÷ cñ c™|ð c¹¸d½
ñ cr.   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))rR   rÔ   ©r*   N)r    r!   r)   rÑ   Ú	self_attnr�   Úmlpr   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrá   s   &&&€r,   r!   ÚHeliumDecoderLayer.__init__  sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä(°ÔLˆŒä˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÖ%r.   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r0   r9   r®   Nr†   rå   Ú	use_cacherä   rq   rg   )r$   rh   Ú
LongTensorr   Úboolr?   r   r   )rS   rK   s   "€r,   rT   rõ      sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr.   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r9   r®   r†   rå   rÿ   rä   © )rû   rø   rü   rù   )
r(   r9   r®   r†   rå   rÿ   rä   rq   ÚresidualÚ_s
   &&&&&&&,  r,   r<   ÚHeliumDecoderLayer.forward   s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr.   )r)   rû   rù   rü   rø   r   )NNNFNrŸ   rJ   s   @@r,   ró   ró     s    ù‡ € ÷cõ c÷÷ ò r.   ró   c                   ó`   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltR	tV tR
# )ÚHeliumPreTrainedModeli@  ÚmodelTró   rå   r9   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rQ   r   )rS   rK   s   "€r,   rT   Ú"HeliumPreTrainedModel.__annotate__@  s   ø‡ ‚ áÑò r.   r  N)rC   rD   rE   rF   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendró   rÑ   Ú_can_record_outputsÚ__annotate_func__rG   rH   )rK   s   @r,   r  r  @  s\   ø‡ € ð  ÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ+Ø�oðÐ÷ ƒ r.   r  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚHeliumModeliS  c                ó    <€ V ^8„  d   QhRS[ /# rQ   r   )rS   rK   s   "€r,   rT   ÚHeliumModel.__annotate__U  s   ø€ ÷ ñ ™|ñ r.   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r÷   ©rR   FN)r    r!   Úpad_token_idÚpadding_idxÚ
vocab_sizer"   Ú	Embeddingr)   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersró   Úlayersr   rú   ÚnormrM   Ú
rotary_embÚgradient_checkpointingÚ	post_initrá   s   && €r,   r!   ÚHeliumModel.__init__U  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°vÔ>ˆŒØ&+ˆÔ#ð 	�‰Öùò es   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r0   Ú	input_idsNr®   r†   rå   Úinputs_embedsrÿ   rq   rg   )	r$   r   rh   r   ÚFloatTensorr  r   r   r   )rS   rK   s   "€r,   rT   r  h  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r.   c                óˆ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr  )rN   )rR   r.  r®   rå   r†   )r†   r®   rä   r†   rå   rÿ   )Úlast_hidden_staterå   )Ú
ValueErrorr"  r   rR   Úget_seq_lengthr$   rp   r@   rN   rÈ   r   r(  r&  r%  r'  r   )r(   r-  r®   r†   rå   r.  rÿ   rq   Úpast_seen_tokensÚcausal_maskr9   rä   Údecoder_layers   &&&&&&&,     r,   r<   ÚHeliumModel.forwarde  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r.   )r"  r)  r&  r'  r  r(  r   )NNNNNN)rC   rD   rE   rF   r!   r   r   r   r<   rG   rH   rI   rJ   s   @@r,   r  r  S  s=   ù‡ € ÷ó ð   ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r.   r  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚHeliumForCausalLMi�  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr9   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )Fr‘   N)
r    r!   r  r	  r   r"   r”   r)   r:  r*  r›   s   &&€r,   r!   ÚHeliumForCausalLM.__init__¤  sU   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör.   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r0   r-  Nr®   r†   rå   r.  Úlabelsrÿ   Úlogits_to_keeprq   rg   )
r$   r   rh   r   r/  r  r¢   r   r   r   )rS   rK   s   "€r,   rT   ÚHeliumForCausalLM.__annotate__¯  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r.   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )a   
Example:

```python
>>> from transformers import AutoTokenizer, HeliumForCausalLM

>>> model = HeliumForCausalLM.from_pretrained("google/helium-7b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/helium-7b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```r-  r®   r†   rå   r.  rÿ   Nr|   r<  rA  r   )Úlossr<  rå   r9   r
  r  )r	  r1  r~   r¢   Úslicer:  Úloss_functionrR   r   r   rå   r9   r
  )r(   r-  r®   r†   rå   r.  rA  rÿ   rB  rq   Úoutputsr9   Úslice_indicesr<  rE  s   &&&&&&&&&,     r,   r<   ÚHeliumForCausalLM.forward­  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   )r:  r	  r   )NNNNNNNr   )rC   rD   rE   rF   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr!   r   r   r<   rG   rH   rI   rJ   s   @@r,   r9  r9  �  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r.   r9  c                   ó   € ] tR tRtRtR# )ÚHeliumForSequenceClassificationiè  r  N©rC   rD   rE   rF   rG   r  r.   r,   rP  rP  è  ó   † Ûr.   rP  c                   ó   € ] tR tRtRtR# )ÚHeliumForTokenClassificationiì  r  NrQ  r  r.   r,   rT  rT  ì  rR  r.   rT  )rç   )rÀ   )r  r  r9  rP  rT  );rØ   Úcollections.abcr   r$   Útorch.nnr"   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr   Úmodeling_layersr	   r
   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_heliumr   r±   r   rM   r�   r¨   r¾   rÆ   rÏ   rÑ   ró   r  r  r9  rP  rT  Ú__all__r  r.   r,   Ú<module>rf     s5  ðó* Ý $ã Ý å !ß .Ý )Ý /÷ñ ÷
 Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý .ôJ�B—I‘Iô Jô"6<˜BŸI™Iô 6<ôr�—	‘	ô õ 	U÷%ò26ôô>>)�b—i‘iô >)ôB(Ð3ô (ðV ô˜Oó ó ðð$ ôF
Ð'ó F
ó ðF
ðR ôG
Ð-¨ó G
ó ðG
ôT	Ð&FÐH]ô 	ô	Ð#@ÐBWô 	ò‚r.   