+
    œ°³jÕ  ã                   óº  € ^ RI t ^ RIt^ RIHt ^RIHt ^RIHt ^RIH	t	H
t
Ht ^RIHt ^RIHtHtHtHtHt ^RIHt ]P,                  ! ]4      t ! R R	]P2                  4      t ! R
 R]4      t ! R R]4      tR tRR lt] ! R R]4      4       t ! R R]4      t  ! R R]4      t! ! R R]!]4      t" ! R R]	4      t# ! R R]
4      t$ ! R R]4      t%. ROt&R# ) é    N)Úlogging)Úno_inherit_decorator)ÚGemmaForCausalLMÚGemmaForSequenceClassificationÚGemmaForTokenClassification)ÚGraniteAttention)ÚLlamaDecoderLayerÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbedding©ÚHeliumConfigc                   óB   a a€ ] tR t^t oRV 3R lltR tR tRtVtV ;t	# )ÚHeliumRMSNormc                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R # ©N)ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s   &&&€Új/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/helium/modular_helium.pyr   ÚHeliumRMSNorm.__init__    s-   ø€ Ü‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Öó    c                ót  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  P                  \        P                  4      V,          P                  V4      # )é   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor   Úfloat32ÚpowÚmeanÚrsqrtr   r   )r   Úhidden_statesÚinput_dtypeÚvariances   &&  r    ÚforwardÚHeliumRMSNorm.forward%   s„   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ—‘—‘œuŸ}™}Ó-°Õ=×AÑAÀ+ÓNÐNr"   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler   Úshaper   )r   s   &r    Ú
extra_reprÚHeliumRMSNorm.extra_repr,   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr"   )r   r   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r0   r5   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   Ú__classdict__s   @@r    r   r      s   ù‡ € ÷$ò
O÷Jò Jr"   r   c                   ó   € ] tR t^0tRtR# )ÚHeliumRotaryEmbedding© N©r7   r8   r9   r:   r;   rB   r"   r    rA   rA   0   ó   † Ûr"   rA   c                   ó   € ] tR t^4tRtR# )Ú	HeliumMLPrB   NrC   rB   r"   r    rF   rF   4   rD   r"   rF   c                óz   € V R,          pV R,          p\         P                  ! V) V3RR7      P                  R4      # )z*Rotates half the hidden dims of the input.©Údim).:r   Nr$   ).:é   Nr$   r&   éþÿÿÿ)r   ÚstackÚflatten)ÚxÚx1Úx2s   &  r    Úrotate_halfrQ   8   s6   € à	
ˆ9�€BØ	
ˆ9�€BÜ�;Š;˜˜˜R�y bÔ)×1Ñ1°"Ó5Ð5r"   c                ó   € VP                  V4      pVP                  V4      pVRRVP                  R,          ^,          13,          P                  ^RR7      pVRRVP                  R,          ^,          13,          P                  ^RR7      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.NrH   r&   )Ú	unsqueezer4   Úrepeat_interleaverQ   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r    Úapply_rotary_pos_embr\   ?   s¹   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
:Ñ
:¸1À"Ð
:Ó
E€CØ
ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
:Ñ
:¸1À"Ð
:Ó
E€Cà�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GàÐÐr"   c                   óB   a a€ ] tR t^^t oRV3R lV 3R llltRtVtV ;t# )ÚHeliumAttentionc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r$   ÚconfigÚ	layer_idxN©r   Úint)Úformatr?   s   "€r    Ú__annotate__ÚHeliumAttention.__annotate__`   s   ø€ ÷ 4ñ 4™|ð 4¹¸d½
ñ 4r"   c                óâ   <€ \         SV `  W4       \        P                  ! VP                  VP                  R R7      V n        ^\        P                  ! V P                  4      ,          V n	        R# )F)ÚbiasN)
r   r   r   ÚLinearr   Úo_projÚmathÚsqrtÚhead_dimÚscaling©r   ra   rb   r   s   &&&€r    r   ÚHeliumAttention.__init__`   sI   ø€ Ü‰Ñ˜Ô+Ü—i’i × 2Ñ 2°F×4FÑ4FÈUÔSˆŒØœ4Ÿ9š9 T§]¡]Ó3Õ3ˆŽr"   )rk   ro   r   ©r7   r8   r9   r:   r   r;   r<   r=   r>   s   @@r    r^   r^   ^   s   ù‡ € ÷4÷ 4õ 4r"   r^   c                   óB   a a€ ] tR t^ft oRV3R lV 3R llltRtVtV ;t# )ÚHeliumDecoderLayerc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r`   rc   )re   r?   s   "€r    rf   ÚHeliumDecoderLayer.__annotate__g   s#   ø€ ÷ cñ c™|ð c¹¸d½
ñ cr"   c                óÞ   <€ \         SV `  W4       \        V4      V n        \	        VP
                  VP                  R 7      V n        \	        VP
                  VP                  R 7      V n        R# )©r   N)	r   r   rF   Úmlpr   r   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrp   s   &&&€r    r   ÚHeliumDecoderLayer.__init__g   sR   ø€ Ü‰Ñ˜Ô+ä˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÖ%r"   )r{   ry   r|   r   rr   r>   s   @@r    rt   rt   f   s   ù‡ € ÷c÷ cõ cr"   rt   c                   ó   € ] tR t^otRtR# )ÚHeliumPreTrainedModelrB   NrC   rB   r"   r    r   r   o   rD   r"   r   c                   ó>   a a€ ] tR t^st oV3R lV 3R lltRtVtV ;t# )ÚHeliumModelc                ó    <€ V ^8„  d   QhRS[ /# )r$   ra   r   )re   r?   s   "€r    rf   ÚHeliumModel.__annotate__t   s   ø€ ÷ 	ñ 	™|ñ 	r"   c           	     ó8  <€ \         SV `  V4       \        P                  ! \	        VP
                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        RV n        V P                  4        R# u upi )rx   FN)r   r   r   Ú
ModuleListÚrangeÚnum_hidden_layersrt   Úlayersr   r   rz   ÚnormÚgradient_checkpointingÚ	post_initrp   s   && €r    r   ÚHeliumModel.__init__t   s{   ø€ Ü‰Ñ˜Ô Ü—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ø&+ˆÔ#ð 	�‰Öùò es   ¸B)rŠ   rˆ   r‰   rr   r>   s   @@r    r�   r�   s   s   ù‡ € ÷	÷ 	ó 	r"   r�   c                   ó   € ] tR t^€tRtR# )ÚHeliumForCausalLMrB   NrC   rB   r"   r    rŽ   rŽ   €   rD   r"   rŽ   c                   ó   € ] tR t^„tRtR# )ÚHeliumForSequenceClassificationrB   NrC   rB   r"   r    r�   r�   „   rD   r"   r�   c                   ó   € ] tR t^ˆtRtR# )ÚHeliumForTokenClassificationrB   NrC   rB   r"   r    r’   r’   ˆ   rD   r"   r’   )rJ   )r   r�   rŽ   r�   r’   )'rl   r   Útorch.nnr   Úutilsr   Úutils.genericr   Úgemma.modeling_gemmar   r   r   Úgranite.modeling_graniter   Úllama.modeling_llamar	   r
   r   r   r   Úconfiguration_heliumr   Ú
get_loggerr7   ÚloggerÚModuler   rA   rF   rQ   r\   r^   rt   r   r�   rŽ   r�   r’   Ú__all__rB   r"   r    Ú<module>rž      sä   ðó ã Ý å Ý 1ß pÑ pÝ 7ß vÕ vÝ .ð 
×	Ò	˜HÓ	%€ôJ�B—I‘Iô Jô"	Ð0ô 	ô	�ô 	ò6ôð> ô4Ð&ó 4ó ð4ôcÐ*ô cô	Ð0ô 	ô
Ð'¨ô 
ô	Ð(ô 	ô	Ð&Dô 	ô	Ð#>ô 	ò‚r"   