+
    œ°³jsT  ã                   óœ  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtHt ^RI	H
t
 ^RIHtHt ^RIHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHt ^RIHtHt ^RIHt ^RIH t H!t!H"t" ^RI#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+ ]! R4       ! R R]PX                  4      4       t- ! R R]PX                  4      t.R t/]! R4      R,R l4       t0R R lt1R-R R llt2]! ]04       ! R  R!]PX                  4      4       t3 ! R" R#]4      t4 ! R$ R%]PX                  4      t5]! ! R& R']4      4       t6]! ! R( R)]64      4       t7]! ! R* R+]6]
4      4       t8. R.Ot9R# )/é    )ÚCallableN)Únn)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚBitNetConfigÚRMSNormc                   óZ   a a€ ] tR t^+t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚBitNetRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/bitnet/modeling_bitnet.pyÚ__annotate__ÚBitNetRMSNorm.__annotate__-   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z,
BitNetRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer#   Ú	__class__s   &&&€r(   r.   ÚBitNetRMSNorm.__init__-   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r"   Úhidden_statesr$   )r0   ÚTensor)r&   r'   s   "€r(   r)   r*   5   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r+   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r"   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor0   Úfloat32ÚpowÚmeanÚrsqrtr3   r2   )r4   r9   Úinput_dtypeÚvariances   &&  r(   ÚforwardÚBitNetRMSNorm.forward5   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r+   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler2   Úshaper3   )r4   s   &r(   Ú
extra_reprÚBitNetRMSNorm.extra_repr<   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr+   )r3   r2   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r.   rF   rK   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r6   r'   s   @@r(   r    r    +   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr+   r    c                   óD   a a€ ] tR t^@t oV3R lV 3R lltR tRtVtV ;t# )Ú	BitNetMLPc                ó    <€ V ^8„  d   QhRS[ /# ©r"   Úconfigr   )r&   r'   s   "€r(   r)   ÚBitNetMLP.__annotate__A   s   ø€ ÷ 	]ñ 	]™|ñ 	]r+   c                ó$  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        \        VP                  VP                  R7      V n        R# )F©Úbias©r#   N)r-   r.   rY   r5   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fnr    Úrms_norm_epsÚffn_sub_norm©r4   rY   r6   s   &&€r(   r.   ÚBitNetMLP.__init__A   s¿   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŒÜ)¨&×*BÑ*BÈ×H[ÑH[Ô\ˆÖr+   c           	     ó®   € V P                  V P                  V P                  V P                  V4      4      V P	                  V4      ,          4      4      pV# ©N)rc   rg   re   ra   rb   )r4   Úxrc   s   && r(   rF   ÚBitNetMLP.forwardL   sF   € Ø—N‘N 4×#4Ñ#4°T·[±[ÀÇÁÐPQÓARÓ5SÐVZ×VbÑVbÐcdÓVeÕ5eÓ#fÓgˆ	ØÐr+   )re   rY   rc   rg   ra   r5   r_   rb   ©	rM   rN   rO   rP   r.   rF   rQ   rR   rS   rT   s   @@r(   rV   rV   @   s   ù‡ € ÷	]ó 	]÷ò r+   rV   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..N©Údimr=   )rJ   r0   Úcat)rl   Úx1Úx2s   &  r(   Úrotate_halfru   Q   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r+   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezeru   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r(   Úapply_rotary_pos_embr€   X   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr+   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r"   r9   Ún_repr$   )r0   r:   Úint)r&   s   "r(   r)   r)   r   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur+   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)ºNNNr…   Nr…   r…   )rJ   ÚexpandÚreshape)r9   r‚   ÚbatchÚnum_key_value_headsÚslenÚhead_dims   &&    r(   Ú	repeat_kvrŒ   r   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr+   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r"   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutÚkwargs)r   ÚModuler0   r:   r%   r   r   )r&   s   "r(   r)   r)   ~   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r+   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r"   )rq   r>   )ÚpÚtrainingr=   )rŒ   Únum_key_value_groupsr0   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr@   r?   r>   r”   r™   Ú
contiguous)rŽ   r�   r�   r‘   r’   r“   r”   r•   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r(   Úeager_attention_forwardr¤   ~   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r+   c                   óX   a a€ ] tR t^—t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚBitNetAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r"   rY   Ú	layer_idx©r   rƒ   )r&   r'   s   "€r(   r)   ÚBitNetAttention.__annotate__›   s   ø€ ÷ Xñ X™|ð X¹ñ Xr+   c                óî  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        \)        VP
                  VP*                  R7      V n        R# )r‹   Tr\   r^   Ng      à¿)r-   r.   rY   r©   Úgetattrr5   Únum_attention_headsr‹   r‰   rš   r“   Úattention_dropoutÚ	is_causalr   r`   Úattention_biasÚq_projÚk_projÚv_projÚo_projr    rf   Úattn_sub_norm©r4   rY   r©   r6   s   &&&€r(   r.   ÚBitNetAttention.__init__›   sf  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô +¨6×+=Ñ+=À6×CVÑCVÔWˆÖr+   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r"   r9   Úposition_embeddingsr’   NÚpast_key_valuesr•   r$   )r0   r:   rI   r   r   r   )r&   r'   s   "€r(   r)   r«   ³   s‡   ø€ ÷ ')ñ ')á—|‘|ð')ñ #¡5§<¡<±·±Ð#=Õ>ð')ñ Ÿ™ tÕ+ð	')ñ
  �ð')ñ Ñ-Õ.ð')ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ')r+   c                ó2  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pV P+                  V4      pWï3# )Nr”   ç        r“   r=   )rJ   r‹   r²   Úviewrœ   r³   r´   r€   Úupdater©   r   Úget_interfacerY   Ú_attn_implementationr¤   r™   r¯   r“   r‡   rŸ   r¶   rµ   )r4   r9   rº   r’   r»   r•   Úinput_shapeÚhidden_shapeÚquery_statesr    r¡   r{   r|   Úattention_interfacer£   r¢   s   &&&&&,          r(   rF   ÚBitNetAttention.forward³   s—  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ×(Ñ(¨Ó5ˆØ—k‘k +Ó.ˆØÐ(Ð(r+   )r¯   r¶   rY   r‹   r°   r³   r©   rš   rµ   r²   r“   r´   rk   )
rM   rN   rO   rP   Ú__doc__r.   rF   rQ   rR   rS   rT   s   @@r(   r¦   r¦   —   s#   ù‡ € áG÷Xó X÷0')÷ ')ò ')r+   r¦   c                   óT   a a€ ] tR t^Ýt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚBitNetDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# r¨   rª   )r&   r'   s   "€r(   r)   ÚBitNetDecoderLayer.__annotate__Þ   s   ø€ ÷ cñ c™|ð c¹ñ cr+   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))rY   r©   r^   N)r-   r.   r5   r¦   Ú	self_attnrV   Úmlpr    rf   Úinput_layernormÚpost_attention_layernormr·   s   &&&€r(   r.   ÚBitNetDecoderLayer.__init__Þ   sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä(°ÔLˆŒä˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÖ%r+   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r"   r9   r’   NÚposition_idsr»   Ú	use_cacherº   r•   r$   )r0   r:   Ú
LongTensorr   ÚboolrI   r   r   )r&   r'   s   "€r(   r)   rË   è   sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr+   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r9   r’   rÓ   r»   rÔ   rº   © )rÏ   rÍ   rÐ   rÎ   )
r4   r9   r’   rÓ   r»   rÔ   rº   r•   ÚresidualÚ_s
   &&&&&&&,  r(   rF   ÚBitNetDecoderLayer.forwardè   s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr+   )r5   rÏ   rÎ   rÐ   rÍ   )NNNFNrn   rT   s   @@r(   rÉ   rÉ   Ý   s    ù‡ € ÷có c÷÷ ò r+   rÉ   c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚBitNetRotaryEmbeddingi  Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   Ú"BitNetRotaryEmbedding.__annotate__
  s   ø€ ÷ Oñ O™|ñ Or+   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r-   r.   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrY   Úrope_parametersrã   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r4   rY   rÞ   Úrope_init_fnrí   r6   s   &&&  €r(   r.   ÚBitNetRotaryEmbedding.__init__	  s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr+   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r"   rY   r$   )r   rI   r0   r:   r%   )r&   r'   s   "€r(   r)   rá     s.   ø€ ÷ 5ñ 5±ð 5ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ 5r+   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetar‹   Ng      ð?©r>   )ré   r­   r5   r®   r0   Úaranger%   r?   )rY   rÞ   r•   Úbaserq   Úattention_factorrí   s   &&,    r(   rê   Ú5BitNetRotaryEmbedding.compute_default_rope_parameters  s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r+   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)r>   rÞ   ÚmpsÚcpuF)Údevice_typeÚenabledrp   rõ   )Nr…   Nr=   )r…   Nr…   )rí   r†   rJ   r?   r0   r%   rÞ   Ú
isinstanceÚtypeÚstrr   rœ   rr   r{   rë   r|   r>   )
r4   rl   rÓ   Úinv_freq_expandedÚposition_ids_expandedrý   ÚfreqsÚembr{   r|   s
   &&&       r(   rF   ÚBitNetRotaryEmbedding.forward.  s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rë   rY   rí   rç   rï   rè   rã   rk   )rM   rN   rO   rP   r   r.   Ústaticmethodrê   r0   Úno_gradr   rF   rQ   rR   rS   rT   s   @@r(   rÝ   rÝ     su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r+   rÝ   c                   ó`   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltR	tV tR
# )ÚBitNetPreTrainedModeliA  ÚmodelTrÉ   r»   r9   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rX   r   )r&   r'   s   "€r(   r)   Ú"BitNetPreTrainedModel.__annotate__A  s   ø‡ ‚ áÑò r+   rØ   N)rM   rN   rO   rP   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrÉ   r¦   Ú_can_record_outputsÚ__annotate_func__rQ   rR   )r'   s   @r(   r
  r
  A  s\   ø‡ € ð  ÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ+Ø�oðÐ÷ ƒ r+   r
  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚBitNetModeliT  c                ó    <€ V ^8„  d   QhRS[ /# rX   r   )r&   r'   s   "€r(   r)   ÚBitNetModel.__annotate__V  s   ø€ ÷ ñ ™|ñ r+   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r^   ©rY   FN)r-   r.   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr5   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrÉ   Úlayersr    rf   ÚnormrÝ   Ú
rotary_embÚgradient_checkpointingÚ	post_initr·   s   && €r(   r.   ÚBitNetModel.__init__V  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°vÔ>ˆŒØ&+ˆÔ#ð 	�‰Öùò es   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r"   Ú	input_idsNr’   rÓ   r»   Úinputs_embedsrÔ   r•   r$   )	r0   rÕ   r:   r   ÚFloatTensorrÖ   r   r   r   )r&   r'   s   "€r(   r)   r  i  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r+   c                óˆ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr  )rÞ   )rY   r0  r’   r»   rÓ   )rÓ   r’   rº   rÓ   r»   rÔ   )Úlast_hidden_stater»   )Ú
ValueErrorr$  r   rY   Úget_seq_lengthr0   rö   rJ   rÞ   rx   r   r*  r(  r'  r)  r   )r4   r/  r’   rÓ   r»   r0  rÔ   r•   Úpast_seen_tokensÚcausal_maskr9   rº   Údecoder_layers   &&&&&&&,     r(   rF   ÚBitNetModel.forwardf  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r+   )r$  r+  r(  r)  r!  r*  r"  )NNNNNN)rM   rN   rO   rP   r.   r   r   r   rF   rQ   rR   rS   rT   s   @@r(   r  r  T  s=   ù‡ € ÷ó ð   ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r+   r  c                   ót   a a€ ] tR tRt oRR/tRtRtRR/tV 3R lt]	]
RV3R lR	 ll4       4       tR
tVtV ;t# )ÚBitNetForCausalLMiž  zlm_head.weightzmodel.embed_tokens.weightNÚlm_headÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )Fr\   N)
r-   r.   r  r  r"  r   r`   r5   r<  r,  rh   s   &&€r(   r.   ÚBitNetForCausalLM.__init__¥  sU   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör+   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r"   r/  Nr’   rÓ   r»   r0  ÚlabelsrÔ   Úlogits_to_keepr•   r$   )
r0   rÕ   r:   r   r1  rÖ   rƒ   r   r   r   )r&   r'   s   "€r(   r)   ÚBitNetForCausalLM.__annotate__°  s»   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ × Ñ  4Õ'ð;
ñ ˜$•;ð;
ñ ™eŸl™lÕ*ð;
ñ Ñ+Õ,ð;
ñ 
 ñ;
r+   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )a¤  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, transformers.,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, transformers., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, BitNetForCausalLM

>>> model = BitNetForCausalLM.from_pretrained("microsoft/bitnet-b1.58-2B-4T")
>>> tokenizer = AutoTokenizer.from_pretrained("microsoft/bitnet-b1.58-2B-4T")

>>> prompt = f'<|begin_of_text|>User: Hey, are you conscious? Can you talk to me?<|eot_id|>Assistant: '
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=100)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"User: Hey, are you conscious? Can you talk to me?Assistant: No, I'm not conscious. I'm an artificial intelligence designed to assist with information and tasks. How can I help you today?"
```r/  r’   rÓ   r»   r0  rÔ   Nr…   ÚlogitsrA  r"  )ÚlossrE  r»   r9   r  rØ   )r  r3  rÿ   rƒ   Úslicer<  Úloss_functionrY   r"  r   r»   r9   r  )r4   r/  r’   rÓ   r»   r0  rA  rÔ   rB  r•   Úoutputsr9   Úslice_indicesrE  rF  s   &&&&&&&&&,     r(   rF   ÚBitNetForCausalLM.forward®  s   € ðH ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r+   )r<  r  r"  )NNNNNNNr   )rM   rN   rO   rP   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr.   r   r   rF   rQ   rR   rS   rT   s   @@r(   r;  r;  ž  sQ   ù‡ € à*Ð,GÐHÐØ€HØ€HØÐ/Ð0€Jõð Ø÷;
ñ ;
ó ó ÷;
ð ;
r+   r;  )é   )r½   )r;  r  r
  ):Úcollections.abcr   r0   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   r
   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_bitnetr   r–   r    rV   ru   r€   rŒ   r¤   r¦   rÉ   rÝ   r
  r  r;  Ú__all__rØ   r+   r(   Ú<module>rc     sQ  ðõ( %ã Ý å !ß .Ý )ß LÝ /Ý BÝ 9ß Oß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý .ñ ˜YÓ'ôJ�B—I‘Ió Jó (ðJô(�—	‘	ô ò"(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ôB)�b—i‘ió B)ó +ðB)ôJ(Ð3ô (ôV6<˜BŸI™Iô 6<ðr ô˜Oó ó ðð$ ôF
Ð'ó F
ó ðF
ðR ôL
Ð-¨ó L
ó ðL
ò^ H‚r+   