+
    œ°³j“%  ã                   óô  € ^ RI Ht ^ RIt^ RIHt ^ RIHt ^RIHt ^RIH	t	 ^RI
Ht ^RIHt ^R	IHt ^R
IHt ^RIHtHtHt ^RIHtHtHtHtHtHtHtHtHtH t  ^RI!H"t" ]PF                  ! ]$4      t%]! RR7      ] ! R R]4      4       4       t& ! R R]"4      t' ! R R]4      t( ! R R]4      t) ! R R]4      t* ! R R]4      t+ ! R R]4      t, ! R R]4      t- ! R  R!]4      t. ! R" R#]4      t/. R$Ot0R# )%é    )ÚCallableN)Ústrict)Únn)ÚACT2CLS)ÚCache)ÚPreTrainedConfig)ÚRopeParameters)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForTokenClassificationÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚNemotronMLPz!swiss-ai/Apertus-8B-Instruct-2509)Ú
checkpointc                   óæ   a a€ ] tR t^.t oRtRtR.tRtRRRRRRR	R
RR
RRRRRR/tRR.R.3RRR.R.3RR.R.3/t	Rt
RtRt^ t^ tRtRtRtRtRtRt^t^t^tR tRtR tR!tV 3R" ltV3R# ltR$tVtV ;t # )%ÚApertusConfigaV  
```python
>>> from transformers import ApertusModel, ApertusConfig

>>> # Initializing a Apertus-8B style configuration
>>> configuration = ApertusConfig()

>>> # Initializing a model from the Apertus-8B style configuration
>>> model = ApertusModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```ÚapertusÚpast_key_valuesç    `ãfAzlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.self_attn.o_projÚrowwisezlayers.*.mlp.up_projzlayers.*.mlp.down_projÚembed_tokensÚ	input_idsÚinputs_embedsÚlayersÚhidden_statesÚattention_maskÚnormi   i   i 8  NÚxielui   g{®Gáz”?gñhãˆµøä>TFç        c                ó¨   <€ V P                   f   V P                  V n         V P                  f   RRRRRRRRR	R
RR/V n        \        SV `  ! R/ VB  R # )NÚ	rope_typeÚllama3Ú
rope_thetar   Úfactorg       @Ú original_max_position_embeddingsi    Úlow_freq_factorg      ð?Úhigh_freq_factorg      @© )Únum_key_value_headsÚnum_attention_headsÚrope_parametersÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s   &,€Úl/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/apertus/modular_apertus.pyr9   ÚApertusConfig.__post_init__e   sc   ø€ Ø×#Ñ#Ò+Ø'+×'?Ñ'?ˆDÔ$à×ÑÒ'à˜XØ˜jØ˜#Ø2°DØ! 3Ø" Cð$ˆDÔ ô 	‰ÒÑ' Ô'ó    c                óx  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ R,          ;R&   S[;R&   S[ ;R	&   S[;R
&   S[;R&   S[;R&   S[ R,          ;R&   S[ R,          ;R&   S[ S[S[ ,          ,          R,          ;R&   S[;R&   S[S[,          R,          ;R&   S[;R&   S[S[ ,          ;R&   # )é   Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersr6   Nr5   Ú
hidden_actÚmax_position_embeddingsÚinitializer_rangeÚrms_norm_epsÚ	use_cacheÚpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddingsr7   Úattention_biasÚattention_dropout)ÚintÚstrÚfloatÚboolÚlistr	   Údict)ÚformatÚ__classdict__s   "€r=   Ú__annotate__ÚApertusConfig.__annotate__.   s:  ø‡ ‚ ñH ÑñI ñJ ÑñK ñL Ñ"ñM ñN ÑñO ñP Ñ!ñQ ñR ˜t�Ñ*ñS ñT ÑñU ñV !Ñ(ñW ñX Ñ#ñY ñZ Ññ[ ñ\ Ññ] ñ^ ˜•*Ñ ñ_ ñ` ˜•*Ñ ña ñb ™™S�	•/ DÕ(Ñ,ñc ñd Ñ%ñe ñf $¡dÕ*¨TÕ1Ñ8ñg ñh Ñ ñi ñj ™s•{Ñ(òk r?   )r5   r7   )!Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planrB   rC   rD   rE   r6   r5   rF   rG   rH   rI   rJ   rK   rL   rM   rN   r7   rO   rP   r9   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r<   rX   s   @@r=   r   r   .   sþ   ù‡ € ñð €JØ#4Ð"5ÐØ€Mà# YØ# YØ# YØ#Ð%EØ#Ð%EØ# YØ 	Ø  )ð	Ðð 	˜+˜¨Ð(9Ð:Ø�OÐ%5Ð6¸Ð8IÐJØ�/Ð" _Ð$5Ð6ðÐð €JØ€KØ"ÐØÐØ!ÐØ&*ÐØ€JØ#(ÐØ#ÐØ€LØ€IØ €LØ €LØ+,€LØ %ÐØ48€OØ €NØ%(Ðõ(÷o … r?   r   c                   ó2   a a€ ] tR t^ut oV 3R ltRtVtV ;t# )Ú
ApertusMLPc                óZ  <€ \         SV `  V4       \        P                  ! V P                  V P
                  R R7      V n        \        P                  ! V P
                  V P                  R R7      V n        VP                  R8X  d&   \        R,          ! VP                  R7      V n        R# R# )F)Úbiasr*   )ÚdtypeN)r8   Ú__init__r   ÚLinearrC   rD   Úup_projÚ	down_projrF   r   rn   Úact_fn)r:   Úconfigr<   s   &&€r=   ro   ÚApertusMLP.__init__v   sz   ø€ Ü‰Ñ˜Ô Ü—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒØ×Ñ Ô'Ü! 'Ö*°·±Ô>ˆDŽKñ (r?   )rs   rr   rq   )r[   r\   r]   r^   ro   rf   rg   rh   ri   s   @@r=   rk   rk   u   s   ù‡ € ÷?õ ?r?   rk   c                   ó   € ] tR t^~tRtR# )ÚApertusRMSNormr4   N©r[   r\   r]   r^   rf   r4   r?   r=   rw   rw   ~   ó   † Ûr?   rw   c                   ó   € ] tR t^‚tRtR# )ÚApertusRotaryEmbeddingr4   Nrx   r4   r?   r=   r{   r{   ‚   ry   r?   r{   c                   óX   a a€ ] tR t^†t oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚApertusAttentionc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )rA   rt   Ú	layer_idxN©r   rQ   )rW   rX   s   "€r=   rY   ÚApertusAttention.__annotate__‡   s#   ø€ ÷ Iñ I™}ð I¹¸t½ñ Ir?   c                óº   <€ \         SV `  W4       \        V P                  VP                  4      V n        \        V P                  VP                  4      V n        R # ©N)r8   ro   rw   Úhead_dimrI   Úq_normÚk_norm©r:   rt   r   r<   s   &&&€r=   ro   ÚApertusAttention.__init__‡   s@   ø€ Ü‰Ñ˜Ô+Ü$ T§]¡]°F×4GÑ4GÓHˆŒÜ$ T§]¡]°F×4GÑ4GÓHˆŽr?   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )rA   r'   Úposition_embeddingsr(   Nr   r;   Úreturn)ÚtorchÚTensorÚtupler   r   r   )rW   rX   s   "€r=   rY   r�   Œ   s‚   ø€ ÷ ()ñ ()á—|‘|ð()ñ #¡5§<¡<±·±Ð#=Õ>ð()ñ Ÿ™ tÕ+ð	()ñ
  �ð()ñ Ñ+Õ,ð()ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ()r?   c                óT  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      pV P                  V	4      p	Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \         4      pT! V VV	V
V3RV P"                  '       g   RMV P$                  RV P&                  /VB w  rïVP(                  ! . VORN5!  P+                  4       pV P-                  V4      pWï3# )NÚdropoutr+   Úscalingéÿÿÿÿ)Úshaper„   Úq_projÚviewÚ	transposeÚk_projÚv_projr…   r†   r   Úupdater   r
   Úget_interfacert   Ú_attn_implementationr   ÚtrainingrP   r‘   ÚreshapeÚ
contiguousÚo_proj)r:   r'   rŠ   r(   r   r;   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss   &&&&&,          r=   ÚforwardÚApertusAttention.forwardŒ   s£  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r?   )r†   r…   rƒ   ©	r[   r\   r]   r^   ro   rª   rf   rg   rh   ri   s   @@r=   r}   r}   †   s    ù‡ € ÷Iõ I÷
()÷ ()ò ()r?   r}   c                   óT   a a€ ] tR t^·t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚApertusDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )rA   rt   r   r€   )rW   rX   s   "€r=   rY   Ú ApertusDecoderLayer.__annotate__¸   s   ø€ ÷ *ñ *™}ð *¹ñ *r?   c                óÆ   <€ \         SV `  W4       \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        V =V =R# ))ÚepsN)	r8   ro   rw   rC   rI   Úattention_layernormÚfeedforward_layernormÚinput_layernormÚpost_attention_layernormr‡   s   &&&€r=   ro   ÚApertusDecoderLayer.__init__¸   sR   ø€ Ü‰Ñ˜Ô+Ü#1°&×2DÑ2DÈ&×J]ÑJ]Ô#^ˆÔ Ü%3°F×4FÑ4FÈF×L_ÑL_Ô%`ˆÔ"àÐ ØÒ)r?   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
rA   r'   r(   NÚposition_idsr   rJ   rŠ   r;   r‹   )rŒ   r�   Ú
LongTensorr   rT   rŽ   r   r   )rW   rX   s   "€r=   rY   r°   À   sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr?   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r'   r(   r¹   r   rJ   rŠ   r4   )r³   Ú	self_attnr´   Úmlp)
r:   r'   r(   r¹   r   rJ   rŠ   r;   ÚresidualÚ_s
   &&&&&&&,  r=   rª   ÚApertusDecoderLayer.forwardÀ   s©   € ð !ˆØ×0Ñ0°Ó?ˆØŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×2Ñ2°=ÓAˆØŸ™ Ó/ˆØ Õ0ˆØÐr?   )r³   r´   )NNNFNr¬   ri   s   @@r=   r®   r®   ·   s   ù‡ € ÷*ó *÷÷ ò r?   r®   c                   ó   € ] tR t^ßtRtR# )ÚApertusPreTrainedModelr4   Nrx   r4   r?   r=   rÂ   rÂ   ß   ry   r?   rÂ   c                   ó   € ] tR t^ãtRtR# )ÚApertusModelr4   Nrx   r4   r?   r=   rÄ   rÄ   ã   ry   r?   rÄ   c                   ó2   a a€ ] tR t^çt oV 3R ltRtVtV ;t# )ÚApertusForCausalLMc                ó$   <€ \         SV `  ! R/ VB # )a
  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, ApertusForCausalLM

>>> model = ApertusForCausalLM.from_pretrained("swiss-ai/Apertus-8B-Instruct-2509")
>>> tokenizer = AutoTokenizer.from_pretrained("swiss-ai/Apertus-8B-Instruct-2509")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r4   )r8   rª   )r:   Úsuper_kwargsr<   s   &,€r=   rª   ÚApertusForCausalLM.forwardè   s   ø€ ô. ‰wŠÑ. Ñ.Ð.r?   r4   )r[   r\   r]   r^   rª   rf   rg   rh   ri   s   @@r=   rÆ   rÆ   ç   s   ù‡ € ÷/õ /r?   rÆ   c                   ó   € ] tR tRtRtR# )ÚApertusForTokenClassificationi  r4   Nrx   r4   r?   r=   rË   rË     ry   r?   rË   )r   rÄ   rÆ   rË   rÂ   )1Úcollections.abcr   rŒ   Úhuggingface_hub.dataclassesr   r   Úactivationsr   Úcache_utilsr   Úconfiguration_utilsr   Úmodeling_rope_utilsr	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   r   r   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Únemotron.modeling_nemotronr   Ú
get_loggerr[   Úloggerr   rk   rw   r{   r}   r®   rÂ   rÄ   rÆ   rË   Ú__all__r4   r?   r=   Ú<module>rÚ      sú   ðõ %ã Ý .Ý å "Ý  Ý 3Ý 1Ý 5Ý &ß @Ñ @÷÷ ÷ õ 5ð 
×	Ò	˜HÓ	%€ñ Ð>Ô?ØôB(Ð$ó B(ó ó @ðB(ôJ?�ô ?ô	�\ô 	ô	Ð1ô 	ô.)�~ô .)ôb%Ð+ô %ôP	Ð1ô 	ô	�:ô 	ô/Ð)ô /ô6	Ð$?ô 	ò‚r?   