+
    œ°³jþx  ã                  ó"  € ^ RI Ht ^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RI	t	^ RI
Ht ^ RIHt ^RIHt ^RIHt ^RIHtHt ^R	IHtHt ^R
IHtHtHtHt ^RIHtHtH t  ^RI!H"t" ^RI#H$t$ ^RI%H&t&H't'H(t(H)t)H*t* ^RI+H,t, ]'       d   ^RIH-t- ^RI.H/t/ ^RI0H1t1 ]*Pd                  ! ]34      t4 ! R R]Pj                  4      t6]( ! R R4      4       t7]( ! R R4      4       t8]( ! R R4      4       t9 ! R R]Pj                  4      t: ! R R]4      t;R# )é    )ÚannotationsN)Úpartial)ÚTYPE_CHECKING)Ú	safe_open)ÚCache)Úget_model_conversion_mapping)ÚWeightRenamingÚ$convert_and_load_state_dict_in_model)Ú&LAYER_PATTERN_TO_MASK_FUNCTION_MAPPINGÚcreate_causal_mask)ÚBaseModelOutputWithPastÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚLoadStateDictConfigÚPreTrainedModelÚ_get_resolved_checkpoint_files)Ú	AutoModel)ÚUnpack)ÚContextManagersÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úlog_state_dict_report)ÚMtpCache)ÚPreTrainedConfig)ÚLogitsProcessorListc                  ó4   a € ] tR t^4tRtRtRtV 3R ltRtV ;t	# )ÚGradientCheckpointingLayeraÜ  Base class for layers with gradient checkpointing.

This class enables gradient checkpointing functionality for a layer. By default, gradient checkpointing is disabled
(`gradient_checkpointing = False`). When `model.set_gradient_checkpointing()` is called, gradient checkpointing is
enabled by setting `gradient_checkpointing = True` and assigning a checkpointing function to `_gradient_checkpointing_func`.

Important:

    When using gradient checkpointing with `use_reentrant=True`, inputs that require gradients (e.g. hidden states)
    must be passed as positional arguments (`*args`) rather than keyword arguments to properly propagate gradients.

    Example:

        ```python
        >>> # Correct - hidden_states passed as positional arg
        >>> out = self.layer(hidden_states, attention_mask=attention_mask)

        >>> # Incorrect - hidden_states passed as keyword arg
        >>> out = self.layer(hidden_states=hidden_states, attention_mask=attention_mask)
        ```
Fc                	óX  <€ V P                   '       Ed   V P                  '       dö   R pV P                  P                  pRV R2pRV9   d    VR,          '       d   R VR&   VR,          pRpV P                  '       gE   RV9   d   VR,          e   RVR&   VR,          pRpR	V9   d   VR	,          e   RVR	&   VR
,          pRpV'       d.   VP                  R4      R,           p\        P                  V4       V P                  ! \        \        SV `,  3/ VB .VO5!  # \        SV `,  ! V/ VB # )Fz7Caching is incompatible with gradient checkpointing in z	. SettingÚ	use_cachez `use_cache=False`,TÚpast_key_valuesNz `past_key_values=None`,Ú
layer_pastz `layer_past=None`,Ú,Ú.)Úgradient_checkpointingÚtrainingÚ	__class__Ú__name__Ú_can_checkpoint_with_cacheÚrstripÚloggerÚwarning_onceÚ_gradient_checkpointing_funcr   ÚsuperÚ__call__)ÚselfÚargsÚkwargsÚdo_warnÚ
layer_nameÚmessager)   s   &*,   €Ú]/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/modeling_layers.pyr1   Ú#GradientCheckpointingLayer.__call__P   s/  ø€ Ø×&×&Ñ&¨4¯=¯=¨=ØˆGØŸ™×0Ñ0ˆJØOÐPZÈ|Ð[dÐeˆGà˜fÔ$¨°×)<Ô)<Ø&+��{Ñ#ØÐ0Õ0�Ø�à×2×2Ð2à$¨Ô.°6Ð:KÕ3LÒ3XØ04�FÐ,Ñ-ØÐ9Õ9�GØ"�Gà 6Ô)¨f°\Õ.BÒ.NØ+/�F˜<Ñ(ØÐ4Õ4�GØ"�G÷ Ø!Ÿ.™.¨Ó-°Õ3�Ü×#Ñ# GÔ,à×4Ò4´W¼U¹WÑ=MÑ5XÐQWÑ5XÐ`Ð[_Ó`Ð`Ü‰wÒ Ð0¨Ñ0Ð0ó    © )
r*   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r'   r+   r1   Ú__static_attributes__Ú__classcell__©r)   s   @r8   r    r    4   s"   ø† ñð, #Ðð "'Ð÷1ó 1r:   r    c                  óP   a € ] tR t^ptRtV 3R lt]]RR R ll4       4       tRt	V ;t
# )Ú GenericForSequenceClassificationÚmodelc                	ó<  <€ \         SV `  V4       VP                  V n        \        W P                  \
        P                  ! V4      4       \        P                  ! VP                  4       P                  V P                  R R7      V n        V P                  4        R# )F©ÚbiasN)r0   Ú__init__Ú
num_labelsÚsetattrÚbase_model_prefixr   Úfrom_configÚnnÚLinearÚget_text_configÚhidden_sizeÚscoreÚ	post_init©r2   Úconfigr)   s   &&€r8   rI   Ú)GenericForSequenceClassification.__init__t   sl   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä�×,Ñ,¬i×.CÒ.CÀFÓ.KÔLÜ—Y’Y˜v×5Ñ5Ó7×CÑCÀTÇ_Á_Ð[`ÔaˆŒ
ð 	�‰Ör:   c               ó<   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRR/	# )é   Ú	input_idsútorch.LongTensor | NoneÚattention_maskútorch.Tensor | NoneÚposition_idsr#   úCache | NoneÚinputs_embedsútorch.FloatTensor | NoneÚlabelsr"   úbool | Noner4   úUnpack[TransformersKwargs]Úreturnr   r;   )Úformats   "r8   Ú__annotate__Ú-GenericForSequenceClassification.__annotate__€   sd   € ÷ 8
ñ 8
à*ð8
ð ,ð8
ð .ð	8
ð
 &ð8
ð 0ð8
ð (ð8
ð ð8
ð -ð8
ð 
*ñ8
r:   c                	óL  € \        W P                  4      ! V3R VRVRVRVRV/VB p	V	P                  p
V P                  V
4      pVe   VP                  ^ ,          pMVP                  ^ ,          pV P
                  P                  4       P                  f   V^8w  d   \        R4      hV P
                  P                  4       P                  f   RpMÚVe©   WP
                  P                  4       P                  8g  P                  VP                  \        P                  4      p\        P                  ! VP                  R,          VP                  \        P                  R7      pWþ,          P                  R4      pM.Rp\        P!                  V P"                  P$                   R24       V\        P                  ! WËP                  R	7      V3,          pRpVe   V P'                  W¶VV P
                  R
7      p\)        VVV	P*                  V	P,                  V	P.                  R7      # )r[   r]   r#   r_   r"   Nz=Cannot handle batch sizes > 1 if no padding token is defined.)ÚdeviceÚdtypezŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`©ri   )Úlogitsra   Úpooled_logitsrU   )Úlossrl   r#   Úhidden_statesÚ
attentionséÿÿÿÿ)ÚgetattrrL   Úlast_hidden_staterR   ÚshaperU   rP   Úpad_token_idÚ
ValueErrorÚtori   ÚtorchÚint32ÚarangeÚargmaxr-   r.   r)   r*   Úloss_functionr   r#   ro   rp   )r2   rY   r[   r]   r#   r_   ra   r"   r4   Útransformer_outputsro   rl   Ú
batch_sizeÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesrm   rn   s   &&&&&&&&,         r8   ÚforwardÚ(GenericForSequenceClassification.forward~   sþ  € ô 8?¸t×E[ÑE[Ô7\Øñ8
à)ð8
ð &ð8
ð ,ð	8
ð
 (ð8
ð  ð8
ð ñ8
Ðð ,×=Ñ=ˆØ—‘˜MÓ*ˆàÒ Ø"Ÿ™¨Õ+‰Jà&×,Ñ,¨QÕ/ˆJà�;‰;×&Ñ&Ó(×5Ñ5Ò=À*ÐPQÄ/ÜÐ\Ó]Ð]Ø�;‰;×&Ñ&Ó(×5Ñ5Ò=Ø!#ÑØÒ"à%¯©×)DÑ)DÓ)F×)SÑ)SÑS×WÑWÐX^×XeÑXeÔgl×grÑgrÓsˆLÜ!ŸLšL¨¯©¸Õ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/Õ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¿}¹}ÔMÐOaÐaÕbˆàˆØÒØ×%Ñ%¨VÐR_Ðhl×hsÑhsÐ%ÓtˆDä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r:   )rJ   rR   ©NNNNNNN©r*   r<   r=   r>   rL   rI   r   r   r‚   r@   rA   rB   s   @r8   rD   rD   p   s*   ø† àÐõð Øö8
ó ó ö8
r:   rD   c                  ó\   a € ] tR t^»tRtV 3R ltR tR t]]	RR R ll4       4       t
RtV ;t# )	ÚGenericForQuestionAnsweringrE   c                	óæ   <€ \         SV `  V4       \        W P                  \        P
                  ! V4      4       \        P                  ! VP                  ^4      V n	        V P                  4        R# )rX   N)r0   rI   rK   rL   r   rM   rN   rO   rQ   Ú
qa_outputsrS   rT   s   &&€r8   rI   Ú$GenericForQuestionAnswering.__init__¿   sO   ø€ Ü‰Ñ˜Ô ä�×,Ñ,¬i×.CÒ.CÀFÓ.KÔLÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör:   c                	ó@   € \        W P                  4      P                  # ©N©rr   rL   Úembed_tokens)r2   s   &r8   Úget_input_embeddingsÚ0GenericForQuestionAnswering.get_input_embeddingsÈ   s   € Ü�t×3Ñ3Ó4×AÑAÐAr:   c                	ó:   € V\        W P                  4      n        R # rŒ   r�   )r2   Úvalues   &&r8   Úset_input_embeddingsÚ0GenericForQuestionAnswering.set_input_embeddingsË   s   € Ø=BŒ�×,Ñ,Ó-Ö:r:   c               ó<   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRR/	# )rX   rY   rZ   r[   r\   r]   r#   r^   r_   r`   Ústart_positionsÚend_positionsr4   rc   rd   r   r;   )re   s   "r8   rf   Ú(GenericForQuestionAnswering.__annotate__Ð   sd   € ÷ %
ñ %
à*ð%
ð ,ð%
ð .ð	%
ð
 &ð%
ð 0ð%
ð 1ð%
ð /ð%
ð -ð%
ð 
&ñ%
r:   c                	ó®  € \        W P                  4      ! V3R VRVRVRV/VB p	V	P                  p
V P                  V
4      pVP	                  ^RR7      w  rÍVP                  R4      P                  4       pVP                  R4      P                  4       pRpVe   Ve   V P                  ! WÍWg3/ VB p\        VVVV	P                  V	P                  R7      # )r[   r]   r#   r_   ©ÚdimN)rn   Ústart_logitsÚ
end_logitsro   rp   rq   )rr   rL   rs   r‰   ÚsplitÚsqueezeÚ
contiguousr|   r   ro   rp   )r2   rY   r[   r]   r#   r_   r–   r—   r4   ÚoutputsÚsequence_outputrl   rœ   r�   rn   s   &&&&&&&&,      r8   r‚   Ú#GenericForQuestionAnswering.forwardÎ   sþ   € ô ,3°4×9OÑ9OÔ+PØñ,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð ñ,
ˆð "×3Ñ3ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆØÒ&¨=Ò+DØ×%Ò% lÀÑiÐbhÑiˆDä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r:   )r‰   r„   )r*   r<   r=   r>   rL   rI   r�   r“   r   r   r‚   r@   rA   rB   s   @r8   r‡   r‡   »   s6   ø† àÐõòBòCð Øö%
ó ó ö%
r:   r‡   c                  óP   a € ] tR t^øtRtV 3R lt]]RR R ll4       4       tRt	V ;t
# )ÚGenericForTokenClassificationrE   c           	     	óü  <€ \         SV `  V4       VP                  V n        \        W P                  \
        P                  ! V4      4       \        VR R4      e   VP                  pM\        VRR4      e   VP                  pMRp\        P                  ! V4      V n        \        P                  ! VP                  4       P                  VP                  \        VRR4      R7      V n        V P#                  4        R# )Úclassifier_dropoutNÚhidden_dropoutgš™™™™™¹?Útoken_classification_biasTrG   )r0   rI   rJ   rK   rL   r   rM   rr   r§   r¨   rN   ÚDropoutÚdropoutrO   rP   rQ   rR   rS   )r2   rU   r§   r)   s   && €r8   rI   Ú&GenericForTokenClassification.__init__ü   sË   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä�×,Ñ,¬i×.CÒ.CÀFÓ.KÔLÜ�6Ð/°Ó6ÒBØ!'×!:Ñ!:ÑÜ�VÐ-¨tÓ4Ò@Ø!'×!6Ñ!6Ñà!$ÐÜ—z’zÐ"4Ó5ˆŒÜ—Y’YØ×"Ñ"Ó$×0Ñ0Ø×ÑÜ˜Ð!<¸dÓCô
ˆŒ
ð 	�‰Ör:   c               ó<   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRR/	# )rX   rY   rZ   r[   r\   r]   r#   r^   r_   r`   ra   r"   rb   r4   rc   rd   r   r;   )re   s   "r8   rf   Ú*GenericForTokenClassification.__annotate__  sd   € ÷ !
ñ !
à*ð!
ð ,ð!
ð .ð	!
ð
 &ð!
ð 0ð!
ð (ð!
ð ð!
ð -ð!
ð 
ñ!
r:   c                	ó4  € \        W P                  4      ! V3R VRVRVRVRV/VB p	V	P                  p
V P                  V
4      p
V P	                  V
4      pRpVe   V P                  W¶V P                  4      p\        VVV	P                  V	P                  R7      # )r[   r]   r#   r_   r"   N)rn   rl   ro   rp   )
rr   rL   rs   r«   rR   r|   rU   r   ro   rp   )r2   rY   r[   r]   r#   r_   ra   r"   r4   r¡   r¢   rl   rn   s   &&&&&&&&,    r8   r‚   Ú%GenericForTokenClassification.forward  sÅ   € ô ,3°4×9OÑ9OÔ+PØñ,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð "×3Ñ3ˆØŸ,™, Ó7ˆØ—‘˜OÓ,ˆàˆØÒØ×%Ñ% f°d·k±kÓBˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r:   )r«   rJ   rR   r„   r…   rB   s   @r8   r¥   r¥   ø   s*   ø† àÐõð* Øö!
ó ó ö!
r:   r¥   c                  ó>   a € ] tR tRtRR V 3R llltR R ltRtV ;t# )ÚMtpLayeri7  c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
rX   rU   r   Údecoder_layer_clsztype[nn.Module]Únorm_clsÚ	layer_idxÚintÚuse_post_normÚboolr;   )re   s   "r8   rf   ÚMtpLayer.__annotate__8  sC   € ÷ jñ jà ðjð +ðjð "ð	jð
 ðjð ñjr:   c                	ó¼  <€ \         SV `  4        Wn        WPn        V! VP                  VP
                  R 7      V n        V! VP                  VP
                  R 7      V n        \        P                  ! VP                  ^,          VP                  RR7      V n
        V! W4      V n        V'       d&   V! VP                  VP
                  R 7      V n        R# RV n        R# )©ÚepsFrG   N)r0   rI   rU   r¸   rQ   Úrms_norm_epsÚenormÚhnormrN   rO   Úeh_projÚ	mtp_blockÚ	post_norm)r2   rU   r´   rµ   r¶   r¸   r)   s   &&&&&&€r8   rI   ÚMtpLayer.__init__8  s¤   ø€ ô 	‰ÑÔØŒØ*ÔÙ˜f×0Ñ0°f×6IÑ6IÔJˆŒ
Ù˜f×0Ñ0°f×6IÑ6IÔJˆŒ
Ü—y’y ×!3Ñ!3°aÕ!7¸×9KÑ9KÐRWÔXˆŒÙ*¨6Ó=ˆŒßR_™ &×"4Ñ"4¸&×:MÑ:MÔNˆŽÐeiˆŽr:   c               ó4   € V ^8„  d   QhRRRRRRRRRRR	R
RR/# )rX   r_   útorch.TensorÚprevious_hidden_stateÚposition_embeddingsú!tuple[torch.Tensor, torch.Tensor]r[   r\   r]   r#   r^   rd   r;   )re   s   "r8   rf   rº   I  sP   € ÷ ñ à#ðð  ,ðð ?ð	ð
 ,ðð *ðð &ðð 
+ñr:   c                	óÀ  € \        V P                  R R4      '       d:   \        P                  ! V P	                  V4      V P                  V4      .RR7      pM8\        P                  ! V P                  V4      V P	                  V4      .RR7      pV P                  V4      p	V P                  ! V	3RVRVRVRV/VB p	V P                  '       d   V P                  V	4      p	V	# )Úmtp_hidden_states_firstFrš   r[   rÈ   r]   r#   rq   )
rr   rU   rx   ÚcatrÀ   r¿   rÁ   rÂ   r¸   rÃ   )
r2   r_   rÇ   rÈ   r[   r]   r#   r4   Úprojection_inputro   s
   &&&&&&&,  r8   r‚   ÚMtpLayer.forwardI  sß   € ô �4—;‘;Ð 9¸5×AÒAÜ$Ÿyšy¨$¯*©*Ð5JÓ*KÈTÏZÉZÐXeÓMfÐ)gÐmoÔpÑä$Ÿyšy¨$¯*©*°]Ó*CÀTÇZÁZÐPeÓEfÐ)gÐmoÔpÐØŸ™Ð%5Ó6ˆØŸšØñ
à)ð
ð !4ð
ð &ð	
ð
 ,ð
ð ñ
ˆð ××ÐØ ŸN™N¨=Ó9ˆMàÐr:   )rU   rÁ   r¿   rÀ   rÂ   rÃ   r¸   ©T)r*   r<   r=   r>   rI   r‚   r@   rA   rB   s   @r8   r²   r²   7  s   ø† ÷jò j÷"ó r:   r²   c                  óÄ   a € ] tR tRtRtRtRtRR.tRR.tR V 3R llt	R R	 lt
R
 R ltR R ltRR R llt]RR R ll4       t]R R l4       t]R R l4       tRtV ;t# )ÚMtpModelig  Tzshared_head.head.weightzembed_tokens.weightzshared_head.weightc               ó    € V ^8„  d   QhRRRR/# )rX   Ú
main_modelr   Únum_mtp_layersr·   r;   )re   s   "r8   rf   ÚMtpModel.__annotate__r  s   € ÷ ñ  ?ð ÀCñ r:   c                	óF  <€ \         SV `  VP                  P                  4       4       R V n        W n        VP                  4       p\        VP                  R,          4      p\        R VP                  R,          P                  4        4       4      pRV n        RV n        \        V P                  R4      '       d#   RV n        V P                  P                  V n        \        P                   ! \#        V4       Uu. uF$  p\%        V P                  WEW`P                  4      NK&  	  up4      V n        V P                  '       d8   V! V P                  P&                  V P                  P(                  R7      V n        V P-                  V4       V P/                  4        R# u upi )ÚForCausalLMc              3  óN   "  € T F  w  rR V9   g   K  \        V4      x € K  	  R# 5i)ÚnormN)Útype)Ú.0ÚnameÚmodules   &  r8   Ú	<genexpr>Ú$MtpModel.__init__.<locals>.<genexpr>z  s(   é € ð 
á E‘�Ø˜‰~ô ŒD��LˆLÛ Eùs   ‚%’%TFÚchain_hidden_post_normr¼   Nrq   )r0   rI   rU   Úget_mtp_configÚ	loss_typerÔ   Úget_decoderrÚ   ÚlayersÚnextÚnamed_modulesr¸   Úuse_shared_post_normÚhasattrrà   rN   Ú
ModuleListÚranger²   rQ   r¾   Úshared_post_normÚtie_with_main_modelrS   )r2   rÓ   rÔ   Ú
base_modelÚ	layer_clsrµ   Úkr)   s   &&&    €r8   rI   ÚMtpModel.__init__r  sI  ø€ Ü‰Ñ˜×*Ñ*×9Ñ9Ó;Ô<à&ˆŒØ,Ôà×+Ñ+Ó-ˆ
Ü˜×*Ñ*¨2Õ.Ó/ˆ	Üñ 
à *× 1Ñ 1°"Õ 5× CÑ CÔ Eó
ó 
ˆð "ˆÔØ$)ˆÔ!Ü�4—;‘;Ð 8×9Ò9Ø!&ˆDÔØ(,¯©×(JÑ(JˆDÔ%ô —m’mÜX]Ð^lÔXmÓnÑXmÐSTŒX�d—k‘k 9¸×;MÑ;MÖNÑXmÑnó
ˆŒð ×$×$Ð$Ù$,¨T¯[©[×-DÑ-DÈ$Ï+É+×JbÑJbÔ$cˆDÔ!ð 	× Ñ  Ô,à�‰Öùò os   Ã<*Fc               ó   € V ^8„  d   QhRR/# )rX   rÓ   r   r;   )re   s   "r8   rf   rÕ   ’  s   € ÷ Bñ B¨oñ Br:   c                ó–   € VP                  4       V n        VP                  V n        VP	                  4       p\        VRR4      V n        R# )z8Tie the embedding/head/rotary layer with the main model.Ú
rotary_embN)r�   rŽ   Úlm_headÚshared_headrã   rr   ró   )r2   rÓ   rí   s   && r8   rì   ÚMtpModel.tie_with_main_model’  sC   € ð '×;Ñ;Ó=ˆÔØ%×-Ñ-ˆÔð  ×+Ñ+Ó-ˆ
Ü! *¨l¸DÓAˆŽr:   c               ó    € V ^8„  d   QhRRRR/# )rX   ro   rÆ   rd   r;   )re   s   "r8   rf   rÕ   œ  s   € ÷ 	ñ 	°ð 	Àñ 	r:   c                óì   € \        V P                  RR4      pVe	   W,          pV P                  V4      p\        V P                  RR4      pVe%   WCP                  R,          8  d   VRRV13,          pV# )z[Apply the shared head the same way the main model does (muP scaling, unpadded vocab slice).Úlogits_mup_width_multiplierNÚunpadded_vocab_size.rq   )rr   rU   rõ   rt   )r2   ro   Ú
multiplierrl   rú   s   &&   r8   Ú_project_to_logitsÚMtpModel._project_to_logitsœ  sx   € ä˜TŸ[™[Ð*GÈÓNˆ
ØÒ!Ø)Õ6ˆMØ×!Ñ! -Ó0ˆÜ% d§k¡kÐ3HÈ$ÓOÐØÒ*Ð/BÇ\Á\ÐRTÕEUÔ/UØ˜CÐ!5Ð"5Ð!5Ð5Õ6ˆFØˆr:   c               ó(   € V ^8„  d   QhRRRRRRRR/# )rX   r¶   r·   r_   rÆ   Ú	mtp_cacher   r]   r;   )re   s   "r8   rf   rÕ   §  s,   € ÷ -ñ -Øð-Ø-9ð-ØFNð-Ø^jñ-r:   c                ó~  € V P                   P                  '       d   V P                   P                  V,          MV P                   pRVRVRRRVRVRV/p\        V P                   RR4      pVe	   Wq,          MRp/ p	Ve^   V\        9   dS   \        V,          p
\        V
\        4      '       d%   V
P                  4        F  w  r¼V! R/ VB W›&   K  	  MV
! R/ VB W˜&   M\        R/ VB V	R	&   \        V	4      ^8”  d   \        R
4      hR	RRRRR/pV	P                  4        UUu/ uF  w  rïWÞ,          VbK  	  p	ppV	# u uppi )z¬
Create the (potentially several) masks required for layer `layer_idx`. This relies on the `layer_types` from the MTP config
if defined, and otherwise uses full attention.
rU   r_   r[   Nr#   r]   r¶   Úlayer_typesÚfull_attentionzLYou should have at most 2 masks, 1 for attention, and 1 for linear attentionÚsliding_attentionÚlinear_attentionÚ	conv_maskr;   )rU   Úis_heterogeneousÚper_layer_configrr   r   Ú
isinstanceÚdictÚitemsr   Úlenrv   )r2   r¶   r_   rÿ   r]   Úlayer_configÚmask_kwargsÚmtp_layer_typesÚmtp_layer_typeÚmasksÚmask_functionÚactual_patternÚactual_functionÚ%internal_layer_expected_kwarg_mappingrï   Úvs   &&&&&           r8   Úcreate_masks_for_mtp_layerÚ#MtpModel.create_masks_for_mtp_layer§  sW  € ð CGÇ+Á+×B^×B^ÐB^�t—{‘{×3Ñ3°IÖ>Ðdh×doÑdoˆà�lØ˜]Ø˜dØ˜yØ˜Là˜ð
ˆô " $§+¡+¨}¸dÓCˆØ7FÒ7R˜Ö3ÐX\ˆØˆØÒ%¨.Ô<bÔ*bÜBÀ>ÕRˆMä˜-¬×.Ò.Ø7D×7JÑ7JÖ7LÑ3�NÙ,;Ñ,J¸kÑ,J�EÓ)ò 8Mñ )6Ñ(D¸Ñ(D�Ò%ä&8Ñ&G¸;Ñ&GˆEÐ"Ñ#äˆu‹:˜Œ>ÜÐkÓlÐlð Ð.ØÐ!1Ø ð1
Ð-ð JOÏÉÌÔWÉÁÀÐ6Õ9¸1Ò<ÉˆÑWàˆùó Xs   Ä D9c               ó@   € V ^8„  d   QhRRRRRRRRRRR	R
RRRRRRRR/
# )rX   rY   rÆ   Úlast_hidden_statesr[   r\   r]   rÿ   zMtpCache | Nonera   rZ   Ú	do_sampler¹   Úlogits_processorzLogitsProcessorList | NoneÚfull_input_idsrd   rÉ   r;   )re   s   "r8   rf   rÕ   Ö  sz   € ÷ W9ñ W9àðW9ð )ðW9ð ,ð	W9ð
 *ðW9ð #ðW9ð (ðW9ð ðW9ð 5ðW9ð ,ðW9ð 
+ñW9r:   c
           
     ót  € VP                   ^ ,          p. p. pRp\        V P                  4       EFÑ  w  ppV P                  V4      P	                  VP
                  4      pV P                  e   V P                  VVR7      MRpV P                  VVWT4      pV! VV3RVRVRV/VBV
B pV P                  '       d   V P                  V4      pVf   \        RR4      M\        RR4      pV P                  VRVR3,          4      pVep   \        P                  P                  V^ V3RR7      RVR13,          P                  4       pWàP                   ! VV3R	V P"                  P$                  R
V/V
B ,          pVR,          P	                  VP
                  R7      pVe,   V	e(   V! V	VP	                  \&        P(                  R7      4      pVP+                  VR,          4       V'       dJ   \        P                  P-                  VR\&        P(                  R7      p\&        P.                  ! V^R7      pM\&        P0                  ! VRRR7      pVP+                  V4       \&        P2                  ! VR,          V.RR7      p\&        P2                  ! VR,          VP5                  V^4      .RR7      p\&        P2                  ! VR,          VRRR13,          ^,           .RR7      pV	f   EK·  \&        P2                  ! V	V.RR7      p	EKÔ  	  \&        P2                  ! V^R7      p\&        P2                  ! V^R7      pVVV3# )aG  
Sample 1 new token for each mtp layers present in this model. Note that the inputs are assumed to be already sliced and correct
here, i.e. if the main model just processed inputs corresponding to tokens at positions [N-1, N] in the sequence, then from it
you draft a new token for position N+1, and the `input_ids`/`position_ids`/`attention_mask` here are assumed to correspond to
data for tokens at positions [N, N+1], i.e. shifted by 1 from the main model, by the newly drafted token. The `last_hidden_states`
though will correspond to the same as the main model, i.e. positions [N-1, N] in the sequence length dimension.

`full_input_ids` correspond to the full sequence of `input_ids`, which is used in case we have any `logits_processor` as some
processors may require to check the length/value of the full previous sequence of ids.
N)r]   rÈ   r]   r#   ºNNN)r’   .Ú
vocab_sizeÚshift_labelsrk   )rj   )r›   rj   )Únum_samplesT)r›   Úkeepdimrš   rq   iœÿÿÿ)r  rq   r  )r  Nr  )r  :é   NN)rt   Ú	enumeraterä   rŽ   rw   ri   ró   r  rç   rë   Úslicerü   rN   Ú
functionalÚpadr    r|   rU   r  rx   Úfloat32ÚappendÚsoftmaxÚmultinomialr{   rÌ   Únew_ones)r2   rY   r  r[   r]   rÿ   ra   r  r  r  r4   r~   Údrafted_logitsÚdrafted_tokensrn   ÚiÚ	mtp_layerr_   rÈ   r  Úslice_indicesrl   r   Únext_token_logitsÚprobsÚnext_mtp_tokenÚnew_candidate_idsÚcandidate_logitss   &&&&&&&&&&,                 r8   r‚   ÚMtpModel.forwardÖ  s#  € ð0 —_‘_ QÕ'ˆ
àˆØˆØˆÜ% d§k¡k×2‰LˆAˆyà ×-Ñ-¨iÓ8×;Ñ;Ð<N×<UÑ<UÓVˆMàMQÏ_É_ÒMh�—‘ ¸L�ÔIÐnrð  ð
 ×3Ñ3°A°}ÀiÓ^ˆEá!*ØØ"ñ"ð %8ð"ð *ð	"ð
 !*ð"ð ð"ð ñ"Ðð ×(×(Ð(Ø%)×%:Ñ%:Ð;MÓ%NÐ"ð 06ª~œE " dœOÄ5ÈÈtÓCTˆMØ×,Ñ,Ð-?ÀÀ=ÐRSÐ@SÕ-TÓUˆFð Ò!ä!Ÿ}™}×0Ñ0°¸!¸Q¸ÀtÐ0ÓLÈSÐRSÑRTÈWÕU×`Ñ`Ób�Ø×*Ò*Ø˜FñØ/3¯{©{×/EÑ/EðØT`ðØdjñõ �ð
 !' xÕ 0× 3Ñ 3¸9×;KÑ;KÐ 3Ó LÐØÒ+°Ò0JÙ$4°^ÐEV×EYÑEYÔ`e×`mÑ`mÐEYÓEnÓ$oÐ!à×!Ñ!Ð"3°JÕ"?Ô@ßÜŸ™×-Ñ-Ð.?ÀRÌuÏ}É}Ð-Ó]�Ü!&×!2Ò!2°5ÀaÔ!H‘ä!&§¢Ð.?ÀRÐQUÔ!V�Ø×!Ñ! .Ô1ô Ÿ	š	 9¨UÕ#3°^Ð"DÈ"ÔMˆIÜ"ŸYšY¨°uÕ(=¸~×?VÑ?VÐWaÐcdÓ?eÐ'fÐlnÔoˆNÜ Ÿ9š9 l°5Õ&9¸<ÈÈ2É3ÈÕ;OÐRSÕ;SÐ%TÐZ\Ô]ˆLð Õ)Ü!&§¢¨N¸NÐ+KÐQSÔ!T“ñm 3ôp "ŸIšI n¸!Ô<ÐÜ Ÿ9š9 ^¸Ô;ÐØ Ð"2°DÐ8Ð8r:   c               ó    € V ^8„  d   QhRRRR/# )rX   rÓ   r   rd   rÑ   r;   )re   s   "r8   rf   rÕ   0  s   € ÷ iñ i¨ð iÐX`ñ ir:   c           
     	ó	  € VP                   P                  pVP                   P                  4       P                  pVP                  P                  4       p. pV FP  p\        P                  ! R V4      p	V	e"   \        V	P                  ^4      4      V8  d   K?  VP                  V4       KR  	  \        V4      ^ 8X  d#   \        VP                  P                   R24      h\        P                  ! RP!                  R V 4       4      4      p
VP                   P                  4       P"                  pV P%                  VP                   P&                  RRR4      p\)        V4      ;_uu_ 4        V ! W4      pRRR4       \+        VRRRRRR7      w  rïTpRpVe‡   VR,          P-                  4        UUu/ uF  w  ppV
P                  V4      f   K  VVbK   	  pppV Uu. uF9  p\.        P0                  P3                  V4      VP5                  4       9   g   K7  VNK;  	  pp/ p\7        4       pVRJ;'       dQ    \8        ;QJ d*    R	 VP5                  4        4       F  '       g   K   RM	  RM! R	 VP5                  4        4       4      pV'       g   \:        P<                  R
8X  d   RMRpV F‚  p\?        VRRVR7      pVPA                  V4       VPC                  4        FK  pVe   VVPC                  4       9   g   Ve   K"  V
P                  V4      f   K7  VPE                  V4      VV&   KM  	  K„  	  \G        WUV,           4       Uu. uF  p\I        RV R2RVV,
           R2R7      NK!  	  ppVPK                  \M        XRR7      4       VPK                  VPN                  4       \Q        VV\S        VW!P                   P&                  R7      R7      w  ppV F  pVPU                  RRR4       K  	  VPW                  V4       VPX                  '       d2   \[        RV P                   RV R\]        VPX                  4       24      hVP_                  V4       \a        VVRV\b        R7       V#   + '       g   i     ELü; iu uppi u upi u upi )z\.(\d+)Nz7 does not seem to register any known MTP layer patternsÚ|c              3  ó.   "  € T F  pR V R2x € K  	  R# 5i)Ú(Ú)Nr;   )rÛ   Úpatterns   & r8   rÞ   Ú+MtpModel.from_pretrained.<locals>.<genexpr>@  s   é € Ð'ZÑGY¸G¨1¨W¨I°Q­ÓGYùs   ‚FT)Úpretrained_model_name_or_pathÚvariantÚ	gguf_fileÚuse_safetensorsÚ
user_agentÚis_remote_codeÚ
weight_mapc              3  ó„   "  € T F6  p\        V\        P                  4      '       d   VP                  MTR 8H  x € K8  	  R# 5i)ÚmpsN)r  rx   ri   rÚ   )rÛ   Úds   & r8   rÞ   r?  ]  s/   é € ð 0
ÙM`È”z !¤U§\¡\×2Ò2ˆQ�VŠV¸¸eÖCÓM`ùs   ‚>A Úwin32ÚpreadÚmmapÚptÚcpu)Ú	frameworkri   Úbackendzlayers.r&   z.mtp_block.)Úsource_patternsÚtarget_patterns)Ú
add_legacy)Úweight_mappingÚ
device_maprj   )rE   Ú
state_dictÚload_configzThe following z weights are missing from z9 (checkpoint keys not matching the conversion mapping?): )rE   r@  Úignore_mismatched_sizesÚloading_infor-   )2rU   Úname_or_pathrP   Únum_hidden_layersÚ"_keys_to_ignore_on_load_unexpectedÚcopyÚreÚsearchr·   Úgroupr)  r  rv   r)   r*   ÚcompileÚjoinrÔ   Úget_init_contextrj   r   r   r
  ÚosÚpathÚbasenameÚvaluesÚsetÚanyÚsysÚplatformr   ÚaddÚkeysÚ	get_slicerê   r	   Úextendr   Ú_weight_conversionsr
   r   Ú__exit__Ú#_adjust_missing_and_unexpected_keysÚmissing_keysÚRuntimeErrorÚsortedrì   r   r-   )ÚclsrÓ   rU  r4   r@  r[  Úmtp_patternsÚfinal_mtp_patternsr>  Úmatch_objectÚ	mtp_regexrÔ   ÚcontextsÚ	mtp_modelÚcheckpoint_filesÚsharded_metadataÚ	mtp_filesÚmtp_weight_maprï   r  ÚfileÚmtp_state_dictÚall_pointerÚis_mpsrP  Úfile_pointerÚNÚweight_conversionsrY  Ú_s   &&&,                          r8   Úfrom_pretrainedÚMtpModel.from_pretrained/  sS  € à(2×(9Ñ(9×(FÑ(FÐ%Ø&×-Ñ-×=Ñ=Ó?×QÑQÐð "×DÑD×IÑIÓKˆð  ÐÛ#ˆGÜŸ9š9 Z°Ó9ˆLØÒ'¬C°×0BÑ0BÀ1Ó0EÓ,FÐIZÔ,ZÙØ×%Ñ% gÖ.ñ	 $ô
 Ð!Ó" aÔ'Ü 
× 4Ñ 4× =Ñ =Ð>Ð>uÐvÓwÐwÜ—J’J˜sŸx™xÑ'ZÑGYÓ'ZÓZÓ[ˆ	ð $×*Ñ*×:Ñ:Ó<×KÑKˆØ×'Ñ'¨
×(9Ñ(9×(?Ñ(?ÀÈÈtÓTˆÜ˜X×&Õ&Ù˜JÓ7ˆI÷ 'ô .LØ*GØØØ ØØ ô.
Ñ*Ðð %ˆ	ØˆàÒ'à!1°,Õ!?×!EÑ!EÔ!GôÙ!G™˜˜AÈ9×K[ÑK[Ð\]ÓK^”��1’Ñ!Gð ñ ñ +;ÓpÑ*: $¼b¿g¹g×>NÑ>NÈtÓ>TÐXf×XmÑXmÓXoÑ>oŸ˜Ñ*:ˆIÐpð ˆÜ“eˆØ 4Ð'÷ 
ð 
¯C«Cñ 0
ØMW×M^ÑM^ÔM`ó0
¯C¯CªCñ 0
ØMW×M^ÑM^ÔM`ó0
ó -
ˆ÷ $¤s§|¡|°wÔ'>‘'ÀFˆÛˆDÜ$ T°TÀ%ÐQXÔYˆLØ�O‰O˜LÔ)Ø!×&Ñ&Ö(�à"Ò.°1¸×8KÑ8KÓ8MÔ3MØ"Ô*¨y×/?Ñ/?ÀÓ/BÔ/Nà(4×(>Ñ(>¸qÓ(A�N 1Ó%ó )ñ ô$ Ð,À.Õ.PÔQó	
ñ R�ô Ø")¨!¨¨A À'È!ÐN_ÕJ_ÐI`Ð`kÐ@l÷ñ Rð	 	ð 
ð 	×!Ñ!Ô">¸yÐUZÔ"[Ô\Ø×!Ñ! *×"@Ñ"@ÔAô ?ØØ%Ü+Ø1¸j×PaÑPa×PgÑPgôô
‰ˆ�aó ˆAØ�J‰J�t˜T 4Ö(ñ ð 	×5Ñ5°lÔCð ×$×$Ð$ÜØ  §¡ Ð.HÐIfÐHgð hKÜKQÐR^×RkÑRkÓKlÐJmðoóð ð 	×%Ñ% jÔ1äØØ*GØ$)Ø%Üõ	
ð Ð÷i '×&Ð&üó ùò qùò.
s*   Å*	Q&Æ0Q:ÇQ:Ç4R ÈR Í%RÑ&Q7	c               ó   € V ^8„  d   QhRR/# ©rX   rd   r¹   r;   )re   s   "r8   rf   rÕ   œ  s   € ÷ ñ ¨Tñ r:   c                	ó   € R # rÏ   r;   ©rv  s   &r8   Ú_can_set_attn_implementationÚ%MtpModel._can_set_attn_implementation›  ó   € ñ r:   c               ó   € V ^8„  d   QhRR/# rŒ  r;   )re   s   "r8   rf   rÕ   ¡  s   € ÷ ñ °ñ r:   c                	ó   € R # rÏ   r;   rŽ  s   &r8   Ú_can_set_experts_implementationÚ(MtpModel._can_set_experts_implementation   r‘  r:   )	rŽ   rä   râ   rÔ   ró   rõ   rë   r¸   rç   )NFNNrŒ   )r*   r<   r=   r>   Ú_supports_sdpaÚ_supports_flex_attnÚ_supports_flash_attnr\  Ú_keys_to_ignore_on_load_missingrI   rì   rü   r  r‚   Úclassmethodr‰  r�  r”  r@   rA   rB   s   @r8   rÑ   rÑ   g  s�   ø† ð €NØÐØÐà*CÐEZÐ)[Ð&à';Ð=RÐ&SÐ#÷ð õ@Bõ	õ-÷^W9ðr öió ðiðV ôó ðð ôó ör:   rÑ   )<Ú
__future__r   rd  r^  rj  Ú	functoolsr   Útypingr   rx   Útorch.nnrN   Úsafetensorsr   Úcache_utilsr   Úconversion_mappingr   Úcore_model_loadingr	   r
   Úmasking_utilsr   r   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   r   Úmodels.autor   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.loading_reportr   r   Úconfiguration_utilsr   Úgeneration.logits_processr   Ú
get_loggerr*   r-   ÚModuler    rD   r‡   r¥   r²   rÑ   r;   r:   r8   Ú<module>r®     sñ   ðõ #ã 	Û 	Û 
Ý Ý  ã Ý Ý !å Ý <ß Tß U÷ó ÷ aÑ `Ý "Ý $ß aÕ aÝ 7÷ Ý%Ý5Ý>ð 
×	Ò	˜HÓ	%€ô91 §¡ô 91ðx ÷G
ð G
ó ðG
ðT ÷9
ð 9
ó ð9
ðx ÷;
ð ;
ó ð;
ô|-ˆr�y‰yô -ô`|ˆö |r:   