+
    œ°³jÑ.  ã                  ó  € ^ RI Ht ^ RIt^ RIt^ RIt^ RIHt ^RIHtH	t	 ^RI
Ht ^RIHt ^RIHtHt ^RIHt ^R	IHtHtHt ^R
IHtHtHtHtHtHtHtHtHtH t  ]	PB                  ! ]"4      t#]'       d   ^ RI$H%t%  ! R R4      t&R# )é    )ÚannotationsN)ÚTYPE_CHECKING)Úis_torch_greater_or_equalÚlogging)Úcreate_and_tag_model_card)ÚDistributedConfig)Ú$apply_fully_sharded_data_parallelismÚis_fsdp_managed_module)Úapply_pipeline_parallelism)Ú_validate_tp_plan_stylesÚapply_tensor_parallelismÚgather_state_dict_for_save)
Ú_distributed_barrierÚ_ensure_torch_distributedÚ_get_torch_distributed_rankÚ!_get_torch_distributed_world_sizeÚ!_is_torch_distributed_initializedÚgather_full_state_dictÚ)initialize_fully_sharded_data_parallelismÚinitialize_pipeline_parallelismÚinitialize_tensor_parallelismÚ!save_model_checkpoint_distributedc                  óŽ  € ] tR t^4t$ RtRtRtR]R&   RtR]R&   Rt	Rt
R]R&   RtR]R&   R	 R
 lt]R R l4       t]R R l4       t]R R l4       t]P"                  R R l4       t]P"                  R R l4       t]RRRR/R R ll4       t]R R l4       tR R ltRRRR R!RR"RR#RR$RR%R/R& R' lltRR /R( R) lltR* R+ ltR,tR# )-ÚDistributedMixinzFDistributed orchestration and save/load hooks for [`PreTrainedModel`].Núdict[str, str] | NoneÚ_tp_planÚ_ep_planú!dict[str, tuple[str, str]] | NoneÚ_pp_planÚ
_fsdp_planc               ó   € V ^8„  d   QhRR/# )é   ÚreturnÚNone© )Úformats   "Ú_/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/distributed/mixin.pyÚ__annotate__ÚDistributedMixin.__annotate__>   s   € ÷ [ñ [ Tñ [ó    c           
     óÈ  € \        V 4      p\        \        VRR4      ;'       g    / 4      V n        \        \        VRR4      ;'       g    / 4      V n        \        \        VRR4      ;'       g    / 4      V n        \        \        VRR4      ;'       g    / 4      V n        V P                  V J då   V P
                  P                  V P                  P                  ;'       g    / 4       V P                  P                  V P                  P                  ;'       g    / 4       V P                  P                  V P                  P                  ;'       g    / 4       V P                  P                  V P                  P                  ;'       g    / 4       V P                  4        EF¡  w  r#\        VRR4      ;p'       dR   V P                  P                  VP                  4       P!                  4        UUu/ uF  w  rVV RV 2VbK  	  upp4       \        VRR4      ;p'       dR   V P                  P                  VP                  4       P!                  4        UUu/ uF  w  rVV RV 2VbK  	  upp4       \        VRR4      ;p'       dR   V P
                  P                  VP                  4       P!                  4        UUu/ uF  w  rVV RV 2VbK  	  upp4       \        VRR4      ;p'       g   EKP  V P                  P                  VP                  4       P!                  4        UUu/ uF  w  rVV RV 2VbK  	  upp4       EK¤  	  R# u uppi u uppi u uppi u uppi )zQCopy class-level plans onto the instance and merge config/children contributions.r   Nr   r   r    Ú.)ÚtypeÚdictÚgetattrr   r   r   r    Ú
base_modelÚupdateÚconfigÚbase_model_pp_planÚbase_model_tp_planÚbase_model_ep_planÚbase_model_fsdp_planÚnamed_childrenÚcopyÚitems)ÚselfÚ	model_clsÚnameÚmoduleÚplanÚkÚvs   &      r'   Úinit_parallel_plansÚ$DistributedMixin.init_parallel_plans>   s›  € ä˜“Jˆ	ÜœW Y°
¸DÓA×GÐGÀRÓHˆŒÜœW Y°
¸DÓA×GÐGÀRÓHˆŒÜœW Y°
¸DÓA×GÐGÀRÓHˆŒÜœw y°,ÀÓE×KÐKÈÓLˆŒà�?‰?˜dÓ"Ø�M‰M× Ñ  §¡×!?Ñ!?×!EÐ!EÀ2ÔFØ�M‰M× Ñ  §¡×!?Ñ!?×!EÐ!EÀ2ÔFØ�M‰M× Ñ  §¡×!?Ñ!?×!EÐ!EÀ2ÔFØ�O‰O×"Ñ" 4§;¡;×#CÑ#C×#IÐ#IÀrÔJà ×/Ñ/×1‰LˆDÜ˜v z°4Ó8Ð8ˆtÖ8Ø—‘×$Ñ$À4Ç9Á9Ã;×CTÑCTÔCVÔ%WÑCV¹4¸1¨¨¨a°¨s m°QÒ&6ÑCVÒ%WÔXÜ˜v z°4Ó8Ð8ˆtÖ8Ø—‘×$Ñ$À4Ç9Á9Ã;×CTÑCTÔCVÔ%WÑCV¹4¸1¨¨¨a°¨s m°QÒ&6ÑCVÒ%WÔXÜ˜v z°4Ó8Ð8ˆtÖ8Ø—‘×$Ñ$À4Ç9Á9Ã;×CTÑCTÔCVÔ%WÑCV¹4¸1¨¨¨a°¨s m°QÒ&6ÑCVÒ%WÔXÜ˜v |°TÓ:Ð:ˆt×:Ñ:Ø—‘×&Ñ&ÀTÇYÁYÃ[×EVÑEVÔEXÔ'YÑEX¹T¸Q¨4¨&°°!°¨°qÒ(8ÑEXÒ'Y×Zó 2ùã%Wùã%Wùã%Wùã'Ys   Ç5MÉMËMÌ*Mc               ó   € V ^8„  d   QhRR/# ©r"   r#   zdict[str, str]r%   )r&   s   "r'   r(   r)   W   s   € ÷ 
ñ 
˜ñ 
r*   c                ó   € \        V P                  R4      '       dh   V P                  P                  P                  '       dB   V P                  '       g$   \        RV P                  P                   R24      hV P                  # V P                  # )z)The full tp plan for the model's modules.Údistributed_configzGExpert parallelism was requested (`enable_expert_parallel=True`), but `zs` does not define an expert-parallel plan. Add a `base_model_ep_plan` to its config, or disable expert parallelism.)	Úhasattrr2   rF   Úenable_expert_parallelr   Ú
ValueErrorÚ	__class__Ú__name__r   ©r:   s   &r'   Útp_planÚDistributedMixin.tp_planV   sx   € ô �4—;‘;Ð 4×5Ò5¸$¿+¹+×:XÑ:X×:o×:oÐ:oØ—=—=�=Ü ðØŸ™×/Ñ/Ð0ð 1YðZóð ð
 —=‘=Ð Ø�}‰}Ðr*   c               ó   € V ^8„  d   QhRR/# rD   r%   )r&   s   "r'   r(   r)   d   s   € ÷ ñ ˜>ñ r*   c                	ó   € V P                   # ©N)r    rL   s   &r'   Ú	fsdp_planÚDistributedMixin.fsdp_planc   s   € à�‰Ðr*   c               ó   € V ^8„  d   QhRR/# )r"   r#   zdict[str, tuple[str, str]]r%   )r&   s   "r'   r(   r)   h   s   € ÷ ñ Ð3ñ r*   c                	ó   € V P                   # rQ   )r   rL   s   &r'   Úpp_planÚDistributedMixin.pp_plang   s   € à�}‰}Ðr*   c               ó   € V ^8„  d   QhRR/# )r"   r>   r   r%   )r&   s   "r'   r(   r)   l   s   € ÷ ñ Ð1ñ r*   c                	óÆ  € Vf
   / V n         R # \        V\        4      '       g   \        R4      h\	        V4       V P                  4        UUu. uF  w  r#VNK	  	  pppVP                  4        Fd  pVP                  RR4      pRpV F#  p\        P                  ! Wh4      '       g   K!  Rp M	  V'       d   KJ  \        P                  ! RV R24       Kf  	  Wn         R # u uppi )Nz&Can only set a dictionary as `tp_plan`Ú*z\d+FTzLayer pattern 'z�' does not match any parameters in the model. This rule may not be applied during tensor parallelization, or may lead to dimension mismatches)r   Ú
isinstancer.   rI   r   Únamed_parametersÚkeysÚreplaceÚreÚmatchÚwarningsÚwarn)	r:   r>   r<   Ú_Úmodel_param_namesÚlayer_patternÚregex_patternÚpattern_matchedÚ
param_names	   &&       r'   rM   rN   k   sÏ   € àŠ<ØˆDŒMÙÜ˜$¤×%Ò%ÜÐEÓFÐFä  Ô&à15×1FÑ1FÔ1HÔIÑ1H¡g d›TÑ1HÐÑIØ!ŸY™Yž[ˆMØ)×1Ñ1°#°vÓ>ˆMØ#ˆOÛ/�
Ü—8’8˜M×6Ô6Ø&*�OÙñ 0÷ #‘?Ü—’Ø% m _ð 5dð döñ )ð Žùó Js   ÁCc               ó   € V ^8„  d   QhRR/# )r"   r>   r   r%   )r&   s   "r'   r(   r)   †   s   € ÷ ñ Ð=ñ r*   c                	ón   € Vf
   / V n         R # \        V\        4      '       g   \        R4      hWn         R # )Nz&Can only set a dictionary as `pp_plan`)r   r[   r.   rI   )r:   r>   s   &&r'   rV   rW   …   s/   € àŠ<ØˆDŒMÙÜ˜$¤×%Ò%ÜÐEÓFÐFàŽr*   Údevice_meshÚ
device_mapc               ó    € V ^8„  d   QhRRRR/# )r"   rF   zDistributedConfig | dict | Noner#   z/tuple[DistributedConfig | None, object, object]r%   )r&   s   "r'   r(   r)   �   s   € ÷ $;ñ $;à;ð$;ð 
9ñ$;r*   c          	     	ó†  € Vf   R W23# \        V\        4      '       d   \        P                  ! V4      pVP                  ^8”  g   VP
                  ^8”  d`   \        4        \        4       pVP                  VP
                  ,          V8w  d*   \        RVP                   RVP
                   RV R24      hVP                  ^8”  d<   VP                  f   RVn	        \        VP                  VP                  VVR7      w  r2M=VP
                  ^8”  d   \        V4      w  r2MVP                  ^8”  d   \        V4      w  r2WV3# )Nz	tp_size (z) * fsdp_size (z) is not equal to world_size (Ú)Úauto)Útp_sizerk   rl   )r[   r.   r   Ú	from_dictrq   Ú	fsdp_sizer   r   ÚRuntimeErrorrM   r   r   Úpp_sizer   )ÚclsrF   rk   rl   Ú
world_sizes   &&$$ r'   Úprepare_distribute_modelÚ)DistributedMixin.prepare_distribute_model�   sI  € ð Ò%Ø˜Ð0Ð0äÐ(¬$×/Ò/Ü!2×!<Ò!<Ð=OÓ!PÐà×%Ñ%¨Ô)Ð-?×-IÑ-IÈAÔ-MÜ%Ô'Ü:Ó<ˆJØ!×)Ñ)Ð,>×,HÑ,HÕHÈJÔVÜ"ØÐ 2× :Ñ :Ð;¸?ÐK]×KgÑKgÐJhð i3Ø3=°,¸aðAóð ð
 ×%Ñ%¨Ô)Ø!×)Ñ)Ò1Ø-3Ð"Ô*Ü&CØ"×*Ñ*Ø*×2Ñ2Ø'Ø%ô	'Ñ#ˆJ˜ð  ×)Ñ)¨AÔ-Ü&OÐPbÓ&cÑ#ˆJ˜Ø×'Ñ'¨!Ô+Ü&EÐFXÓ&YÑ#ˆJà!¨{Ð:Ð:r*   c               ó    € V ^8„  d   QhRRRR/# )r"   Úmodelz	nn.ModulerF   úDistributedConfig | Noner%   )r&   s   "r'   r(   r)   ·   s   € ÷ ñ àðð 5ñr*   c                ó  € VEe   W!P                   n        W1n        VP                  Vn        VP                  ^8”  d[   VP
                  ^8”  d
   VR,          MTp\        VP                  \        4      '       d   VP                  Vn        \        W4      pV# VP                  ^8”  d*   VP
                  ^8”  d
   VR,          MTp\        W4      pV# VP                  ^8”  d(   VP
                  ^8”  d
   VR,          MTp\        W4      pV# )z:Apply TP or FSDP2 after model init, before weight loading.ÚtpÚfsdpÚpp)r2   rF   Ú_device_meshrq   Ú_tp_sizeÚndimr[   rM   r.   r   rs   r	   ru   r   )rv   r{   rF   rk   Útp_meshÚ	fsdp_meshÚpp_meshs   &&&&   r'   Úmaybe_distribute_modelÚ'DistributedMixin.maybe_distribute_model¶   sò   € ð Ó"Ø.@�L‰LÔ+Ø!,ÔØ/×7Ñ7ˆEŒNà!×)Ñ)¨AÔ-Ø/:×/?Ñ/?À!Ô/C˜+ dÖ+È�ÜÐ0×8Ñ8¼$×?Ò?Ø$6×$>Ñ$>�E”MÜ0°Ó@�ð ˆð $×-Ñ-°Ô1Ø3>×3CÑ3CÀaÔ3G˜K¨Ö/È[�	Ü<¸UÓN�ð
 ˆð $×+Ñ+¨aÔ/Ø/:×/?Ñ/?À!Ô/C˜+ dÖ+È�Ü2°5ÓB�Øˆr*   c               ó    € V ^8„  d   QhRRRR/# )r"   Úis_main_processÚboolr#   r%   )r&   s   "r'   r(   r)   Ò   s   € ÷ !ñ !¸ð !Àñ !r*   c                óX   € Tp\        4       '       d   T;'       d    \        4       ^ 8H  pV# )z7Return whether this rank should write checkpoint files.)r   r   )r:   rŠ   Úsave_on_this_ranks   && r'   Úshould_save_on_this_rankÚ)DistributedMixin.should_save_on_this_rankÒ   s/   € à+ÐÜ,×.Ò.Ø 1× XÐ XÔ6QÓ6SÐWXÑ6XÐØ Ð r*   Úpush_to_hubFr�   TÚrepo_idÚfiles_timestampsÚcommit_messageÚtokenÚ	create_prc               ó<   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRR/	# )r"   Úsave_directoryzstr | os.PathLiker�   r‹   r�   r‘   z
str | Noner’   zdict | Noner“   r”   zstr | bool | Noner•   r#   r$   r%   )r&   s   "r'   r(   r)   Ù   sf   € ÷ %ñ %ð *ð%ð
 ð%ð  ð%ð ð%ð &ð%ð #ð%ð !ð%ð ð%ð 
ñ%r*   c          	     ó¢  € \        R4      '       g   \        R4      h\        V4      '       g   \        R4      h\	        VRR4      f   \        R4      h\        W4       V'       dj   V'       d`   \        WPP                  VR7      p
V
P                  \        P                  P                  VR4      4       V P                  VVVVVV	R	7       R# R# R# )
zBSave an FSDP-wrapped model via DCP and optionally push to the Hub.z2.7zFsave_pretrained(..., distributed_checkpoint=True) requires torch>=2.7.z\save_pretrained(..., distributed_checkpoint=True) is only supported for FSDP-wrapped models.r�   Nz�save_pretrained(..., distributed_checkpoint=True) requires the model to have been initialized with a distributed_config (_device_mesh is None).)r”   z	README.md)r“   r”   r•   )r   ÚOSErrorr
   rI   r/   r   r   Ú
model_tagsÚsaveÚosÚpathÚjoinÚ_upload_modified_files)r:   Úmodel_to_saver—   r�   r�   r‘   r’   r“   r”   r•   Ú
model_cards   &&&$$$$$$$ r'   Úsave_distributed_checkpointÚ,DistributedMixin.save_distributed_checkpointÙ   sÂ   € ô )¨×/Ò/ÜÐbÓcÐcÜ% m×4Ò4ÜØnóð ô �= .°$Ó7Ò?ÜðPóð ô 	*¨-ÔHç×,Ü2°7¿O¹OÐSXÔYˆJØ�O‰OœBŸG™GŸL™L¨¸ÓEÔFØ×'Ñ'ØØØ Ø-ØØ#ð (ö ñ -‰;r*   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r"   Ú
state_dictr.   rF   r|   r�   r‹   r#   r%   )r&   s   "r'   r(   r)      s4   € ÷ ñ ð ðð 5ð	ð  ðð 
ñr*   c               ó  € Vf   V# VP                   ^8”  d8   \        W P                  V P                  VP                   4      pV'       g   / pV# VP                  ^8”  d'   \        4       '       g   \        R4      h\        V4      # V# )zNGather TP- or FSDP-sharded weights to full CPU tensors for checkpoint writing.z‰Saving an FSDP-wrapped model requires torch.distributed to be initialized. Call save_pretrained from every rank after init_process_group.)rq   r   r   r�   rs   r   rI   r   )r:   r    r¥   rF   r�   s   &&&&$r'   Ú"gather_sharded_state_dict_for_saveÚ3DistributedMixin.gather_sharded_state_dict_for_save   s�   € ð Ò%ØÐà×%Ñ%¨Ô)Ü3ØŸM™M¨4×+<Ñ+<Ð>P×>XÑ>XóˆJ÷ %Ø�
ØÐà×'Ñ'¨!Ô+Ü4×6Ò6Ü ðUóð ô *¨-Ó8Ð8àÐr*   c               ó    € V ^8„  d   QhRRRR/# )r"   rF   r|   r#   r$   r%   )r&   s   "r'   r(   r)     s   € ÷ #ñ #ÐIað #Ðfjñ #r*   c                ón   € Vf   R# VP                   ^8”  g   VP                  ^8”  d   \        4        R# R# )zQBarrier so non-writer ranks wait for rank 0 to finish gathered checkpoint writes.N)rq   rs   r   )r:   rF   s   &&r'   Ú&barrier_after_gathered_checkpoint_saveÚ7DistributedMixin.barrier_after_gathered_checkpoint_save  s3   € àÒ%ÙØ×%Ñ%¨Ô)Ð-?×-IÑ-IÈAÔ-MÜ Ö"ñ .Nr*   )r   r    r   r   )rK   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r�   r   Ú__annotations__r   r‚   r   r    rA   ÚpropertyrM   rR   rV   ÚsetterÚclassmethodrx   r‡   rŽ   r¢   r§   r«   Ú__static_attributes__r%   r*   r'   r   r   4   sT  ‡ ÙPà€LØ&*€HÐ#Ó*Ø&*€HÐ#Ó*Ø€HØ26€HÐ/Ó6Ø(,€JÐ%Ó,õ[ð0 ô
ó ð
ð ôó ðð ôó ðð ‡^�^ôó ðð2 ‡^�^ôó ðð ð$;ð ð	$;ð
 ö$;ó ð$;ðL ôó ðõ6!ð%ð
 "ð%ð #'ð%ð #ð%ð )-ð%ð &*ð%ð $(ð%ð  ÷%ðNð #'÷÷<#ñ #r*   r   )'Ú
__future__r   rœ   r_   ra   Útypingr   Úutilsr   r   Ú	utils.hubr   Úconfiguration_utilsr   r   r	   r
   Úpipeline_parallelr   Útensor_parallelr   r   r   r   r   r   r   r   r   r   r   r   r   Ú
get_loggerrK   ÚloggerÚtorch.nnÚnnr   r%   r*   r'   Ú<module>rÁ      sd   ðõ #ã 	Û 	Û Ý  ç 6Ý 1Ý 2ß NÝ 9÷ñ ÷
÷ ÷ ð 
×	Ò	˜HÓ	%€÷ Ý÷o#ó o#r*   