+
    œ°³j92  ã                  óf  € ^ RI Ht ^ RIt^ RIHt ^ RIHtHt ^RIH	t	H
t
HtHt ]P                  ! ]4      t]'       d   ^ RIHt ^RIHt ]	! 4       '       d   ^ RItR R	 ltR
 R ltR R ltR R ltR R ltR#R R lltR tR$R R lltR R ltR R ltR R ltR R lt R R  lt!R! R" lt"R# )%é    )ÚannotationsN)Ú	timedelta)ÚTYPE_CHECKINGÚ	TypeGuard)Úis_torch_availableÚis_torch_distributed_availableÚis_torch_greater_or_equalÚlogging©ÚDTensor)ÚDistributedConfigc               ó   € V ^8„  d   QhRR/# ©é   ÚreturnÚbool© )Úformats   "Ú_/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/distributed/utils.pyÚ__annotate__r   $   s   € ÷ .ñ .¨4ñ .ó    c                 ób   € \        4       '       g   R # \        P                  P                  4       # )F)r   ÚtorchÚdistributedÚis_initializedr   r   r   Ú!_is_torch_distributed_initializedr   $   s#   € Ü)×+Ò+ÙÜ×Ñ×+Ñ+Ó-Ð-r   c               ó    € V ^8„  d   QhRRRR/# )r   ÚobjÚobjectr   zTypeGuard[DTensor]r   )r   s   "r   r   r   *   s   € ÷ $ñ $�Fð $Ð1ñ $r   c                óH   € \        4       '       g   R # ^ RIHp \        W4      # )Fr   )r   Útorch.distributed.tensorr   Ú
isinstance)r   r   s   & r   Ú
is_dtensorr#   *   s   € Ü)×+Ò+ÙÝ0ä�cÓ#Ð#r   c               ó   € V ^8„  d   QhRR/# ©r   r   Úintr   )r   s   "r   r   r   2   s   € ÷ (ñ ( Sñ (r   c                 ób   € \        4       '       g   ^ # \        P                  P                  4       # )r   )r   r   r   Úget_rankr   r   r   Ú_get_torch_distributed_rankr)   2   s#   € Ü,×.Ò.ÙÜ×Ñ×%Ñ%Ó'Ð'r   c               ó   € V ^8„  d   QhRR/# r%   r   )r   s   "r   r   r   8   s   € ÷ .ñ .¨3ñ .r   c                 ób   € \        4       '       g   ^# \        P                  P                  4       # )é   )r   r   r   Úget_world_sizer   r   r   Ú!_get_torch_distributed_world_sizer.   8   s#   € Ü,×.Ò.ÙÜ×Ñ×+Ñ+Ó-Ð-r   c               ó   € V ^8„  d   QhRR/# r   r   )r   s   "r   r   r   >   s   € ÷ `ñ `˜dñ `r   c                 ó~   € \        4       ;'       d-    \        \        P                  P	                  R R4      4      ^ 8H  # )Ú
LOCAL_RANKz-1)r   r&   ÚosÚenvironÚgetr   r   r   Úis_local_dist_rank_0r5   >   s.   € Ü,Ó.×_Ð_´3´r·z±z·~±~ÀlÐTXÓ7YÓ3ZÐ^_Ñ3_Ð_r   c               ó   € V ^8„  d   QhRR/# )r   Údevice_typez
str | Noner   )r   s   "r   r   r   B   s   € ÷ /ñ /¨:ñ /r   c                óþ  € \         P                  P                  4       '       Eg9   V f)   \         P                  P	                  4       P
                  p V R8X  d   \        P                  R4       Rp  \        \        P                  R,          4      p\        \        P                  R,          4      p\        \        P                  R,          4      pRR	RR
RRRRRRRR/pVP                  V 4      pRpV R8w  d6   \        \         V 4      P                  V4       \         P                  ! W4      p\         P                  P                  VVVV\!        ^R7      R7       R# R#   \"         d   p\%        R4      ThRp?ii ; i)z†Initialize torch.distributed if not already initialized.

If `device_type` is not given, it is detected from the current accelerator.
NÚmpsú^PyTorch's built-in DeviceMesh/DTensor stack does not support an MPS mesh. Falling back to CPU.ÚcpuÚRANKr1   Ú
WORLD_SIZEÚcudaÚncclÚglooÚxpuÚxcclÚhpuÚhcclÚneuronÚtpuÚtpu_dist)Úhours)ÚbackendÚrankÚ
world_sizeÚ	device_idÚtimeoutz‘We tried to initialize torch.distributed for you, but it failed. Make sure you init torch distributed in your script to use distributed training.)r   r   r   Ú_CÚ_get_acceleratorÚtypeÚloggerÚwarning_oncer&   r2   r3   r4   ÚgetattrÚ
set_deviceÚdeviceÚinit_process_groupr   Ú	ExceptionÚOSError)r7   rJ   Ú
local_rankrK   Úbackend_maprI   rL   Úes   &       r   Ú_ensure_torch_distributedr\   B   s[  € ô
 ×Ñ×+Ñ+×-Ó-ØÒÜŸ(™(×3Ñ3Ó5×:Ñ:ˆKØ˜%ÔÜ×ÑØpôð  ˆKð"	Ü”r—z‘z &Õ)Ó*ˆDÜœRŸZ™Z¨Õ5Ó6ˆJÜœRŸZ™Z¨Õ5Ó6ˆJð ˜Ø�vØ�vØ�vØ˜(Ø�zðˆKð "—o‘o kÓ2ˆGð
 ˆIØ˜eÔ#Üœ˜{Ó+×6Ñ6°zÔBÜ!ŸLšL¨ÓA�	Ü×Ñ×0Ñ0ØØØ%Ø#ä!¨Ô*ð 1ö ñ= .øôL ô 	Üð^óð ðûð	ús   Á1C+E  Å E<Å+E7Å7E<c                 óB  € \        4       '       g   R# \        P                  P                  4       P                  p V R8w  d@   \        P
                  P                  \        \        V 4      P                  4       .R7       R# \        P
                  P                  4        R# )u#  Barrier bound to the current accelerator device.

Passing `device_ids` is required when the process group was initialized without a
`device_id`; with it, the call is a no-op compared to plain `barrier()`. Safe to call
when torch.distributed has not been initialized â€” returns immediately.
Nr;   )Ú
device_ids)	r   r   rN   rO   rP   r   ÚbarrierrS   Úcurrent_device)r7   s    r   Ú_distributed_barrierra   t   sl   € ô -×.Ò.ÙÜ—(‘(×+Ñ+Ó-×2Ñ2€KØ�eÔÜ×Ñ×!Ñ!¬g´e¸[Ó.I×.XÑ.XÓ.ZÐ-[Ð!Ö\ä×Ñ×!Ñ!Ö#r   c               ó    € V ^8„  d   QhRRRR/# )r   Útp_planzstr | dict[str, str] | NoneÚtp_sizez
int | Noner   )r   s   "r   r   r   …   s   € ÷ ,#ñ ,#Ø(ð,#Ø3=ñ,#r   c                ó’  € Ve   V f   \        R4      hV e   Ve   \        R4      hVEf"   \        R4      '       g   \        R4      h\        P                  P                  4       P                  pVR8X  d   \        P                  R4       Rp\        \        V4      pVR8w  dY   VP                  \        \        P                  R,          4      4       VP                  4       p\        P                  ! WF4      pTpM"\        P                  ! V4      pT;'       g    / p\        P                   P#                  VP                  V34      pW23# VP$                  ^8”  d&   R	VP&                  9  d   \        R
4      hVR	,          p\        P                  ! VP(                   R\        \        P                  R,          4       24      pW23# )z 
Sets up the device mesh and initialized the backend for tensor parallelism.
This function is called when the model is loaded and the TP plan is set to 'auto'.
z-tp_plan has to be set when tp_size is passed.zY`tp_plan` and `device_map` are mutually exclusive. Choose either one for parallelization.ú2.5z3Tensor parallel is only supported for `torch>=2.5`.r9   r:   r;   r1   ÚtpzsWhen using `tp_plan` and n-d `device_mesh`, it must contain a 'tp' dimension. Please provide a valid `device_mesh`.Ú:)Ú
ValueErrorr	   rX   r   rN   rO   rP   rQ   rR   rS   rT   r&   r2   r3   r`   rU   r   Úinit_device_meshÚndimÚmesh_dim_namesr7   )rc   rd   Údevice_meshÚ
device_mapr7   r`   ÚindexÚ	tp_devices   &&&&    r   Úinitialize_tensor_parallelismrq   …   s‘  € ð Ò˜wšÜÐHÓIÐIØÒ˜zÒ5ÜÐtÓuÐuØÓÜ(¨×/Ò/ÜÐOÓPÐPô —h‘h×/Ñ/Ó1×6Ñ6ˆØ˜%ÔÜ×ÑØpôð  ˆKÜ ¤¨Ó4ˆà˜%ÔØ×%Ñ%¤c¬"¯*©*°\Õ*BÓ&CÔDØ"×1Ñ1Ó3ˆEÜŸš [Ó8ˆIØ"‰JäŸš [Ó1ˆIØ$×*Ð*¨ˆJä×'Ñ'×8Ñ8¸¿¹È'ÈÓTˆð Ð"Ð"ð ×Ñ˜aÔØ˜;×5Ñ5Ô5Ü ð<óð ð & dÕ+ˆKÜ—\’\ [×%<Ñ%<Ð$=¸Q¼sÄ2Ç:Á:ÈlÕC[Ó?\Ð>]Ð"^Ó_ˆ
àÐ"Ð"r   c               ó   € V ^8„  d   QhRR/# ©r   Údistributed_configr   r   )r   s   "r   r   r   ´   s   € ÷ ñ ÐBSñ r   c                óâ  € V P                   ^8”  d   \        R4      '       g   \        R4      h\        P                  P                  4       P                  pVR8w  d`   \        \        P                  P                  R^ 4      4      p\        \        V4      P                  V4       \        P                  ! W4      pM\        P                  ! V4      pV P                   p. . reV^8”  d#   VP                  V4       VP                  R4       \        P                  P!                  V\#        V4      \#        V4      R7      p\%        V4      ^8”  d!   VP'                  RP)                  V4      4       W73# )r,   ú2.7z?FSDP2 requires `torch>=2.7` (distributed checkpoint save/load).r;   r1   Úfsdp©rl   Ú_)Ú	fsdp_sizer	   rX   r   rN   rO   rP   r&   r2   r3   r4   rS   rT   rU   Úappendr   rj   ÚtupleÚlenÚ_flattenÚjoin)rt   r7   rY   rn   rz   ÚdimsÚnamesÚmeshs   &       r   Ú)initialize_fully_sharded_data_parallelismrƒ   ´   s  € ð ×#Ñ# aÔ'Ô0IÈ%×0PÒ0PÜÐWÓXÐXä—(‘(×+Ñ+Ó-×2Ñ2€Kà�eÔÜœŸ™Ÿ™¨°aÓ8Ó9ˆ
Ü”�{Ó#×.Ñ.¨zÔ:Ü—\’\ +Ó:‰
ä—\’\ +Ó.ˆ
à"×,Ñ,€Ià�bˆ%Ø�1„}Ø�‰�IÔØ�‰�VÔô ×Ñ×-Ñ-¨k¼5À»;ÔW\Ð]bÓWcÐ-Ód€Dä
ˆ4ƒy�1„}Ø�‰�c—h‘h˜u“oÔ&àÐÐr   c               ó   € V ^8„  d   QhRR/# rs   r   )r   s   "r   r   r   Ó   s   € ÷ ñ Ø)ñr   c                óŽ  € \        R 4      '       g   \        R4      h\        P                  P	                  4       P
                  p\        V4       \        P                  P                  4       pV P                  pW#8w  d   \        RV RV R24      hVR8w  d`   \        \        P                  P                  R^ 4      4      p\        \        V4      P!                  V4       \        P"                  ! W4      pM\        P"                  ! V4      pW#8X  g   Q RV RV R24       h\        P                  P%                  W3RR7      pWV3# )	rf   zBPipeline parallelism with DistributedConfig requires `torch>=2.5`.zworld_size (z) must be equal to pp_size (Ú)r;   r1   rx   )Úpp)r	   rX   r   rN   rO   rP   r\   r   r-   Úpp_sizeÚRuntimeErrorr&   r2   r3   r4   rS   rT   rU   rj   )rt   r7   rK   rˆ   rY   rn   r‚   s   &      r   Úinitialize_pipeline_parallelismrŠ   Ó   s  € ô % U×+Ò+ÜÐZÓ[Ð[ä—(‘(×+Ñ+Ó-×2Ñ2€KÜ˜kÔ*ä×"Ñ"×1Ñ1Ó3€JØ ×(Ñ(€GØÔÜ˜\¨*¨Ð5QÐRYÐQZÐZ[Ð\Ó]Ð]à�eÔÜœŸ™Ÿ™¨°aÓ8Ó9ˆ
Ü”�{Ó#×.Ñ.¨zÔ:Ü—\’\ +Ó:‰
ä—\’\ +Ó.ˆ
àÔ Ðc L°°Ð<XÐY`ÐXaÐabÐ"cÓcÐ Ü×Ñ×-Ñ-¨k¸:ÐV]Ð-Ó^€DàÐÐr   c               ó   € V ^8„  d   QhRR/# )r   r   zdict[str, torch.Tensor]r   )r   s   "r   r   r   î   s   € ÷ ñ Ð%<ñ r   c                ó–   € \        R4      '       g   \        R4      h^ RIHpHp V! RRR7      pV! WR7      p\        4       ^ 8X  d   V# / # )zvGather FSDP-sharded params to full plain CPU tensors.

Only rank 0 accumulates the result; other ranks return ``{}``.
rv   ú0Distributed checkpointing requires `torch>=2.7`.)ÚStateDictOptionsÚget_model_state_dictT)Úfull_state_dictÚcpu_offload)Úoptions)r	   rX   Ú'torch.distributed.checkpoint.state_dictrŽ   r�   r)   )ÚmodelrŽ   r�   r’   r�   s   &    r   Úgather_full_state_dictr•   î   sM   € ô
 % U×+Ò+ÜÐHÓIÐI÷ _á¨tÀÔF€GÙ*¨5ÔB€OÜ"Ó$¨Ô)ØÐØ€Ir   c               ó    € V ^8„  d   QhRRRR/# ©r   Úcheckpoint_dirÚstrr   ÚNoner   )r   s   "r   r   r     s   € ÷ ñ ¸Sð ÀTñ r   c           	     óÄ   € \        R4      '       g   \        R4      h^ RIHu Hp ^ RIHp ^ RIHp V! V 4      pVP                  VV! VRRR7      R7       \        4        R# )	uÌ  Save model parameters as standard HF-format sharded safetensors using
DCP + HuggingFaceStorageWriter with consolidation enabled.

Every rank first writes its own shard in parallel under
`<checkpoint_dir>/sharded/`, then a consolidation pass reads those shards
and emits HF-compatible `model-*-of-N.safetensors` (+ index) at
`<checkpoint_dir>/`. The result is a directory `from_pretrained` reads
through its normal path â€” no special flag needed at load time.
rv   r�   N)ÚHuggingFaceStorageWriter)r�   T)ÚpathÚsave_distributedÚenable_consolidation)Ústorage_writer)r	   rX   Útorch.distributed.checkpointr   Ú
checkpointÚ'torch.distributed.checkpoint.hf_storagerœ   r“   r�   Úsavera   )r”   r˜   Údcprœ   r�   Ú
state_dicts   &&    r   Ú!save_model_checkpoint_distributedr§     s`   € ô % U×+Ò+ÜÐHÓIÐI÷ /Ð.ÝPÝLá% eÓ,€JØ‡H�HØÙ/ØØ!Ø!%ô
ð ô ô Ör   c               ó    € V ^8„  d   QhRRRR/# r—   r   )r   s   "r   r   r   "  s   € ÷ Pñ PÀð PÈñ Pr   c                ó–   € \        R4      '       g   \        R4      h^ RIHu Hp ^ RIHp V! W4      pVP                  RV/VR7       R# )zSave optimizer state via DCP.rv   r�   N)Úget_optimizer_state_dictÚ	optimizer©Úcheckpoint_id)r	   rX   r¡   r   r¢   r“   rª   r¤   )r”   r«   r˜   r¥   rª   Úoptimizer_state_dicts   &&&   r   Úsave_optimizer_distributedr¯   "  sE   € ä$ U×+Ò+ÜÐHÓIÐI÷ /Ð.ÝPá3°EÓEÐØ‡H�HˆkÐ/Ð0À€HÖOr   c               ó    € V ^8„  d   QhRRRR/# r—   r   )r   s   "r   r   r   0  s   € ÷ Eñ EÀð EÈñ Er   c                ó¬   € \        R4      '       g   \        R4      h^ RIHu Hp ^ RIHpHp V! W4      pVP                  RV/VR7       V! WV4       R# )zLoad optimizer state via DCP.rv   r�   N)rª   Úset_optimizer_state_dictr«   r¬   )	r	   rX   r¡   r   r¢   r“   rª   r²   Úload)r”   r«   r˜   r¥   rª   r²   r®   s   &&&    r   Úload_optimizer_distributedr´   0  sP   € ä$ U×+Ò+ÜÐHÓIÐI÷ /Ð.ßjá3°EÓEÐØ‡H�HˆkÐ/Ð0À€HÔOÙ˜UÐ/CÖDr   )N)NNN)#Ú
__future__r   r2   Údatetimer   Útypingr   r   Úutilsr   r   r	   r
   Ú
get_loggerÚ__name__rQ   r!   r   Úconfiguration_utilsr   r   r   r#   r)   r.   r5   r\   ra   rq   rƒ   rŠ   r•   r§   r¯   r´   r   r   r   Ú<module>r¼      s�   ðõ #ã 	Ý ß +ç jÓ jð 
×	Ò	˜HÓ	%€÷ Ý0å6ñ ×ÒÛõ.õ$õ(õ.õ`÷/òd$÷",#õ^õ>õ6õ&õBP÷Er   