+
    –°³j/4  ã                   óô  € ^ RI t ^ RIHt ^ RIHt ^ RIt^ RIHt ^ RI	H
t
 ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ R	IHt ^ R
IHt ^ RIHtHtHtHtHtHtHt ^ RIH t H!t! ^ RI"H#t#H$t$ ^ RI%H&t& ^ RI'H(t( ^ RI)H*t*H+t+H,t, ^ RI-H.t. ^ RI/H0t0 ^ RI1H2t2 ^ RI3H4t4 ]5]6]7]]8,          R,          ]]8,          3,          3,          t9R.t:R$R R llt;R%R R llt<R R lt=R$R R llt>R R lt? ! R  R!]4      t@R%R" R# lltAR# )&é    N)ÚSequence)Úcast)Ú_get_device_module)ÚShardedTensor)ÚTensorProperties)ÚShard)ÚChunkShardingSpec)Úunflatten_state_dict)ÚDefaultLoadPlanner)ÚBytesStorageMetadataÚChunkStorageMetadataÚMetadataÚMetadataIndexÚSTATE_DICT_TYPEr   ÚTensorStorageMetadata)ÚLoadPlanÚLoadPlanner)Ú_create_read_itemsÚ create_read_items_for_chunk_list)Úload_state_dict)ÚStorageReader)Ú_element_wise_addÚ_element_wise_subÚ_normalize_device_info)Ú_get_default_group)Ú_create_chunk_sharded_tensor)Ú_remote_device)ÚDTensorÚ!load_sharded_optimizer_state_dictc                ó<   € V ^8„  d   QhR\         R\        R\        /# )é   Úglobal_rankÚdevice_typeÚreturn)ÚintÚstr)Úformats   "Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/distributed/checkpoint/optimizer.pyÚ__annotate__r)   8   s!   € ÷ ñ ¤#ð ´Cð ÄSñ ó    c                 óš   € VR 8X  d   R # \        V4      pVP                  4       '       d!   \        WVP                  4       ,          4      # R # )Úcpu)r   Úis_availabler   Údevice_count)r"   r#   Údevice_modules   && r(   Ú_gen_rank_devicer0   8   sI   € Ø�eÔÙÜ& {Ó3€MØ×!Ñ!×#Ò#Ü%Ø }×'AÑ'AÓ'CÕCó
ð 	
ñ r*   c                óR   € V ^8„  d   QhR\         P                  R,          R\        /# )r!   ÚpgNr$   )ÚdistÚProcessGroupr	   )r'   s   "r(   r)   r)   C   s'   € ÷ ñ Ü×Ñ˜DÕ ðäñr*   c                 óä  € \         P                  P                  V 4      P                  pV f>   \	        \         P
                  ! 4       4       Uu. uF  pRV R\        W!4       2NK  	  ppML\	        V P                  4       4       Uu. uF)  pRV R\        \         P                  ! W4      V4       2NK+  	  pp\        ^ \        \        \        \        ,          ,          V4      R7      # u upi u upi )Núrank:Ú/©ÚdimÚ
placements)r3   Údistributed_c10dÚ_get_pg_default_deviceÚtypeÚrangeÚget_world_sizer0   ÚsizeÚget_global_rankr	   r   Úlistr   r&   )r2   Úpg_device_typeÚidxr:   s   &   r(   Ú_create_colwise_specrE   C   sé   € ô ×*Ñ*×AÑAÀ"ÓE×JÑJ€NØ	‚zô œT×0Ò0Ó2Ô3ó
á3�ð �C�5˜Ô*¨3Ó?Ð@ÓAÙ3ð 	ð 
ˆ
ô ˜RŸW™W›YÔ'ó
á'�ð �C�5˜Ô*¬4×+?Ò+?ÀÓ+HÈ.ÓYÐZÓ[Ù'ð 	ð 
ô ØÜœœ^¬cÕ1Õ2°JÓ?ôð ùò
ùò

s   ÁC(Â/C-c                óD   € V ^8„  d   QhR\         P                  R\        /# )r!   Úvalr$   )ÚtorchÚTensorÚbool)r'   s   "r(   r)   r)   W   s   € ÷ ñ œ5Ÿ<™<ð ¬Dñ r*   c                 ó  € \        V 4      \        J d•   \        V P                  4       4      ^ 8X  d   R# \        V P                  4       ^ ,          P                  4      \        J d   R# \        V P                  4       ^ ,          P                  4      \
        J d   \        R4      h R# \        V 4      \
        J dF   \        V P                  4      \
        J g   \        V P                  4      \        J d   \        R4      hR# )r   FTz1Cannot handle DTensor nested inside ShardedTensorzCannot handle nested DTensor)r=   r   ÚlenÚlocal_shardsÚtensorr   Ú
ValueErrorÚ_local_tensor)rG   s   &r(   Ú_is_nested_tensorrQ   W   sÄ   € ÜˆCƒy”MÓ!Üˆs×ÑÓ!Ó" aÔ'ÙÜ�× Ñ Ó" 1Õ%×,Ñ,Ó-´Ó>ÙÜ�× Ñ Ó" 1Õ%×,Ñ,Ó-´Ó8ÜÐPÓQÐQð 9ñ ô	 
ˆc‹”gÓ	ÜˆS×ÑÓ¤7Ó*¬d°3×3DÑ3DÓ.EÌÓ.VäÐ7Ó8Ð8Ùr*   c                ór   € V ^8„  d   QhR\         R\        \        ,          R\        R\        P
                  /# )r!   Úpropsr@   r#   r$   )r   r   r%   r&   rH   rI   )r'   s   "r(   r)   r)   f   s4   € ÷ ñ ÜðÜ#+¬C¥=ðÜ?Bðä
‡\�\ñr*   c           	      óX  € VR 8X  d3   \        \        P                  \        V4      P	                  4       4      pM.\        P                  ! V\        V4      P	                  4       4      p\        P
                  ! VV P                  V P                  V P                  V P                  VR7      # )r,   )r@   ÚdtypeÚlayoutÚrequires_gradÚ
pin_memoryÚdevice)
r   rH   rY   r   Úcurrent_deviceÚemptyrU   rV   rW   rX   )rS   r@   r#   rY   s   &&& r(   Ú_alloc_tensorr\   f   s†   € ð �eÔÜ”e—l‘lÔ$6°{Ó$C×$RÑ$RÓ$TÓU‰ä—’ØÔ+¨KÓ8×GÑGÓIó
ˆô �;Š;ØØ�k‰kØ�|‰|Ø×)Ñ)Ø×#Ñ#Øôð r*   c                ót   € V ^8„  d   QhR\         R\        \        \        P                  R,          3,          /# )r!   Ú
state_dictr$   N)r   ÚtupleÚSTATE_DICT_2D_LAYOUTr3   r4   )r'   s   "r(   r)   r)   z   s2   € ÷  ñ  Üð ä
Ô¤×!2Ñ!2°TÕ!9Ð9Õ:ñ r*   c                óÜ  € / pRpV P                  4        FÑ  w  r4RVP                  4       3W&   \        V4      '       g   K,  \        VP	                  4       4      ^8X  g   \        R4      h\        V\        4      '       g   \        R4      hVP	                  4       ^ ,          pVP                  P                  VP                  P                  3W&   VP                  P                  pKÓ  	  VV3# )a  
Load the right TP slice of the optimizer state.

This is not easy since the per-tensor slicing can't be inferred from checkpoint metadata.
We take advantage of the model state_dict producing a sliced ST to figure out what we need to load.
This is pretty fragile and it might be easier for FSDP to compute this info for us.
Returns a dictionary where keys are the same of the state_dict and the value is a tuple of
(offset, size) for the current rank TP slice.
N.B. The state_dict *MUST* come from FSDP.sharded_state_dict.
Nz%Cannot handle ST with multiple shardsz$Can only handle nested ShardedTensor)Úitemsr@   rQ   rL   rM   ÚAssertionErrorÚ
isinstancer   ÚmetadataÚshard_offsetsÚshard_sizesrN   Ú_process_group)r^   ÚspecsÚdp_pgÚkeyÚvalueÚshards   &     r(   Ú_get_state_dict_2d_layoutrn   z   sÔ   € ð #%€EØ&*€EØ ×&Ñ&Ö(‰
ˆØ˜EŸJ™J›LÐ)ˆ‰
Ü˜U×#Ô#Ü�u×)Ñ)Ó+Ó,°Ô1Ü$Ð%LÓMÐMÜ˜e¤]×3Ò3Ü$Ð%KÓLÐLØ×&Ñ&Ó(¨Õ+ˆEà—‘×,Ñ,Ø—‘×*Ñ*ðˆE‰Jð —L‘L×/Ñ/ŠEñ )ð 	Øðð r*   c                   ót   a a€ ] tR t^�t oV3R lV 3R lltV3R lR ltV3R lV 3R lltV3R ltRtVt	V ;t
# )	Ú_ReaderWithOffsetc                óJ   <€ V ^8„  d   QhRS[ S[S[S[,          3,          RR/# )r!   Úfqn_to_offsetr$   N)Údictr&   r   r%   )r'   Ú__classdict__s   "€r(   r)   Ú_ReaderWithOffset.__annotate__£   s)   ø€ ÷ ñ ¡d©3±¹µÐ+=Õ&>ð À4ñ r*   c                ól   <€ \         SV `  4        Wn        \        / 4      V n        / V n        / V n        R # ©N)ÚsuperÚ__init__rr   r   re   r^   Útranslation)Úselfrr   Ú	__class__s   &&€r(   ry   Ú_ReaderWithOffset.__init__£   s.   ø€ Ü‰ÑÔØ*ÔÜ  ›ˆŒØˆŒØˆÖr*   c                ó    <€ V ^8„  d   QhRS[ /# )r!   r$   )r   )r'   rt   s   "€r(   r)   ru   ª   s   ø€ ÷ *"ñ *"¡8ñ *"r*   c           	     ó>  € . p/ V n         V P                  P                  4        EFë  w  r#V P                  P                  V,          p\        V\        4      '       g   V\        W$V4      ,          pKN  W P                  9  d   V\        W$V4      ,          pKs  V P                  V,          p\        VP                  4       4      ^8X  g   \        R4      hVP                  4       ^ ,          p\        \        P                  ! \        VP                  P                   V4      4      \        P                  ! VP                  P"                  4      R7      .p\%        V\'        \(        V4      V4      pV F”  p	V	P*                  P,                  f   \        R4      h\/        V	P*                  P,                  V4      p
\0        P2                  ! V	P*                  \        P                  ! V
4      R7      pW°P                   V	P*                  &   K–  	  W,          pEKî  	  \5        V4      # )é   z Expected exactly one local shard)ÚoffsetsÚsizesz"dest_index.offset must not be None)Úoffset)rz   r^   rb   re   Ústate_dict_metadatard   r   r   rr   rL   rM   rc   r   rH   ÚSizer   rf   rg   r   r   r   Ú
dest_indexrƒ   r   ÚdataclassesÚreplacer   )r{   ÚrequestsÚfqnÚobjÚmdrƒ   Úoriginal_shardÚlocal_chunksÚreqsÚriÚoriginal_offsetÚoriginal_indexs   &           r(   Úcreate_local_planÚ#_ReaderWithOffset.create_local_planª   s¹  € ØˆØˆÔØŸ™×-Ñ-×/‰HˆCØ—‘×2Ñ2°3Õ7ˆBÜ˜c¤=×1Ò1ØÔ.¨s¸Ó<Õ<�Ùà×,Ñ,Ô,ØÔ.¨s¸Ó<Õ<�Ùà×'Ñ'¨Õ,ˆFä�s×'Ñ'Ó)Ó*¨aÔ/Ü$Ð%GÓHÐHØ ×-Ñ-Ó/°Õ2ˆNä$Ü!ŸJšJÜ)¨.×*AÑ*A×*OÑ*OÐQWÓXóô  Ÿ*š* ^×%<Ñ%<×%HÑ%HÓIô	ðˆLô 4Ø”TÔ/°Ó4°lóˆDó
 �Ø—=‘=×'Ñ'Ò/Ü(Ð)MÓNÐNÜ"3°B·M±M×4HÑ4HÈ&Ó"Q�Ü!,×!4Ò!4Ø—M‘M¬%¯*ª*°_Ó*Eô"�ð 3A× Ñ  §¡Ó/ñ ð Õ‹HñM 0ôN ˜Ó!Ð!r*   c                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# )r!   Úindexr$   )r   rH   rI   )r'   rt   s   "€r(   r)   ru   Ö   s#   ø€ ÷ Iñ I¡=ð I±U·\±\ñ Ir*   c                óT   <€ \         SV `  V P                  P                  W4      4      # rw   )rx   Úlookup_tensorrz   Úget)r{   r–   r|   s   &&€r(   r˜   Ú_ReaderWithOffset.lookup_tensorÖ   s$   ø€ Ü‰wÑ$ T×%5Ñ%5×%9Ñ%9¸%Ó%GÓHÐHr*   c                óT   <€ V ^8„  d   Qh/ S[ S[S[3,          ;R&   S[;R&   S[;R&   # )r!   rz   r^   re   )rs   r   r   r   )r'   rt   s   "€r(   r)   ru   �   s2   ø‡ ‚ Ù‘m¡]Ð2Õ3Ñ3ñ áÑñ ñ Ñò	 r*   )rr   re   r^   rz   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ry   r“   r˜   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r|   rt   s   @@r(   rp   rp   �   s.   ù‡ € ÷ó ÷*"ð *"÷XIó I÷s … r*   rp   c          
      ób   € V ^8„  d   QhR\         R\        R\        R\        R,          R\         /# )r!   Úmodel_state_dictÚoptimizer_keyÚstorage_readerÚplannerNr$   )r   r&   r   r   )r'   s   "r(   r)   r)   Ú   sF   € ÷ Qñ QÜ%ðQäðQô "ðQô ˜4Õð	Qô
 ñQr*   c                ó¾  € VP                  4       p\        V 4      w  rV\        P                  P	                  V4      P
                  p\        V4      pVfm   . p	\        \        P                  ! 4       4       F:  p
\        WzVP                  4       ,          4      pV	P                  RV
 RV 24       K<  	  \        ^ V	R7      pM\        V4      p/ p/ pVP                  P                  4        EF»  w  ppVP                   V,          pV^ ,          V8w  d   K*  \#        V\$        4      '       d   RWß&   KF  VP&                  P)                  4       ^8X  d&   \+        VP,                  VP&                  V4      Wß&   KŠ  Vfp   \/        \+        VP,                  VP&                  V4      \        P0                  ! 4       \        P                  ! 4       VP                  4       \3        4       R7      Wß&   Ký  V^,          pVP5                  VRVP&                  34      ^,          p\7        VP,                  P8                  VP,                  P:                  VP,                  P<                  VP,                  P>                  VP,                  P@                  R7      pVPC                  \D        PF                  ! V4      V4      p. p\        P0                  ! V4      pVPH                   Fm  p\K        \L        VPN                  4      PQ                  4       V8w  d   K2  VP                  \S        \+        VP,                  VPT                  V4      VR7      4       Ko  	  \V        PX                  ! VVVR	7      pVV9   d>   VV,          ^ ,          e,   \K        \Z        \\        ,          VV,          ^ ,          4      Wï&   VWß&   EK¾  	  \_        TTVe   \a        V4      MTR
7       \c        WÔP                   4      pV# )a@  
Load a state_dict in conjunction with FSDP sharded optimizer state.

This is the current recommended way to checkpoint FSDP.

Examples::

>>> # xdoctest: +SKIP
>>> import torch.distributed.checkpoint as dist_cp
>>> # Save
>>> model: torch.nn.Model
>>> optim_params = model.parameters()
>>> optim = torch.optim.SGD(optim_params, lr=0.01)
>>> # Save
>>> with FSDP.state_dict_type(model, StateDictType.SHARDED_STATE_DICT):
>>>     state_dict = {
>>>         "optimizer": FSDP.optim_state_dict(model, optim),
>>>         "model": model.state_dict()
>>>     }
>>>     dist_cp.save_state_dict(
>>>         state_dict=optim_state,
>>>         storage_writer=dist_cp.FileSystemWriter("checkpoint"),
>>>         planner=dist_cp.DefaultSavePlanner(),
>>>     )
>>>
>>> # Load
>>> with FSDP.state_dict_type(model_tp, StateDictType.SHARDED_STATE_DICT):
>>>     model_state_dict = model_tp.state_dict()
>>>     checkpoint = {
>>>         "model": model_state_dict
>>>     }
>>>     dist_cp.load_state_dict(
>>>         state_dict=checkpoint,
>>>         storage_reader=dist_cp.FileSystemReader(checkpoint_file),
>>>         planner=dist_cp.DefaultLoadPlanner(),
>>>     )
>>>     model.load_state_dict(checkpoint["model_state"])
>>>
>>>     optim_state = dist_cp.load_sharded_optimizer_state_dict(
>>>         model_state_dict,
>>>         optimizer_key="optimizer",
>>>         storage_reader=dist_cp.FileSystemReader("checkpoint"),
>>>     )
>>>
>>>     flattened_osd = FSDP.optim_state_dict_to_load(
>>>        model, optim, optim_state["optimizer"]
>>>     )
>>>
>>>     optim.load_state_dict(flattened_osd)
Nr6   r7   r8   z
<bytes_io>)ÚrankÚ
world_sizeÚnum_devices_per_noder2   )rU   rV   rW   Úmemory_formatrX   )rN   re   )Úprocess_group)r^   r§   r¨   )2Úread_metadatarn   r3   r;   r<   r=   r   r>   r?   r   r.   Úappendr	   rE   r„   rb   Úplanner_datard   r   r@   Únumelr\   Ú
propertiesr   Úget_rankr   r™   ÚShardTensorPropertiesrU   rV   rW   r­   rX   Úbuild_metadatarH   r…   Úshards_metadatar   r   Ú	placementrª   r   rg   r   Ú+_init_from_local_shards_and_global_metadatar   r%   r   rp   r
   )r¥   r¦   r§   r¨   re   Úlayout_specsrj   Údp_pg_device_typer/   r:   ÚiÚdevice_infoÚsharding_specr^   rr   rk   rl   Úkey_pathÚspec_keyÚ
alloc_sizer³   Úst_mdrM   Úcurrent_rankÚshard_mdÚsts   &&&&                      r(   r   r   Ú   sJ  € ðp ×+Ñ+Ó-€Hä3Ð4DÓEÑ€LÜ×-Ñ-×DÑDÀUÓK×PÑPÐÜ&Ð'8Ó9€Mà‚}Øˆ
Ü”t×*Ò*Ó,Ö-ˆAÜ0Ø! }×'AÑ'AÓ'CÕ#CóˆKð ×Ñ  a S¨¨+¨Ð7Ö8ñ	 .ô
 *¨a¸JÔG‰ä,¨UÓ3ˆð #%€Jà.0€MØ×2Ñ2×8Ñ8×:‰
ˆˆUØ×(Ñ(¨Õ-ˆØ�A�;˜-Ô'Ùä�eÔ1×2Ò2Ø*ˆJ‰OÙð �:‰:×ÑÓ Ô"Ü+Ø× Ñ  %§*¡*Ð.?óˆJ‹Oð Š]Ü:Ü˜e×.Ñ.°·
±
Ð<MÓNÜ—]’]“_Ü×.Ò.Ó0Ø%2×%?Ñ%?Ó%AÜ%Ó'ôˆJ‹Oð   •{ˆHØ%×)Ñ)¨(°T¸5¿:¹:Ð4FÓGÈÕJˆJä.Ø×&Ñ&×,Ñ,Ø×'Ñ'×.Ñ.Ø#×.Ñ.×<Ñ<Ø#×.Ñ.×<Ñ<Ø ×+Ñ+×6Ñ6ôˆJð "×0Ñ0´·²¸JÓ1GÈÓTˆEØˆLÜŸ=š=¨Ó/ˆLØ!×1Ô1�Üœ¨×(:Ñ(:Ó;×@Ñ@ÓBÀlÔRÙØ×#Ñ#ÜÜ,Ø!×,Ñ,¨h×.BÑ.BÐDUó ð "*ô	öñ 2ô ×JÒJØ˜e°5ôˆBð ˜<Ô'¨L¸Õ,BÀ1Õ,EÒ,QÜ%)¬(´3­-¸ÀhÕ9OÐPQÕ9RÓ%S�Ñ"à ˆJŒOñq ;ôv ØØ%à49Ò4EÔ! -Ô0È7õ	ô & j×2GÑ2GÓH€JàÐr*   )Úcudarw   )Br‡   Úcollections.abcr   Útypingr   rH   Útorch.distributedÚdistributedr3   Útorch._utilsr   Ú+torch.distributed._shard.sharded_tensor.apir   Ú0torch.distributed._shard.sharded_tensor.metadatar   rµ   Ú-torch.distributed._shard.sharded_tensor.shardr   Ú:torch.distributed._shard.sharding_spec.chunk_sharding_specr	   Ú)torch.distributed.checkpoint._nested_dictr
   Ú,torch.distributed.checkpoint.default_plannerr   Ú%torch.distributed.checkpoint.metadatar   r   r   r   r   r   Ú$torch.distributed.checkpoint.plannerr   r   Ú,torch.distributed.checkpoint.planner_helpersr   r   Ú.torch.distributed.checkpoint.state_dict_loaderr   Ú$torch.distributed.checkpoint.storager   Ú"torch.distributed.checkpoint.utilsr   r   r   Ú"torch.distributed.distributed_c10dr   Ú#torch.distributed.fsdp._shard_utilsr   Útorch.distributed.remote_devicer   Útorch.distributed.tensorr   rs   r&   r_   r%   r`   Ú__all__r0   rE   rQ   r\   rn   rp   r   © r*   r(   Ú<module>rÞ      s×   ðó Ý $Ý ã Ý  Ý +Ý Eõõ @Ý XÝ JÝ K÷÷ ñ ÷ G÷õ KÝ >÷ñ õ
 BÝ LÝ :Ý ,ð ˜C  x°¥}°tÕ';¸XÀc½]Ð'JÕ!KÐKÕLÐ ð
 (ð€÷
÷õ(÷õ( ôF:IÐ*ô :I÷zQñ Qr*   