+
    –°³jY÷  ã                   ó0  a € RM tn0 t ^ RIt^ RIt^ RIt^ RIt^ RIHtHtHt ^ RI	H
t
HtHt ^ RIHt ^ RIHtHtHt ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RIHtHtHtHtH t H!t! ^ RI"H#t# ^ R	I$H%t%H&t&H't(H)t)H*t*H+t+H,t,H-t- ^ R
I.H/t/H0t0 ^ RI1H2t2 ^ RI3H4t4 ^ RI5H6t6 ^ RI7H8t9 ^ RI:H;t; . RNOt<Rt=Rt>Rt?Rt@]A]B,          tC]4],          ]Pˆ                  ,          ]E,          ]F,          ]B,          tG]G]H]G,          ,          ]I]G,          ,          ]J]BR3,          ,          tK]J]B]K3,          tL]H]L,          tM]J]B]L]M,          3,          tN]A! 4       tO] ^ k ]P                   R 4       tQ] ! R R4      4       tR] ! R R]R4      4       tSROR R lltT ! R R4      tURPR ltVRRR R/R! R" lltWR# R$ ltXR% R& ltYR' R( ltZR) R* lt[]P¸                  ! 4       R+ R, l4       t]]P¸                  ! 4       R- R. l4       t^R/ R0 lt_R1 R2 lt`R3 R4 lta]P¸                  ! 4       R5 R6 l4       tbR7 R8 ltc]P¸                  ! 4       R9 R: l4       tdRRR R/R; R< llteRRR R/R= R> lltfRRR R/R? R@ lltgRA RB lthR R/RC RD lltiR R/RE RF lltjR R/RG RH lltk]R R/RI RJ ll4       tl]R R/RK RL ll4       tmR# )Qé    N)ÚCallableÚ	GeneratorÚIterable)ÚasdictÚ	dataclassÚfield)Úchain)ÚAnyÚcastÚno_type_check)ÚShardedTensor)Ú_broadcast_state_dictÚ_distribute_state_dictÚ_flatten_state_dictÚ_gather_state_dictÚ_offload_state_dict_to_cpuÚ_unflatten_state_dict)Ú_CHECKPOINT_PREFIX)ÚFullOptimStateDictConfigÚFullStateDictConfigÚFullyShardedDataParallelÚOptimStateDictConfigÚShardedOptimStateDictConfigÚShardedStateDictConfigÚStateDictConfigÚStateDictType)Ú._get_module_fsdp_state_if_fully_sharded_moduleÚFSDP_WRAPPED_MODULE)ÚHYBRID_SHARDING_STRATEGIES)ÚDTensor)Ú_IncompatibleKeys)ÚDistributedDataParallel)Útree_map_onlyÚ	ValueTypeÚStateDictOptionsÚ_flat_paramÚparam_groupsÚparamsÚstatec               #   óð   "  € \         P                  ! 4       p \         P                  ! 4         R x € V '       d   \         P                  ! 4        R # R #   T '       d   \         P                  ! 4        i i ; i5i©N)ÚgcÚ	isenabledÚdisableÚenable)Ú
is_enableds    Úh/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/distributed/checkpoint/state_dict.pyÚ_gc_contextr2   R   sC   é € ä—’“€JÜ‡J‚J„LðÛçÜ�IŠIŽKñ ø�:Ü�IŠI�Kð üs   ‚+A6®A ²!A6Á A3Á3A6c                   óP   a € ] tR t^]t o RtRtRtRtRtRt	Rt
RtRtV 3R ltRtV tR# )r%   a   
This dataclass specifies how get_state_dict/set_state_dict will work.

- ``full_state_dict``: if this is set to True, all the tensors in the
  returned state_dict will be gathered. No ShardedTensor and DTensor
  will be in the returned state_dict.

- ``cpu_offload``: offload all the tensors to cpu. To prevent CPU OOM, if
  ``full_state_dict`` is also true, then only the rank0 will get the
  state_dict and all other ranks will get empty state_dict.

- ``ignore_frozen_params``: if the value is True, the returned state_dict
  won't contain any frozen parameters -- the ``requires_grad`` is False.
  The default value is False.

- ``keep_submodule_prefixes`` (deprecated): when ``submodules`` is not None, this option
  indicates whether to keep the submodule prefixes from the state_dict keys.
  or example, if the submodule is ``module.pretrain`` and the full FQN of
  the parameter is ``pretrain.layer1.weight`` of the param. When this option
  is True, the parameter's key in the returned state_dict will be
  ``pretrain.layer1.weight``. If the options is False, the key will be
  ``layer1.weight``.
  Note that if ``keep_submodule_prefixes`` is False, there may be conflicted
  FQNs, hence there should be only one submodule in ``submodules``.

- ``strict``: the ``strict`` option when ``set_state_dict`` calls
  model.load_state_dict().

- ``broadcast_from_rank0``: when the option is True, rank0 should receive a
   full state_dict and will broadcast the tensors in the state_dict/
   optim_state_dict one by one to other ranks. Other ranks will receive
   the tensors and shard according to the local shards in the model and
   optimizer. ``full_state_dict`` must be set to True when using this option.
   This option currently only supports DTensor, not the legacy ShardedTensor.
FTÚ_fqn_modifiersc                óz   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   # )	é   Úfull_state_dictÚcpu_offloadÚignore_frozen_paramsÚkeep_submodule_prefixesÚstrictÚbroadcast_from_rank0Úflatten_optimizer_state_dictÚdsd_fqn_modifiers)ÚboolÚstr)ÚformatÚ__classdict__s   "€r1   Ú__annotate__ÚStateDictOptions.__annotate__]   s}   ø‡ ‚ ñL Ñ!ñM ñN ÑñO ñP Ñ&ñQ ñR "Ñ(ñS ñT ÑñU ñV Ñ&ñW ñX #'Ñ.ñY ñZ Ñ-ò[ ó    © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r7   r8   r9   r:   r;   r<   r=   r>   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©rB   s   @r1   r%   r%   ]   sB   ø‡ € ñ"ðH "€OØ€KØ!&ÐØ$(ÐØ€FØ!&ÐØ).Ð Ø-Ð÷[ ƒ rE   c                   ó”   a € ] tR t^�t o ]! ]R7      t]! ]R7      t]! ]R7      t	Rt
Rt]P                  t]! ]R7      tV 3R ltRtV tR# )Ú_StateDictInfo)Údefault_factoryTc                ó^  <€ V ^8„  d   Qh/ S[ S[S[P                  ,          S[S[P                  ,          3,          ;R&   S[ S[S[P                  ,          S[S[P                  ,          3,          ;R&   S[S[,          ;R&   S[;R&   S[;R&   S[;R&   S[S[	P                  ,          ;R&   # )r6   Úfqn_param_mappingÚshared_params_mappingÚsubmodule_prefixesÚhandle_modelÚhandle_optimÚfsdp_contextÚfsdp_modules)Údictr@   ÚtorchÚTensorÚFQNS_TÚsetr?   r   ÚlistÚnnÚModule)rA   rB   s   "€r1   rC   Ú_StateDictInfo.__annotate__�   s²   ø‡ ‚ áÙ‰e�l‰lÕÙ‘—‘Õð	õñ $ñ ñ  Ù‰e�l‰lÕÙ‘—‘Õð	õñ $ñ ñ ™C�Ñ=ñ ñ Ññ ñ Ññ ñ Ñ3ñ ñ ‘r—y‘y•/Ñ?ò rE   rF   N)rG   rH   rI   rJ   r   r[   rT   rU   r_   rV   rW   rX   Ú
contextlibÚnullcontextrY   r`   rZ   rL   rM   rN   rO   s   @r1   rQ   rQ   �   sV   ø‡ € ñ
 	˜dÔ#ð ñ 	˜dÔ#ð ñ $)¸Ô#=ÐØ€LØ€LØ'×3Ñ3€LÙ$)¸$Ô$?€L÷ ƒ rE   rQ   c                ót   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        R\        /# )r6   ÚmodelÚnamer>   Úskip_ddp_prefixÚskip_compiler_prefixÚreturn)ra   rb   r@   r?   r^   )rA   s   "r1   rC   rC   ž   sX   € ÷ FEñ FEÜ�9‰9ðFEä
ðFEô ðFEô ð	FEô
 ðFEô ñFErE   c                ó   € VP                  \        R4      pRV9  d   V0# VP                  R4      p. pT p\        V4       EF_  w  r‰\	        V\
        4      '       d@   V	R8w  d   \        RV	 R24      hVP                  pV'       g   VP                  V	4       KY  K[  \	        V\        4      '       d¼   V\        V4      ^,
          8  dh   WX^,           ,          \        8X  dP   RP                  V4      p
\        V\        4      pV
'       d   V
 R2p
VP                   Uu0 uF  qÊ V 2kK
  	  upu # \        V\        4      pV	\        8w  d    VP                  V	4       \        Wy4      pEK)  EK,  \	        V\         P"                  P$                  P&                  4      '       dB   V	R8w  d   \        RV	 R24      hVP(                  pV'       g   VP                  V	4       EKž  EK¡  \+        Wr4      '       dD   \        Wr4      ! 4       P-                  V	4      ;p'       d   \+        W}4      '       d   \        W}4      pVP                  V	4       V	\.        P0                  P                  P2                  8X  d&   V\        V4      ^,
          8w  d   \5        R4      hEKT  \        Wy4      pEKb  	  RP                  V4      P                  \        R4      0# u upi )	aµ  
This API is used to convert the name of a parameter to the FQNs. For FSDP
without `use_orig_params`, the name of FlatParameter can be mapped to
multiple original parameters. As a result, the return type of this function
is `set[str]`.

Args:
    module (nn.Module): the root model.
    name (str): the name
    skip_ddp_prefix (bool): whether to skip DDP's `module` prefix

Returns:
    The canonical FQNs based on the model traversal.
Ú Ú.ÚmodulezExpected 'module', got 'Ú'Ú	_orig_modzExpected '_orig_mod', got 'z-Expect `_extra_state` to be the last obj name)Úreplacer   ÚsplitÚ	enumerateÚ
isinstanceÚDDPÚAssertionErrorro   ÚappendÚFSDPÚlenÚ_FLAT_PARAMÚjoinÚgetattrÚ_fqnsr   r\   Ú_dynamoÚ
eval_frameÚOptimizedModulerq   ÚhasattrÚgetra   ÚmodulesÚ_EXTRA_STATE_KEY_SUFFIXÚRuntimeError)rg   rh   r>   ri   rj   Ú	obj_namesÚfqn_obj_namesÚcurr_objÚiÚcurr_obj_nameÚprefixÚ
flat_paramÚfqnÚremoved_fqns   &&&&&         r1   Ú	_get_fqnsr�   ž   se  € ð. �<‰<Ô*¨BÓ/€DØ
�$„Øˆvˆà—
‘
˜3“€IØ€MØ€HÜ% i×0ÑˆÜ�h¤×$Ò$Ø Ô(Ü$Ð'?À¸ÈaÐ%PÓQÐQØ—‘ˆHß"Ø×$Ñ$ ]Ö3ñ #ä˜¤$×'Ò'Ø”3�y“> AÕ%Ô%¨)¸µEÕ*:¼kÔ*IØŸ™ -Ó0�Ü$ X¬{Ó;�
ßØ &˜x q˜\�FØ4>×4DÒ4DÓEÑ4D¨S˜( 3 %Ó(Ñ4DÑEÒEÜ˜xÔ)<Ó=ˆHØÔ 3Ô3Ø×$Ñ$ ]Ô3Ü" 8Ó;“ò 4ô ˜¤%§-¡-×":Ñ":×"JÑ"J×KÒKØ Ô+Ü$Ð'BÀ=À/ÐQRÐ%SÓTÐTØ×)Ñ)ˆHß'Ø×$Ñ$ ]×3ò (ô
 �x×3Ò3Ü")¨(Ô"FÓ"H×"LÑ"LØ!ó#ð �;ö ô ˜x×5Ò5Ü#*¨8Ó#A˜Ø× Ñ  Ô/Ø¤§
¡
× 1Ñ 1× IÑ IÔIØœ˜I›¨Õ*Ô*Ü&Ð'VÓWÐWò +ô # 8Ó;“ñM 1ðP �H‰H�]Ó#×+Ñ+Ô,>ÀÓCÐDÐDùò7 Fs   ÄKc                   ó   € ] tR t^çtRtR# )Ú_EXTRA_STATErF   N)rG   rH   rI   rJ   rM   rF   rE   r1   r’   r’   ç   s   † ÛrE   r’   c              #   ód   aaa"  € \        4       oR  VVV3R lloS! V R4       Rj  x€L
  R#  L5i)c                óP   € V ^8„  d   QhR\         P                  R\        R\        /# )r6   ro   Úcurr_fqnrk   )ra   rb   r@   r   )rA   s   "r1   rC   Ú0_iterate_valid_model_state.<locals>.__annotate__î   s%   € ÷  *ñ  *œŸ	™	ð  *¬Sð  *´Yñ  *rE   c              3   óè  <"  € SP                  V 4       V'       d   V R 2MRpV P                  4        F`  w  r#VS9   d   K  \        V S4      '       d+   V\        V S4      ! 4       P	                  4       9   d   VRR pMV V 2pS! W44       Rj  x€L
  Kb  	  \        V P                  RR7      V P                  RR7      4       F"  w  r%W P                  9   d   K  V V 2pWE3x € K$  	  \        V P                  R\        P                  P                  4      \        P                  P                  8w  d9   V \        P                  P                  P                   2pV\!        4       3x € R# R#  Là5i)rn   rm   NF)ÚrecurseÚget_extra_stateéÿÿÿÿ)ÚaddÚnamed_childrenr‚   r}   Úvaluesr	   Únamed_buffersÚnamed_parametersÚ_non_persistent_buffers_setÚ	__class__ra   rb   r™   r„   ro   r…   r’   )	ro   r•   rh   Ú	submoduleÚnew_fqnÚobjr>   r˜   Úvisited_moduless	   &&    €€€r1   r˜   Ú+_iterate_valid_model_state.<locals>.recurseî   s]  øé € Ø×Ñ˜FÔ#ç%-�h�Z˜q‘>°2ˆØ%×4Ñ4Ö6‰OˆDØ˜OÔ+Ùô ˜Ð 1×2Ò2ØœG FÐ,=Ô>Ó@×GÑGÓIÔIð # 3 B˜-‘à%˜J t fÐ-�Ù˜yÓ2×2Ò2ñ  7ô Ø× Ñ ¨Ð Ó/°×1HÑ1HÐQVÐ1HÓ1Wö
‰IˆDð ×9Ñ9Ô9ÙØ!˜
 4 &Ð)ˆGØ�,Ôñ
ô �F×$Ñ$Ð&7¼¿¹×9RÑ9RÓSÜ�y‰y×(Ñ(ô)ð "˜
¤2§:¡:×#4Ñ#4×#LÑ#LÐ"MÐNˆGØœ<›>Ð)Ô)ñ	)ñ 3ùs   ƒBE2ÂE0ÂC!E2rm   N)r_   )rg   r>   r˜   r¥   s   &f@@r1   Ú_iterate_valid_model_stater§   ë   s*   úé € Ü&)£e€O÷ *ò  *ñD �u˜bÓ!×!Ô!ùs   …"0§.¨0Ú
submodulesÚoptionsc                óü   € V ^8„  d   QhR\         P                  R\        \        P                  P
                  R3,          R\        R\        \         P                  ,          R,          R\        R,          R\        /# )	r6   rg   Úoptims.Ú
optim_onlyr¨   Nr©   rk   )
ra   rb   Útupler\   ÚoptimÚ	Optimizerr?   r_   r%   rQ   )rA   s   "r1   rC   rC     sv   € ÷ }ñ }Ü�9‰9ð}ä”%—+‘+×'Ñ'¨Ð,Õ-ð}ô ð}ô
 ”B—I‘I• Õ%ð}ô  Õ$ð}ô ñ}rE   c               óì  € V'       d   \         P                  ! R\        ^R7       V'       d   V'       g   \        R4      hT;'       g    \	        4       p/ p/ p\        V 4       F¯  w  rx\        V\        4      '       d   K  \        W4      p	VP                  V4      p
V
e<   \        \        \        ,          WX,          4      P                  V	4       WX,          Wh&   MV	P                  4       WX&   V	 F  p
\        V\        4      '       d   K  W…V
&   K!  	  K±  	  \        VP!                  4       4       F*  w  r¼V F  p
\        \"        P$                  V4      Wj&   K!  	  K,  	  \        4       pV'       dk   \        V4      pV P'                  4        FK  w  r~Wã9  d   K  \        W4      p	\)        V	4      ^8w  d   \+        R4      hVP                  R V	 4       4       KM  	  VP,                  '       d   VP.                  '       g   \1        R4      h\2        P4                  ! V 4      pV'       dè   VP.                  '       dh   \7        VP8                  VP8                  R7      p\;        VP8                  VP8                  ;'       g    VP,                  R7      p\<        P>                  pM<\A        VP8                  R7      p\C        VP8                  R7      p\<        PD                  p\F        PH                  R	 4       p\J        PL                  ! VV VVVR
7      pM\F        PN                  p\Q        R/ \S        V4      BRVRVRVRVR\        \        \T        PV                  ,          V4      RV'       * R\)        V4      ^ 8„  /B # )zO
Verify the model and options passed by the user and generates _StateDictInfo.
z¼Getting submodules only model/optim state_dict is deprecated and will be removed in 2.5. This feature can be achieved by manually filtering out the state_dict returned from get_state_dict.©Ú
stacklevelz;Optimizers are not passed in but optim_only is set to True.z)Submodule FQN should only have 1 instancec              3   ó*   "  € T F	  q R 2x € K  	  R# 5i)rn   NrF   )Ú.0rŽ   s   & r1   Ú	<genexpr>Ú"_verify_options.<locals>.<genexpr>L  s   é € Ð%@¹4°C¨¨Q¥i»4ùs   ‚z?full_state_dict must be True when broadcast_from_rank0 is True.)Úoffload_to_cpuÚ
rank0_only)r·   c           	   3   ó@  "  € \         P                  ! 4       ;_uu_ 4        \         P                  ! R R\        R7       \        P
                  ! V VVVR7      ;_uu_ 4        Rx € RRR4       RRR4       R#   + '       g   i     L; i  + '       g   i     R# ; i5i)ÚignorezFSDP.state_dict_type)ÚmessageÚcategory©ro   Ústate_dict_typeÚstate_dict_configÚoptim_state_dict_configN)ÚwarningsÚcatch_warningsÚfilterwarningsÚFutureWarningry   r¾   r½   s   &&&&r1   Ú$fsdp_state_dict_type_without_warningÚ=_verify_options.<locals>.fsdp_state_dict_type_without_warningj  su   é € ô ×(Ò(×*Õ*Ü×'Ò'ØÐ&<Ä}õô ×)Ò)Ø!Ø$3Ø&7Ø,C÷	ö ó ÷÷	 +Ñ*÷÷ ú÷	 +×*Ð*üs5   ‚B A B
Á A7	Á%B
Á-
BÁ7BÂB
Â
B	Â	Br½   rT   rU   rV   rY   rZ   rW   rX   rF   ),rÁ   ÚwarnrÄ   r†   r%   r§   ru   r’   r�   rƒ   r   r_   r@   ÚupdateÚcopyr`   Úitemsr\   r]   Únamed_modulesrz   rw   r<   r7   Ú
ValueErrorry   rZ   r   r8   r   r   ÚFULL_STATE_DICTr   r   ÚSHARDED_STATE_DICTrd   ÚcontextmanagerÚ	functoolsÚpartialre   rQ   r   ra   rb   )rg   r«   r¬   r¨   r©   rT   rU   rh   ÚparamÚfqnsrŽ   Úparam_Úfqns_rV   ro   rZ   r¿   rÀ   r¾   rÅ   rY   s   &&&$$                r1   Ú_verify_optionsrÖ     s*  € ÷ Ü�ŠðIô Øõ	
÷ Ÿ&ÜØIó
ð 	
ð ×+Ð+Ô)Ó+€GàKMÐØOQÐÜ1°%Ö8‰ˆÜ�eœ\×*Ò*Ùä˜Ó%ˆØ×#Ñ# EÓ*ˆØŠ?Ü””S•Ð,Õ3Ó4×;Ñ;¸DÔAØ+<Õ+CÐ!Ò(ð (,§y¡y£{ÐÑ$ÛˆCÜ˜e¤\×2Ô2Ø). #Ó&ó ñ 9ô  Ð3×9Ñ9Ó;Ö<‰ˆÛˆCÜ)-¬e¯l©l¸FÓ)CÐ!Ó&ó ñ =ô $'£5ÐßÜ˜“_ˆ
Ø!×/Ñ/Ö1‰LˆDØÔ'ÙÜ˜UÓ)ˆDÜ�4‹y˜AŒ~Ü$Ð%PÓQÐQØ×%Ñ%Ñ%@¹4Ó%@Ö@ñ 2ð ×#×#Ð#¨G×,C×,CÐ,CÜØMó
ð 	
ô ×$Ò$ UÓ+€L÷ à×"×"Ð"Ü 3Ø&×2Ñ2¸w×?RÑ?Rô!Ðô '?Ø&×2Ñ2Ø#×/Ñ/×OÐO°7×3OÑ3Oô'Ð#ô ,×;Ñ;‰Oä 6Ø&×2Ñ2ô!Ðô 'BØ&×2Ñ2ô'Ð#ô ,×>Ñ>ˆOä	×	"Ñ	"ñ	ó 
#ð	ô$ !×(Ò(Ø0ØØ+Ø/Ø$;ô
‰ô "×-Ñ-ˆäñ 	Ü
�‹/ñ	à+ð	ð 4ð	ð .ð		ð
 "ð	ô œ$œrŸy™y�/¨<Ó8ð	ð $”^ð	ô ˜&“k A‘oò	ð 	rE   c                ób   € V ^8„  d   QhR\         \        \        3,          R\        R\        RR/# )r6   Úmodel_state_dictÚoptim_state_dictÚinfork   N)r[   r@   r$   ÚOptimizerStateTyperQ   )rA   s   "r1   rC   rC   “  s9   € ÷ +ñ +Üœ3¤	˜>Õ*ð+ä(ð+ô ð+ð 
ñ	+rE   c                 óÀ  € VP                    F  p\        V4      pVe   K  \        R4      h	  VP                  '       d˜   V '       g�   VP                  '       g~   VP
                  '       gl   VP                  '       d   VP                  '       gH   VP                  '       d6   VP                  '       g$   \        R\        P                  ! 4       : R24      hVP                  '       dM   V'       gE   VP                  '       d   VP                  '       g!   VP                  '       g   \        RV 24      hV  F#  p\        V9   g   K  \        V R\         R24      h	  R # )Nz)Expected a fsdp_state with a fsdp module.z}The option indicates that model state_dict is required to save or load, but model state_dict is empty.rank = dist.get_rank()=rn   zgThe option indicates that model state_dict is required to save, or load but optim state_dict is empty. z
 contains z6. This can happen if the model is not the root module.)rZ   r   rw   rW   rV   r9   r8   r7   r;   r<   r†   ÚdistÚget_rankrX   r{   )rØ   rÙ   rÚ   ro   Ú
fsdp_stateÚkeys   &&&   r1   Ú_verify_state_dictrá   “  s2  € ð
 ×#Ô#ˆÜCÀFÓKˆ
ØÔÜ Ð!LÓMÐMñ $ð 	××Ðß Ø×'×'Ð'Ø×)×)Ð)Ø×!×!Ð! d×&:×&:Ð&:Ø�K�KˆKØ×)×)Ð)äð'ä—m’m“oÑ' qð*ó
ð 	
ð ××Ðç Ø×%×%Ð%¨$×*>×*>Ð*>Ø×.×.Ð.äð:Ø:JÐ9KðMóð ó
  ˆÜ˜#ÖÜØ�%�z¤+ ð /*ð *óð ó  rE   c                óŽ   € V ^8„  d   QhR\         P                  \        P                  P                  ,          R\
        R\        /# )r6   r¤   Úapirk   )ra   rb   r\   r®   r¯   r@   r   )rA   s   "r1   rC   rC   Á  s4   € ÷ ñ œŸ	™	¤E§K¡K×$9Ñ$9Õ9ð Äð Ìñ rE   c                 óŠ   € \        W4      pV\        9   d-   \        P                  ! \        V P                  V4      V R 7      pV# ))Úself)r}   Ú_patched_state_dictrÐ   rÑ   r¡   )r¤   rã   Úcalls   && r1   Ú_state_dict_fnrè   Á  s7   € Ü�3Ó€DØÔ"Ô"Ü× Ò ¤¨¯©¸Ó!<À3ÔGˆØ€KrE   c                ór   € V ^8„  d   QhR\         P                  R\        R\        P                  R,          /# )r6   rg   rÚ   rk   N)ra   rb   rQ   rÝ   ÚProcessGroup)rA   s   "r1   rC   rC   È  s3   € ÷ ñ Ü�9‰9ðÜ*ðä	×Ñ˜ÕñrE   c                 ó  € \        V \        4      '       d   T pM6VP                  '       d#   \        \        VP                  ^ ,          4      pMR# VP                  \
        9   d   R# VP                  p\        V\        4      '       d   R# V# )r   N)ru   ry   rZ   r   Úsharding_strategyr   Úprocess_groupr­   )rg   rÚ   Úfsdp_modulerí   s   &&  r1   Ú_get_fsdp_process_grouprï   È  so   € ô �%œ×ÒØ‰Ø	×	×	Ð	Üœ4 ×!2Ñ!2°1Õ!5Ó6‰áà×$Ñ$Ô(BÔBÙà×-Ñ-€MÜ�-¤×'Ò'ÙØÐrE   c                ó€   € V ^8„  d   QhR\         \        \        3,          R\        R\         \        \        3,          /# )r6   Ú
state_dictrÚ   rk   )r[   r@   r
   rQ   )rA   s   "r1   rC   rC   Û  s4   € ÷ ñ Ü”Sœ#�X•ðÜ&4ðä	Œ#Œsˆ(…^ñrE   c                 ó  € VP                   '       dR   VP                  '       d%   \        P                  P	                  4       '       g   RMRp\        WP                  VR7      # VP                  '       d   \        V 4      # V # )r   )r8   Ú
ranks_onlyrF   )r   )r7   r8   r\   ÚdistributedÚis_initializedr   r   )rñ   rÚ   ró   s   && r1   Ú_maybe_full_or_cpu_state_dictrö   Û  sx   € ð ××Ðð ×$×$Ð$¬E×,=Ñ,=×,LÑ,L×,NÒ,Nñ àð 	ô
 "Ø×$4Ñ$4Àô
ð 	
ð 
×	×	Ð	Ü)¨*Ó5Ð5àÐrE   c                ór   € V ^8„  d   QhR\         P                  R\        R\        \        \
        3,          /# )r6   rg   rÚ   rk   )ra   rb   rQ   r[   r@   r$   )rA   s   "r1   rC   rC   î  s1   € ÷ ?;ñ ?;Ü�9‰9ð?;Ü*ð?;ä	Œ#Œyˆ.Õñ?;rE   c           	      óî  € VP                   '       g   / # VP                  4       ;_uu_ 4        \        V R 4      ! 4       pRRR4       \        XP	                  4       4       F�  p\        W4      p\        V4      ^8w  d   \        RV R\        V4       RV 24      h\        \        V4      4      pWS8w  g   KW  R R lpV! W54      '       g   \        RV RV 24      hVP                  V4      W%&   K‘  	  VP                  '       dp   / pV Fe  pVP                   FR  pVP                  V4      '       g   K  VP                  '       d   W%,          Wu&   K:  V\        V4      R p	W%,          Wy&   KT  	  Kg  	  TpVP                  '       dS   V P!                  4        F>  w  r:V
P"                  '       d   K  \        W4      pV F  pVP                  V4       K  	  K@  	  \%        W!4      #   + '       g   i     EL­; i)	rñ   NúExpected 1 FQN for key 'ú', got z: c                ó$   € V ^8„  d   QhR\         /# )r6   rk   )r?   )rA   s   "r1   rC   Ú+_get_model_state_dict.<locals>.__annotate__  s   € ÷ ñ ¤Dñ rE   c                 ó>  € \        V4      \        V 4      8¼  d   R # VP                  R4      pV P                  R4      p^ p\        V4       FN  w  rVWbV,          8X  d3   V^,          pV\        V4      8X  d   V\        V4      ^,
          8H  u # KD  VR9   d   KM   R # 	  R# )Frn   T)ro   rq   )rz   rs   rt   )rà   rŽ   Ú	fqn_splitÚ	key_splitÚfqn_idxÚkey_idxÚkey_names   &&     r1   ÚverifyÚ%_get_model_state_dict.<locals>.verify  s“   € Ü�s“8œs 3›xÔ'Ù ØŸI™I c›N�	ØŸI™I c›N�	Ø�Ü)2°9Ö)=Ñ%�GØ¨WÕ#5Ô5Ø 1�˜Ø"¤c¨)£nÔ4Ø#*¬c°)«n¸qÕ.@Ñ#@Ò@ñ 5à!Ð%<Ô<Ù â$ñ *>ñ rE   zAn unexpected key, z, exists. FQN is )rW   rY   rè   r`   Úkeysr�   rz   rw   ÚnextÚiterr†   ÚpoprV   Ú
startswithr:   r9   rŸ   Úrequires_gradrö   )rg   rÚ   rñ   rà   rÓ   rŽ   r  Únew_state_dictrŒ   r£   rÒ   s   &&         r1   Ú_get_model_state_dictr  í  s¾  € ð ××ÐØˆ	à	×	Ñ	×	Õ	Ü# E¨<Ô8Ó:ˆ
÷ 
ô �J—O‘OÓ%Ö&ˆÜ˜Ó$ˆÜˆt‹9˜Œ>Ü Ø*¨3¨%¨w´s¸4³y°kÀÀDÀ6ÐJóð ô ”4˜“:ÓˆØŽ:õñ" ˜#×#Ò#Ü"Ð%8¸¸Ð=NÈsÈeÐ#TÓUÐUØ(Ÿn™n¨SÓ1ˆJ‹Oñ= 'ð@ ××ÐØ/1ˆãˆCØ×1Ô1�Ø—~‘~ f×-Ò-ÙØ×/×/Ð/Ø*4­/�NÓ'à!¤# f£+ -Ð0�GØ.8­o�NÓ+ó 2ñ ð $ˆ
à× × Ð Ø×0Ñ0Ö2‰JˆCØ×"×"Ð"ÙÜ˜UÓ(ˆDÛ�Ø—‘˜sÖ#ó ñ	 3ô )¨Ó:Ð:÷s 
×	Ð	ús   ­G#Ç#G4	c                ó~   € V ^8„  d   QhR\         P                  R\        \        \        3,          R\
        R\        /# )r6   rg   rñ   rÚ   rk   )ra   rb   r[   r@   r$   rQ   r!   )rA   s   "r1   rC   rC   1  sC   € ÷ A
ñ A
Ü�9‰9ðA
ä”Sœ)�^Õ$ðA
ô ðA
ô ñ	A
rE   c                 óP  € VP                   '       d   V'       g   VP                  '       g   \        / / 4      # / p\        WP                  4       F»  w  rE\        WVP                  4      p\        V VVP                  R R R7      p\        Wg4       Fw  w  r‰VP                  '       d   \        P                  ! 4       ^ 8X  dC   W‰8w  d=   VP                  VR4      p
V
f#   VP                  '       d   \        RV R24      hMW¡V	&   WSV	&   Ky  	  K½  	  R pVP                  '       g   VP                  '       Ed�   \        4       pVP                  4        FS  p\        P                   ! V4      '       g   K!  VP#                  4       ^ 8”  g   K8  VP%                  VP&                  4       KU  	  \        P&                  ! R4      V9   d(   VP)                  \        P&                  ! R4      4       Rp\+        V4      ^ 8X  d/   VP%                  \        P,                  P/                  4       4       M\+        V4      ^8”  d   \1        R4      hVP                  '       d4   \3        VVVP                  4       VP                  VP4                  R7       M-VP                  '       d   \7        WVP                  4       R	7       VP9                  V4       VP;                  4       ;_uu_ 4        \=        \        \?        V R
4      ! WP                  VR7      4      uuRRR4       #   + '       g   i     R# ; i)F)ri   rj   NzMissing key: rn   ÚmetaTzMultiple devices found)Údevicer;   r8   ©r  Úload_state_dict)rñ   r;   Úassign) rW   r<   r!   r§   r>   r�   ÚziprÝ   rÞ   r  r;   r†   r7   r_   r�   r\   Ú	is_tensorÚdimr›   r  Úremoverz   Údistributed_c10dÚ_get_pg_default_devicerÌ   r   r8   r   rÈ   rY   r   rè   )rg   rñ   rÚ   Úlocal_state_dictrà   ÚvaluerÓ   Úfqns_with_prefixrŽ   Úfqn_with_prefixÚ
load_valuer  Údevicess   &&&          r1   Ú_load_model_state_dictr   0  si  € ð ××Ð§Z¸×8Q×8QÐ8QÜ   RÓ(Ð(àÐÜ0°×8NÑ8NÖO‰
ˆÜ˜ T×%;Ñ%;Ó<ˆÜ$ØØØ×"Ñ"Ø!Ø!&ô
Ðô %(¨Ö$?Ñ ˆCà×-×-Ð-´·²³ÀAÔ1EØÔ(Ø'Ÿ^™^¨C°Ó6�
ØÒ%Ø—{—{�{Ü*¨]¸3¸%¸qÐ+AÓBÐBð #ð 3=˜Ñ/Ø05˜_Ó-ó %@ñ Pð, €FØ× × Ð  D×$8×$8Ñ$8Ü“%ˆØ%×,Ñ,Ö.ˆEÜ�Š˜u×%Ô%¨%¯)©)«+¸®/Ø—‘˜EŸL™LÖ)ñ /ô
 �<Š<˜Ó 7Ô*Ø�N‰Nœ5Ÿ<š<¨Ó/Ô0ØˆFÜˆw‹<˜1ÔØ�K‰Kœ×-Ñ-×DÑDÓFÕGÜ�‹\˜AÔÜÐ5Ó6Ð6à×$×$Ð$Ü!ØØ Ø—{‘{“}Ø—{‘{Ø ×,Ñ,öð ×!×!Ð!Ü" :ÈÏÉËÕVØ×ÑÐ*Ô+à	×	Ñ	×	Õ	ÜÜÜ˜5Ð"3Ô4Ø%¯k©kÀ&ôó
÷ 
×	×	Ó	ús   Ë,LÌL%	c                óP   € V ^8„  d   QhR\         P                  P                  RR/# )r6   r®   rk   N)r\   r®   r¯   )rA   s   "r1   rC   rC   u  s$   € ÷ '&ñ '&œUŸ[™[×2Ñ2ð '&°tñ '&rE   c                óä  € V P                   '       d   R# V P                   F)  pV\        ,           F  pVP                  f   K    R# 	  K+  	  V P                   FF  pV\        ,           F2  pVP                  '       g   K  \
        P                  ! V4      Vn        K4  	  KH  	  . pV P                   Ff  pRV9   g   K  VP                  VR,          4       \        VR,          \
        P                  4      '       d   \
        P                  ! R4      MRVR&   Kh  	  V P                  RR7       V P                   F   pRV9   g   K  VP                  ^ 4      VR&   K"  	  V P                  RR7       R# )z@
Initialize optim states by calling the step() with zero grads.
NÚlrg        )ÚclosureT)Úset_to_none)r)   r'   Ú_PARAMSÚgradr
  r\   Ú
zeros_likerx   ru   r]   ÚtensorÚstepr  Ú	zero_grad)r®   Úparam_grouprÒ   Úlrss   &   r1   Ú_init_optim_stater.  u  s2  € ð ‡{‡{€{áð ×)Ô)ˆØ ¤×)Ð)ˆEØ�z‰zÔ%Ûó *ñ *ð
 ×)Ô)ˆØ ¤×)Ð)ˆEØ×"×"Ò"Ü"×-Ò-¨eÓ4�–
ó *ñ *ð €CØ×)Ô)ˆØ�;ÖØ�J‰J�{ 4Õ(Ô)ô ˜k¨$Õ/´·±×>Ò>ô —’˜SÔ!àð ˜Óñ *ð 
‡J�J�t€JÔð ×)Ô)ˆØ�;ÖØ #§¡¨£
ˆK˜Óñ *ð 
‡O�O €OÖ%rE   c                óR   € V ^8„  d   QhR\         R\        \        \        3,          /# )r6   rñ   rk   )rÛ   r[   r@   r$   )rA   s   "r1   rC   rC   Ÿ  s(   € ÷ _ñ _Ô*<ð _ÄÄcÌ9ÀnÕAUñ _rE   c           
     ó  a	a
€ R V	V
3R llo	R o
/ p\        \        V \        ,          4      P                  4        FG  w  r#\         RV 2pVP	                  S	! \        \
        \        \        3,          V4      V4      4       KI  	  \        \        V \        ,          4       Fc  pVP                  \        4      p\        \        \        ,          V4       F,  pVP                  4        F  w  rxW�\         RV RV 2&   K  	  K.  	  Ke  	  V# )a§  
This API flattens the optimizer state_dict to support optimizer resharding for
MPMD, e.g., pipeline parallelism.

Without the API, the original optimizer state_dict looks like:
{
    "state": {
        "layer1.weight": {
            "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor
        },
        "layer2.weight": {
            "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor
        },
    },
    "param_groups": [
        {
            "lr": 0.0,
            "betas": (0.9, 0.95), ...,
            "params": ["layer1.weight", "layer2.weight"]
        }
    ]
}

With this API, the optimizer state_dict looks like:
{
    "state.layer1.weight.step": 10,
    "state.layer2.weight.step": 10,
    "state.layer1.weight.exp_avg": SomeTensor,
    "state.layer2.weight.exp_avg": SomeTensor,
    "state.layer1.weight.exp_avg_sq": SomeTensor,
    "state.layer2.weight.exp_avg_sq": SomeTensor,
    "param_groups.layer1.weight.lr": 0.1,
    "param_groups.layer2.weight.lr": 0.1,
    "param_groups.layer1.weight.betas": (0.9, 0.95),
    "param_groups.layer2.weight.betas": (0.9, 0.95),
}

The "state" section supports arbitrary levels of nesting for optimizers like Shampoo.
c                ó€   € V ^8„  d   QhR\         \        \        3,          R\        R\         \        \        3,          /# )r6   Únested_dictrŒ   rk   )r[   r@   r
   r$   )rA   s   "r1   rC   Ú/_flatten_optim_state_dict.<locals>.__annotate__È  s5   € ÷ ñ Üœ#œs˜(•^ðÜ-0ðä	Œc”9ˆnÕ	ñrE   c                óì   <€ / pV P                  4        F\  w  r4\        V4      pV'       d   V RV 2MTp\        V\        4      '       d   VP	                  S! WF4      4       KP  S! V4       WBV&   K^  	  V# )zÝ
Recursively flatten a nested dictionary with dot-separated keys.

Args:
    nested_dict: The dictionary to flatten
    prefix: The prefix to prepend to all keys

Returns:
    Flattened dictionary with dot-separated keys
rn   )rÊ   r@   ru   r[   rÈ   )	r2  rŒ   Ú	flattenedrà   r  Ústr_keyÚfull_keyÚ_flatten_state_nested_dictÚ_raise_if_type_not_supporteds	   &&     €€r1   r8  Ú=_flatten_optim_state_dict.<locals>._flatten_state_nested_dictÈ  sw   ø€ ð +-ˆ	à%×+Ñ+Ö-‰JˆCä˜#“hˆGß06˜&˜  7 )Ñ,¸GˆHä˜%¤×&Ò&à× Ñ Ñ!;¸EÓ!LÖMñ -¨UÔ3Ø&+˜(Ó#ñ .ð ÐrE   c                 ó–   € \        V \        P                  \        \        \
        34      '       g   \        R \        V 4       R24      hR# )z[Flattening optimizer state_dict only supports tensor, int, float, dict states now. Type is rn   N)ru   r\   r]   ÚintÚfloatr[   ÚNotImplementedErrorÚtype)Úvs   &r1   r9  Ú?_flatten_optim_state_dict.<locals>._raise_if_type_not_supportedæ  sC   € Ü˜!œeŸl™l¬C´¼Ð=×>Ò>Ü%ðä ›7˜) 1ð&óð ñ ?rE   rn   )r   ÚDictValueTypeÚ_STATErÊ   rÈ   r[   r@   r
   ÚListDictValueTypeÚ_PGr  r&  r`   )rñ   ÚretrŽ   r)   Ústate_prefixr,  rÓ   Úkr@  r8  r9  s   &        @@r1   Ú_flatten_optim_state_dictrI  Ÿ  sä   ù€ ÷Rñ ò<ð !#€Cô œ=¨*´VÕ*<Ó=×CÑCÖE‰
ˆÜ ˜  3 %Ð(ˆØ�
‰
Ù&¤t¬D´´c°­N¸EÓ'BÀLÓQö	
ñ Fô Ô-¨z¼#­Ö?ˆØ�‰œwÓ'ˆÜœœS�	 4Ö(ˆCØ#×)Ñ)Ö+‘�Ø*+”s�e˜1˜S˜E  1 #Ð&Ó'ó ,ó )ñ @ð €JrE   c                ó’   € V ^8„  d   QhR\         P                  P                  R\        \        \
        3,          R\        R\        /# )r6   r®   rñ   rÚ   rk   )r\   r®   r¯   r[   r@   r$   rQ   rÛ   )rA   s   "r1   rC   rC     sI   € ÷ Eñ EÜ�;‰;× Ñ ðEä”Sœ)�^Õ$ðEô ðEô ñ	ErE   c                ó¨  € R R lp/ p. p\         V\        V/pV P                   EF(  pVP                  \        . /4       V\        ,           EFF  pVP
                  V,           EF*  p	W’P                  9   d0   Rp
V F&  pV\        8X  d   K  \         RV	 RV 2pWÁ9   d   Rp
 M	  MRp
V
'       g   KO  VR
,          \        ,          p\        V\        4      '       g   \        R\        V4       24      hVP                  V	4       VP                  '       g   Kµ  / WI&   V P                  V,           FZ  p\          RV	 RV 2pWñ9  d$   V! Wñ4      pV\        \        WI,          4      V&   K:  VV,          \        \        WI,          4      V&   K\  	  EK-  	  EKI  	  \        \        \        ,          VR
,          \        ,          4      ^ ,          pV F{  pV\        8X  d   K  V\         RV RV 2,          pWµR
,          9  d   VVR
,          V&   K@  VR
,          V,          V8w  g   KW  \!        RV RV RV R	VR
,          V,           R2	4      h	  EK+  	  V# )zí
This API unflattens the state_dict generated by _flatten_optim_state_dict().
Supports arbitrary levels of nesting in the state section through recursive reconstruction.

See the docstring of _flatten_optim_state_dict() for more detail.
c                ó€   € V ^8„  d   QhR\         R\        \         \        3,          R\        \         \        3,          /# )r6   Úflattened_keyÚflattened_dictrk   )r@   r[   r$   )rA   s   "r1   rC   Ú1_unflatten_optim_state_dict.<locals>.__annotate__  s6   € ÷ 0ñ 0Üð0Ü,0´´i°Õ,@ð0ä	Œc”9ˆnÕ	ñ0rE   c                óP  € V  R2p/ pVP                  4        FŠ  w  rEVP                  V4      '       g   K  V\        V4      R pVP                  R4      pTpVRR  F7  p	W˜9  d   / W‰&   \	        W‰,          \
        4      '       g   \        hW‰,          pK9  	  WXVR,          &   KŒ  	  V# )z¸
Reconstructs a potentially nested value from flattened keys.
For non-nested values, returns the value directly.
For nested values, reconstructs the nested structure with string keys.
rn   Nrš   )rÊ   r	  rz   rs   ru   r[   rw   )
rM  rN  rŒ   r2  rà   r  Úremaining_keyÚpartsÚcurrentÚparts
   &&        r1   Ú_reconstruct_nested_dictÚ=_unflatten_optim_state_dict.<locals>._reconstruct_nested_dict  s»   € ð "�? !Ð$ˆà&(ˆð )×.Ñ.Ö0‰JˆCð —>‘> &×)Ò)áð  ¤ F£ Ð.ˆMð "×'Ñ'¨Ó,ˆEà!ˆGð ˜c˜r›
�àÔ&Ø$&�G‘Mä! '¥-´×6Ò6Ü(Ð(Ø!�-’ñ #ð "'�E˜"•IÓñ; 1ð@ ÐrE   Frn   TúExpected list, got zaAll the parameters in the same parameter group should have the same saved param_group value. But z is z while other(s) is rš   )rC  rE  r'   rx   r&  rT   rU   ru   r`   rw   r?  r
  r)   r   rB  r@   r†   )r®   rñ   rÚ   rU  r)   Úpg_stateÚ
return_osdr,  rÒ   rŽ   Ú	in_paramsrH  Úflatten_keyr(   Ú
state_nameÚflattened_state_keyÚreconstructed_valueÚfirst_param_fqnr  s   &&&                r1   Ú_unflatten_optim_state_dictr`    sX  € õ0ðd €EØ"$€HÜ&,¨e´S¸(Ð%C€Jà×)Õ)ˆØ�‰œ "˜Ô&Ø ¤×)Ñ)ˆEØ×-Ñ-¨e×4Ñ4�ð ×4Ñ4Ô4Ø %�IÛ(˜Ø¤œ<Ù$Ü),¨¨Q¨s¨e°1°Q°CÐ&8˜Ø&Ô4Ø(,˜IÙò )ð !%�Iç Ùà! "�¤gÕ.�Ü! &¬$×/Ò/Ü(Ð+>¼tÀF»|¸nÐ)MÓNÐNØ—‘˜cÔ"ð ×*×*Ð*Ùð  �‘
Ø"'§+¡+¨e×"4Ð"4�JÜ-3¨H°A°c°U¸!¸J¸<Ð*HÐ'à*Ô<á.FØ/ó/Ð+ð
 0ô œ]¨E­JÓ7¸
ÓCð GQØ/õGœœ]¨E­JÓ7¸
ÓCô #5ô? 5ñ *ôf œt¤C�y¨(°2­,´wÕ*?Ó@ÀÕCˆÛˆAØ”GŒ|ÙØ¤#  a¨Ð'8¸¸!¸Ð=Õ>ˆEØ �Ô$Ø"'�˜•˜Q“Ø˜"•˜a• EÖ)Ü"ð=Ø=LÐ<MÈQÈqÈcð RØ˜Ð 3°H¸RµLÀµOÐ3DÀAðGóð ô ñm *ðF ÐrE   c                óž   € V ^8„  d   QhR\         P                  R\        \        P                  P
                  R3,          R\        R\        /# )r6   rg   Ú
optimizers.rÚ   rk   )ra   rb   r­   r\   r®   r¯   rQ   rÛ   )rA   s   "r1   rC   rC   Š  sT   € ÷ IAñ IAÜ�9‰9ðIAä”e—k‘k×+Ñ+¨SÐ0Õ1ðIAô ðIAô ñ	IArE   c                 ó2  € VP                   '       g   / # \        / \        . /pV EF  p\        V4       \	        VR 4      ! 4       pVP
                  '       Ed   VP                  4       ;_uu_ 4        \        P                  ! V VV\        W4      R7      pRRR4       V'       g   K�  \        V\        ,          P                  4       4       FF  pRV9   g   K  V\        ,          P                  V4      V\        ,          VP                  RR4      &   KH  	  V\        ,           F6  pV\        ,           Uu. uF  qfP                  RR4      NK  	  ppW‡\        &   K8  	  EMn\        \        P                   ! R VP"                   4       4      4      p\%        \'        V\)        \+        V4      4      4      4      p	/ p
V P-                  4        Ff  w  r¼\/        W4      p\+        V4      ^8w  d   \1        RV R\+        V4       24      h\3        \5        V4      4      pWÉ9  d   KV  Wœ,          pWúV&   WêV&   Kh  	  \        V\        ,          P                  4       4       F5  pW«,          pV\        ,          P                  V4      V\        ,          V&   K7  	  V\        ,           F-  pV\        ,           Uu. uF  qúV,          NK  	  upV\        &   K/  	  V'       g   EK±  \7        \8        V\        ,          4      P;                  V\        ,          4       \7        \<        V\        ,          4      P?                  V\        ,          4       EK  	  VP@                  '       d   \7        \B        \E        V4      4      p\G        W24      #   + '       g   i     ELð; iu upi u upi )	rñ   ©ÚgroupNrq   z
_orig_mod.rm   c              3   ó:   "  € T F  q\         ,          x € K  	  R # 5ir+   )r&  )r´   Úgs   & r1   rµ   Ú(_get_optim_state_dict.<locals>.<genexpr>¬  s   é € Ð-UÑBT¸Q´¯jªjÓBTùs   ‚rù   rú   )$rX   rC  rE  r.  rè   rZ   rY   ry   rÙ   rï   r`   r  r  rr   r&  r	   Úfrom_iterabler'   r[   r  Úrangerz   rŸ   r�   rw   r  r  r   rB  rÈ   rD  Úextendr=   rÛ   rI  rö   )rg   rb  rÚ   rÙ   r®   ÚosdrH  rg  r(   Úparam_pid_mappingÚfqn_pid_mappingrà   rÒ   rÓ   rŽ   Úpidre  s   &&&              r1   Ú_get_optim_state_dictrp  ‰  sÜ  € ð ××ÐØˆ	ä,2°B¼¸RÐ+@ÐÜˆÜ˜%Ô Ü˜U LÔ1Ó3ˆØ××ÑØ×"Ñ"×$Õ$Ü×+Ò+ØØØÜ1°%Ó>ô	�÷ %÷ ÙÜ˜#œf�+×*Ñ*Ó,Ö-�Ø !Ö#Ø?BÄ6½{¿¹ÈqÓ?Q�Cœ•K §	¡	¨,¸Ó ;Ó<ñ .ð œ—X�X�Ø?@Ä¾zÓJ¹z¸!Ÿ)™) L°"Ö5¹z�ÐJØ#”'“
ó ô œ%×-Ò-Ñ-UÀ%×BTÒBTÓ-UÓUÓVˆFÜ $¤S¨´´s¸6³{Ó1CÓ%DÓ EÐØ ˆOØ#×4Ñ4Ö6‘
�Ü  Ó,�Ü�t“9 ”>Ü(Ø2°3°%°w¼sÀ4»y¸kÐJóð ô œ4 ›:Ó&�ØÔ1Ùà'Õ.�Ø'* Ñ$à'* Ó$ñ 7ô  ˜C¤�K×,Ñ,Ó.Ö/�Ø%Õ*�ð $'¤v¥;§?¡?°3Ó#7�”F•˜CÓ ñ	 0ð œSŸ˜�ØBGÌÆ.Ó!QÁ.¸3°#×"6Ð"6Á.Ñ!Q�”g“ñ "÷ ÚäŒ]Ð,¬VÕ4Ó5×<Ñ<¸SÄ½[ÔIÜÔÐ 0´Õ 5Ó6×=Ñ=¸cÄ#½h×Gñs ðv ×(×(Ð(ÜÜÔ 9Ð:JÓ Kó
Ðô )Ð)9Ó@Ð@÷y %×$Ð$üò$ Kùò: "Rs   Á0$M;Ä3NÊ,NÍ;Nc          
      ó�   € V ^8„  d   QhR\         P                  R\        P                  P                  R\
        R\        R\
        /# )r6   rg   r®   rÙ   rÚ   rk   )ra   rb   r\   r®   r¯   rÛ   rQ   )rA   s   "r1   rC   rC   Ö  sO   € ÷ añ aÜ�9‰9ðaä�;‰;× Ñ ðaô )ðaô ð	aô
 ñarE   c           
     ó&  € / p. p\         V\        V/p/ p\        ;QJ d5    R \        \        V\         ,          4       4       F  '       d   K   RM*	  RM&! R \        \        V\         ,          4       4       4      '       d   V# VP
                   EF  pVP                  \        . /4       V\        ,           EFÒ  p	VP                  V	,           EF¶  p
W£P                  9   dU   Rp\        \        V\        ,          4       F2  pV
\        \        \        ,          V\        ,          4      9   g   K0  Rp M	  MRpV'       g   Kt  VR	,          \        ,          p\        V\        4      '       g   \        R\        V4       24      hVP                  V
4       V	P                   '       dg   V
\        \        V\         ,          4      9   d&   \        \        V\         ,          4      V
,          WJ&   M!VP"                  '       d   \%        RV
 R24      h\        \        V\        ,          4       FY  pV
\        \        \        ,          V\        ,          4      9   g   K0  \'        V\        ,          4      ^,
          V\)        V4      &   K[  	  EK¹  	  EKÕ  	  \'        V\        ,          4      ^ 8X  g   EK  . p\        \        V\        ,          4       FJ  p\'        \        \        \        ,          V\        ,          4      4      ^ 8X  g   K9  VP                  V4       KL  	  \'        V4      ^8w  d   \+        R4      h\'        V\        ,          4      \'        VP
                  4      8w  d   \+        R4      h\'        V\        ,          4      ^,
          V\)        X4      &   EK  	  \        \        V\        ,          4       FY  pVP-                  \)        V4      R	4      pVR	8X  d   K'  VP/                  4        F  w  ppV\        8X  d   K  VW_,          V&   K   	  K[  	  V# )
aÄ  
Extract the corresponding optim state_dict from ``optim_state_dict`` for
``optim`` and return the result optim state_dict.

Args:
    model (nn.Module): the root model.
    optim (torch.optim.Optimizer): the optimizer.
    optim_state_dict (Dict[str, ValueType]): the superset optim state_dict that
        contains the optim state_dict of ``optim``.
    info (_StateDictInfo): state dict information.

Returns:
    The optim state_dict of ``optim``.
c              3   óB   "  € T F  p\        V\        4      x € K  	  R # 5ir+   )ru   r<  )r´   rH  s   & r1   rµ   Ú*_split_optim_state_dict.<locals>.<genexpr>ð  s   é € Ð
UÑ'T !Œ:�aœ×ÐÓ'Tùs   ‚FTrW  z'Missing optimizer state for parameter 'z‡' in checkpoint. The parameter requires gradients but has no saved optimizer state. To load anyway, use StateDictOptions(strict=False).zÛThere are param groups that have zero parameters. In such a case, DSD only support exactly one param group with zero parameters.But the loaded state_dict has zero or more than one param groups that have zero parameters.z`When there is a parameter group that has zero parameters, multiple optimizers are not supported.rš   )rC  rE  Úallr   rB  r'   rx   r&  rT   rU   rD  r`   r@   ru   rw   r?  r
  r;   r†   rz   ÚidrÌ   rƒ   rÊ   )rg   r®   rÙ   rÚ   r)   rX  rY  Ú
pg_mappingr,  rÒ   rŽ   rZ  Úloaded_param_groupr(   rF  Úpg_idxrà   r  s   &&&&              r1   Ú_split_optim_state_dictrz  Ö  sC  € ð* €EØ"$€HÜ&,¨e´S¸(Ð%C€JØ!#€Jç
ƒsÑ
U¤t¬MÐ;KÌFÕ;SÔ'TÓ
U‡s‡s‚sÑ
U¤t¬MÐ;KÌFÕ;SÔ'TÓ
U×UÒUØÐà×)Õ)ˆØ�‰œ "˜Ô&Ø ¤×)Ñ)ˆEØ×-Ñ-¨e×4Ñ4�Ø×4Ñ4Ô4Ø %�IÜ.2Ü)Ð+;¼CÕ+@ö/Ð*ð ¤$¤t¬C¥yÐ2DÄWÕ2MÓ"NÖNØ(,˜IÙ!ò/ð !%�Iß Ùà! "�¤gÕ.�Ü! &¬$×/Ò/Ü(Ð+>¼tÀF»|¸nÐ)MÓNÐNØ—‘˜cÔ"Ø×&×&Ð&Øœd¤=Ð2BÄ6Õ2JÓKÔKÜ%)¬-Ð9IÌ&Õ9QÓ%RÐSVÕ%W˜š
ØŸŸ˜Ü*ØEÀcÀUð KRð Róð ô
 +/Ü%Ð'7¼Õ'<ö+Ð&ð œd¤4¬¥9Ð.@ÄÕ.IÓJÖJÜ=@ÀÌCÅÓ=QÐTUÕ=U˜
¤2Ð&8Ó#9Ó:ô	+ô7 5ñ *ôD ˆ{œ7Õ#Ó$¨×)àˆCÜ&*Ô+<Ð>NÌsÕ>SÖ&TÐ"Ü”tœD¤�IÐ'9¼'Õ'BÓCÓDÈÖIØ—J‘JÐ1Ö2ñ 'Uô �3‹x˜1Œ}Ü ð1óð ô Ð#¤CÕ(Ó)¬S°×1CÑ1CÓ-DÔDÜ ð=óð ô 25°ZÄµ_Ó1EÈÕ1IˆJ”rÐ,Ó-Ô.ño *ôr Ô-Ð/?ÄÕ/DÖEˆØ—‘¤ ;£°Ó4ˆØ�RŒ<Ùà%×+Ñ+Ö-‰JˆC�Ø”gŒ~Ùà$)ˆHÕ˜SÓ!ó	 .ñ Fð ÐrE   c          
      ó¢   € V ^8„  d   QhR\         P                  R\        \        P                  P
                  R3,          R\        R\        RR/# )r6   rg   rb  .rñ   rÚ   rk   N)ra   rb   r­   r\   r®   r¯   rÛ   rQ   )rA   s   "r1   rC   rC   ;  s`   € ÷ gNñ gNÜ�9‰9ðgNä”e—k‘k×+Ñ+¨SÐ0Õ1ðgNô #ðgNô ð	gNð
 
ñgNrE   c                 ó(  a€ VP                   '       g   R # V EF]  p\        V4       V'       dF   \        V9   d   \        WW#4      pM/\	        V\        \        \        \        3,          V4      V4      pM/ pVP                  '       Edš   V P                  4        EF>  w  rg\        W4      p\        WRR7      p	W‰8X  d   K&  \        V4      ^8w  d   \        RV R\        V4       24      hVP                  4       p
V	P                  4       pV\        ,           FV  p\        \        \        \         3,          V4      pV\"        ,           Uu. uF  qîP%                  W«4      NK  	  ppWý\"        &   KX  	  \        \&        V\        ,          4      p\)        VP+                  4       4       F/  pV
V9   g   K  VP                  V4      VVP%                  W«4      &   K1  	  EKA  	  VP-                  4       ;_uu_ 4        \.        P0                  ! V VV\3        W4      R7      pR R R 4       EM@VP4                  '       Ed.   RVn        \7        W3V4      pRVn        R oV3R lp\9        \:        P<                  VV4      pSf   \        R4      h\?        V4      w  pp\?        V4      w  ppVP@                  '       d   \C        VVSR	7       M\E        VVSR	7       V F:  pVV9  g   K  VV9  d   \        R
V R24      hVV,          VV&   VV,          VV&   K<  	  \G        VV4      pV\        ,           F8  p\"        V9  g   K  . \        \        \        \        3,          V4      \"        &   K:  	  \I        VR4      ! VR7       EK`  	  R # u upi   + '       g   i     L/; i)NF)rj   zExpected 1 FQN for 'rú   rd  Tc                 óŽ   <€ V P                  4       ^ 8”  d/   Sf   V P                  oV # SV P                  8w  d   \        R4      hV # )r   zDevice mismatch)r  r  rÌ   )Útr  s   &€r1   Ú_deviceÚ'_load_optim_state_dict.<locals>._devicez  sD   ø€ Ø—5‘5“7˜Q”;à’~Ø!"§¡˜ð �ð   1§8¡8Ô+Ü(Ð):Ó;Ð;Ø�rE   zExpected device to be setr  zExpected key 'z' in osd_mappingr  )rñ   )%rX   r.  rC  rz  r`  r   r[   r@   r$   rZ   rŸ   r�   rz   rw   r  rE  r
   r&  rr   rB  r`   r  rY   ry   Úoptim_state_dict_to_loadrï   r7   rp  r#   r\   r]   r   r<   r   r   r   rè   )rg   rb  rñ   rÚ   r®   rÙ   Úoriginal_fqnÚ_rÓ   Úfqns_with_compilerrŽ   Úfqn_with_compilerrg  Úvalrà   r(   Ú	osd_staterH  r  r  Úflatten_osdÚosd_mappingÚflatten_local_osdÚlocal_osd_mappingÚ	optim_keyÚpgr  s   &&&&                      @r1   Ú_load_optim_state_dictrŽ  :  sI  ø€ ð ××ÐÙäˆÜ˜%Ô ßÜ˜Ô#Ü#:Ø *ó$Ñ ô $?Øœ4¤¤S¬) ^Õ 4°jÓAÀ4ó$Ñ ð  "ÐØ××Ñð $)×#9Ñ#9×#;‘�Ü  Ó5�Ü%.Ø¸eô&Ð"ð Ô-Ùä�t“9 ”>Ü(Ø.¨|¨n¸GÄCÈÃIÀ;ÐOóð ð —h‘h“j�Ø$6×$:Ñ$:Ó$<Ð!Ø)¬#×.Ð.�AÜœt¤C¬ H�~¨qÓ1�CàGJÌ7Æ|óÙGSÀŸ™ CÖ;Á|ð ð ð $*œ“Lñ /ô !¤Ð0@ÄÕ0HÓI�	Ü˜iŸn™nÓ.Ö/�AØ˜a–xØGPÇ}Á}ÐUVÓGW˜	 !§)¡)¨CÓ"CÓDô 0ñ+ $<ð2 ×"Ñ"×$Õ$Ü#'×#@Ò#@ØØØ$Ü1°%Ó>ô	$Ð ÷ %Ñ$ð ×!×!Ñ!Ø#(ˆDÔ Ü4°U¸HÀdÓKÐØ#'ˆDÔ ØˆFõô œeŸl™l¨GÐ5EÓFˆAØŠ~Ü$Ð%@ÓAÐAÜ':Ð;KÓ'LÑ$ˆK˜Ü3FÐGWÓ3XÑ0ÐÐ0Ø×(×(Ð(Ü% kÐ3DÈVÖTä& {Ð4EÈfÕUó
 )�	ØÐ$5Ö5Ø ¨Ô3Ü,Ø,¨Y¨KÐ7GÐHóð ð 4?¸yÕ3IÐ% iÑ0Ø3>¸yÕ3IÐ% iÓ0ñ )ô  5Ø!Ð#4ó Ðð '¤s×+Ð+�Ü "Ö$Ø>@”Dœœc¤9˜nÕ-¨rÓ2´7Ó;ñ ,ô 	�uÐ/Ô0Ð<L×MÐMó} ùò@÷ %×$ús   Å	M<
Ç4$NÎNc          
      óÄ   € V ^8„  d   QhR\         P                  R\        \         P                  ,          R,          R\        R,          R\        \
        \        3,          /# )r6   rg   r¨   Nr©   rk   )ra   rb   r_   r%   r[   r@   r$   )rA   s   "r1   rC   rC   ¥  sQ   € ÷ " ñ " Ü�9‰9ð" ô ”B—I‘I• Õ%ð" ô  Õ$ð	" ô
 
Œ#Œyˆ.Õñ" rE   c          
     ó¸   € \        4       ;_uu_ 4        \        V RRVVR7      p\        W4      p\        V/ V4       VuuRRR4       #   + '       g   i     R# ; i)a  
Return the model state_dict of ``model``.

See ``get_state_dict`` for the detail usage.

Args:
    model (nn.Module): the nn.Module to the model.
    submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
        that belong to the submodules.
    options (StateDictOptions): the options to control how
        model state_dict and optimizer state_dict should be returned. See
        `StateDictOptions` for the details.

Returns:
    The state_dict for ``model``.

:rtype: typing.Dict[str, ValueType]
F©r¬   r¨   r©   NrF   )r2   rÖ   r  rá   )rg   r¨   r©   rÚ   rØ   s   &$$  r1   Úget_model_state_dictr’  ¥  sM   € ô0 
��ÜØØØØ!Øô
ˆô 1°Ó=ÐÜÐ+¨R°Ô6Ø÷ 
��‹ús   “*AÁA	c          
      ó*  € V ^8„  d   QhR\         P                  R\        P                  P                  \
        \        P                  P                  ,          ,          R\        \         P                  ,          R,          R\        R,          R\        /# ©r6   rg   rb  r¨   Nr©   rk   )	ra   rb   r\   r®   r¯   r   r_   r%   rÛ   )rA   s   "r1   rC   rC   Ê  sn   € ÷ * ñ * Ü�9‰9ð* ä—‘×%Ñ%¬´·±×1FÑ1FÕ(GÕGð* ô ”B—I‘I• Õ%ð	* ô
  Õ$ð* ô ñ* rE   c          
     ó(  € \        4       ;_uu_ 4        \        V\        P                  P                  4      '       d   V3M
\        V4      p\        V VRVVR7      p\        WV4      p\        / WT4       VuuRRR4       #   + '       g   i     R# ; i)a�  
Return the combined state_dict for optimizers.

See ``get_state_dict`` for the detail usage.

Args:
    model (nn.Module): the nn.Module to the model.
    optimizers (Union[None, Optimizer, Iterable[Optimizer]]):
        The optimizers that are used to optimize ``model``.
    submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
        that belong to the submodules.
    options (StateDictOptions): the options to control how
        model state_dict and optimizer state_dict should be returned. See
        `StateDictOptions` for the details.

Returns:
    The state_dict for ``optimizers``.

:rtype: OptimizerStateType
Tr‘  N)	r2   ru   r\   r®   r¯   r­   rÖ   rp  rá   )rg   rb  r¨   r©   rÚ   rÙ   s   &&$$  r1   Úget_optimizer_state_dictr–  Ê  s|   € ô6 
��ô ˜*¤e§k¡k×&;Ñ&;×<Ò<ð ‰Mä�zÓ"ð 	ô
 ØØØØ!Øô
ˆô 1°ÀDÓIÐÜ˜2Ð/Ô6Ø÷ 
��‹ús   “A"B Â B	c                ón  € V ^8„  d   QhR\         P                  R\        P                  P                  \
        \        P                  P                  ,          ,          R\        \         P                  ,          R,          R\        R,          R\        \        \        \        3,          \        3,          /# r”  )ra   rb   r\   r®   r¯   r   r_   r%   r­   r[   r@   r$   rÛ   )rA   s   "r1   rC   rC   ÷  s‰   € ÷ X2ñ X2Ü�9‰9ðX2ä—‘×%Ñ%¬´·±×1FÑ1FÕ(GÕGðX2ô ”B—I‘I• Õ%ð	X2ô
  Õ$ðX2ô Œ4””Y�ÕÔ!3Ð3Õ4ñX2rE   c          
     ó@  € \        4       ;_uu_ 4        \        V\        P                  P                  4      '       d   V3M
\        V4      p\        V VRVVR7      p\        W4      p\        WV4      p\        WVV4       WV3uuRRR4       #   + '       g   i     R# ; i)a(  
Return the model state_dict and optimizers state_dict.

``get_state_dict`` can process any module that is parallelized by PyTorch
FSDP/fully_shard, DDP/replicate, tensor_parallel/parallelize_module, and any
combination of these parallelisms. The main functions of ``get_state_dict``
are: 1.) returning a model and optimizer state_dict that can be resharded
with a different number of trainers and/or different parallelisms.
2.) hiding the parallelism-specific state_dict APIs. Users don't have to call
these APIs.
3.) sanity checking the result state_dict.

The keys of the result state dictionary are the canonical FQNs (Fully
Qualified Names).  A canonical FQN refers to the FQN based on a parameter's
position in an nn.Module hierarchy. More specifically, a canonical FQN to a
parameter is the FQN returned by ``module.named_parameters()`` or
``module.named_buffers()`` when the module is not distributed by any
parallelisms. Since the optimizer internally uses parameter IDs to represent
a parameter, there will be a conversion from the parameter IDs to the
canonical FQNs when calling this API.

``get_state_dict`` can also process a module that is not parallelized. In
such a case, ``get_state_dict`` only performs one function -- converting the
optimizer parameter IDs to the canonical FQNs.

Example:
    >>> # xdoctest: +SKIP
    >>> import torch
    >>> from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
    >>> from torch.nn.parallel import DistributedDataParallel as DDP
    >>> from torch.distributed.checkpoint.state_dict import get_state_dict

    >>> fsdp_model = FSDP(copy.deepcopy(model))
    >>> fsdp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)
    >>> ddp_model = DDP(copy.deepcopy(model))
    >>> ddp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)


    >>> ddp_state_dict, ddp_optim_state_dict = get_state_dict(ddp_model, ddp_optim)
    >>> fsdp_state_dict, fsdp_optim_state_dict = get_state_dict(
    ...     fsdp_model, fsdp_optim
    ... )

    >>> # if we simply call ddp_model.state_dict() and fsdp_model.state_dict(),
    >>> # the asserts will fail.
    >>> assert ddp_state_dict == fsdp_state_dict
    >>> assert ddp_optim_state == fsdp_optim_state_dict


Args:
    model (nn.Module): the nn.Module to the model.
    optimizers (Union[None, Optimizer, Iterable[Optimizer]]):
        The optimizers that are used to optimize ``model``.
    submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
        that belong to the submodules.
    options (StateDictOptions): the options to control how
        model state_dict and optimizer state_dict should be returned. See
        `StateDictOptions` for the details.

Returns:
    ``Tuple`` that contain model state_dict and optimizer state_dict.

:rtype: typing.Tuple[typing.Dict[str, ValueType], OptimizerStateType]
Fr‘  N)
r2   ru   r\   r®   r¯   r­   rÖ   r  rp  rá   )rg   rb  r¨   r©   rÚ   rØ   rÙ   s   &&$$   r1   Úget_state_dictr™  ÷  s‹   € ôP 
��ô ˜*¤e§k¡k×&;Ñ&;×<Ò<ð ‰Mä�zÓ"ð 	ô
 ØØØØ!Øô
ˆô 1°Ó=ÐÜ0°ÀDÓIÐÜÐ+¸tÔDØÐ1÷! 
��‹ús   “A.BÂB	c                ó  € V ^8„  d   QhR\         P                  R\        \         P                  \        \        \        3,          3,          \        \        \        3,          ,          R\        \        \        3,          /# )r6   rg   rñ   rk   )ra   rb   r[   r@   r$   )rA   s   "r1   rC   rC   R  sX   € ÷ "6ñ "6Ü�9‰9ð"6ä”R—Y‘Y¤¤S¬) ^Õ 4Ð4Õ5¼¼SÄ)¸^Õ8LÕLð"6ô 
Œ#Œyˆ.Õñ"6rE   c           
      ó  € V'       g   / # \        \        \        VP                  4       4      4      \        P
                  4      '       Ed   \        P                  ! R \        ^R7       \        \        \        P
                  \        \        \        3,          3,          V4      p/ pVP                  4        F�  w  rEV P                  4        F„  w  rgWt8w  d   K  \        W4      p\!        V4      ^8w  d   \#        R4      h\        \        V4      4       R2p	TP%                  VP                  4        U
Uu/ uF  w  r«Wš,           VbK  	  upp
4       K†  	  KŸ  	  V# \        \        \        \        3,          V4      # u upp
i )zÎPassing model_state_dict as a ``Dict[nn.Module, Dict[str, Any]]``is deprecated and will be removed in 2.5. If you need this feature, please preprocess the model_state_dict to achieve the same functionality.r±   z/FQNs for a submodule should only have 1 elementrn   )ru   r  r  r  ra   rb   rÁ   rÇ   rÄ   r   r[   r@   r$   rÊ   rË   r�   rz   rw   rÈ   )rg   rñ   Úcast_state_dictr  r¢   Úsub_state_dictrh   ÚmrÓ   rŒ   Úsubfqnr  s   &&          r1   Ú_unflatten_model_state_dictr   R  s:  € ÷ Øˆ	ä”$”t˜JŸO™OÓ-Ó.Ó/´·±×;Ó;Ü�Šð"ô Øõ	
ô œt¤B§I¡I¬t´C¼°NÕ/CÐ$CÕDÀjÓQˆØ/1ˆØ)8×)>Ñ)>Ö)@Ñ%ˆIØ ×.Ñ.Ö0‘�Ø”>Ùä  Ó-�Ü�t“9 ”>Ü(ØIóð ô !¤ d£Ó,Ð-¨QÐ/�Ø×%Ñ%ØAO×AUÑAUÔAWÔXÑAW±°�V•_ eÒ+ÑAWÒXöó 1ñ *Að Ðä”Dœœi˜Õ(¨*Ó5Ð5ùó	 Ys   Ä4E;c                óŒ   € V ^8„  d   QhR\         P                  R\        \        \        3,          R\
        R,          R\        /# )r6   rg   rØ   r©   Nrk   )ra   rb   r[   r@   r$   r%   r!   )rA   s   "r1   rC   rC   w  sH   € ÷ $Eñ $EÜ�9‰9ð$Eäœ3¤	˜>Õ*ð$Eô  Õ$ð	$Eô
 ñ$ErE   c          	     óÊ   € \        W4      p\        4       ;_uu_ 4        \        V RRVR7      p\        V/ V4       \	        WV4      uuRRR4       #   + '       g   i     R# ; i)aõ  Load the model state_dict.

The counterpart of ``get_model_state_dict`` to set the state_dict to the
model. See ``set_state_dict`` for the detail usage.

Args:
    model (nn.Module): the nn.Module to the model.
    model_state_dict: (Dict[str, ValueType]):
       the model state_dict to load. If the key of the ``model_state_dict``
       is nn.Module, the key is a submodule of ``model`` and the value should
       be the state_dict of the submodule. When loading the state_dict,
       the prefix of the submodule will be append to the state_dict.
    options (StateDictOptions): the options to control how
        model state_dict and optimizer state_dict should be loaded. See
        `StateDictOptions` for the details.

Returns:
    ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
        * **missing_keys** is a list of str containing the missing keys
        * **unexpected_keys** is a list of str containing the unexpected keys

:type model_state_dict: typing.Dict[str, ValueType]
F©r¬   r©   NrF   )r   r2   rÖ   rá   r   )rg   rØ   r©   rÚ   s   &&$ r1   Úset_model_state_dictr¤  w  sO   € ô: .IØó.Ðô 
��Ü˜u b°UÀGÔLˆäÐ+¨R°Ô6Ü% e¸tÓD÷	 
��‹ús   ž(AÁA"	c          
      óê   € V ^8„  d   QhR\         P                  R\        P                  P                  \
        \        P                  P                  ,          ,          R\        R\        R,          RR/# )r6   rg   rb  rÙ   r©   Nrk   )ra   rb   r\   r®   r¯   r   rÛ   r%   )rA   s   "r1   rC   rC   ž  sh   € ÷ (Jñ (JÜ�9‰9ð(Jä—‘×%Ñ%¬´·±×1FÑ1FÕ(GÕGð(Jô )ð(Jô
  Õ$ð(Jð 
ñ(JrE   c          	     ó   € \        4       ;_uu_ 4        \        V\        P                  P                  4      '       d   V3M
\        V4      p\        WRVR7      p\        / W$4       \        WW$4       RRR4       R#   + '       g   i     R# ; i)ad  Load the optimizers state_dict.

The counterpart of ``get_optimizer_state_dict`` to set the state_dict to the
optimizers. See ``set_state_dict`` for the detail usage.

WARN: ``set_optimizer_state_dict`` can only be called before ``backward()`` or after
    ``step()`` is called on the optimizers. Otherwise, the optimizer states won't be
    initialized correctly.

Args:
    model (nn.Module): the nn.Module to the model.
    optimizers (Union[Optimizer, Iterable[Optimizer]]):
        The optimizers that are used to optimize ``model``.
    optim_state_dict: OptimizerStateType:
        the optimizer state_dict to load.
    options (StateDictOptions): the options to control how
        model state_dict and optimizer state_dict should be loaded. See
        `StateDictOptions` for the details.

Returns:
    None

:type optim_state_dict: typing.OptimizerStateType
Tr£  N)	r2   ru   r\   r®   r¯   r­   rÖ   rá   rŽ  )rg   rb  rÙ   r©   rÚ   s   &&&$ r1   Úset_optimizer_state_dictr§  ž  sj   € ô> 
��ô ˜*¤e§k¡k×&;Ñ&;×<Ò<ð ‰Mä�zÓ"ð 	ô
 ˜u¸TÈ7ÔSˆä˜2Ð/Ô6Ü˜uÐ2BÔI÷ 
��Šús   “AA<Á<B	c                ó   € V ^8„  d   QhR\         P                  R\        P                  P                  \
        \        P                  P                  ,          ,          R\        \        \        3,          R\        R\        R,          R\        /# )r6   rg   rb  rØ   rÙ   r©   Nrk   )ra   rb   r\   r®   r¯   r   r[   r@   r$   rÛ   r%   r!   )rA   s   "r1   rC   rC   É  s|   € ÷ =Eñ =EÜ�9‰9ð=Eä—‘×%Ñ%¬´·±×1FÑ1FÕ(GÕGð=Eô œ3¤	˜>Õ*ð	=Eô
 )ð=Eô  Õ$ð=Eô ñ=ErE   c          	     óX  € \        W4      p\        4       ;_uu_ 4        \        V\        P                  P
                  4      '       d   V3M
\        V4      p\        WV'       * VR7      p\        W#V4       \        WW54       \        WV4      uuRRR4       #   + '       g   i     R# ; i)a€  Load the model state_dict and optimizers state_dict.

The counterpart of ``get_state_dict`` to set the state_dict to the model and
optimizers.  The given ``model_state_dict`` and ``optim_state_dict`` do not
have to be returned by ``get_state_dict`` but must meet the following
requirements: 1) all FQNs are canonical FQNs as defined in ``get_state_dict``,
2) if a tensor is sharded, it must be either a ShardedTensor or DTensor,
3) optimizer state_dict cannot contain the parameter IDs; the keys should be
the canonical FQNs.

WARN: ``set_state_dict`` can only be called before ``backward()`` or after ``step()``
    is called on the optimizers. Otherwise, the optimizer states won't be initialized
    correctly.

Args:
    model (nn.Module): the nn.Module to the model.
    optimizers (Union[Optimizer, Iterable[Optimizer]]):
        The optimizers that are used to optimize ``model``.
    model_state_dict: (Union[Dict[nn.Module, Dict[str, ValueType]], Dict[str, ValueType]]):
       the model state_dict to load. If the key of the ``model_state_dict``
       is nn.Module, the key is a submodule of ``model`` and the value should
       be the state_dict of the submodule. When loading the state_dict,
       the prefix of the submodule will be append to the state_dict.
    optim_state_dict: OptimizerStateType:
        the optimizer state_dict to load.
    options (StateDictOptions): the options to control how
        model state_dict and optimizer state_dict should be loaded. See
        `StateDictOptions` for the details.

Returns:
    ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
        * **missing_keys** is a list of str containing the missing keys of the model state_dict.
        * **unexpected_keys** is a list of str containing the unexpected keys of the model state_dict.

:type model_state_dict: typing.Dict[str, ValueType]
:type optim_state_dict: typing.OptimizerStateType
r£  N)r   r2   ru   r\   r®   r¯   r­   rÖ   rá   rŽ  r   )rg   rb  rØ   rÙ   r©   rÚ   s   &&$$$ r1   Úset_state_dictrª  É  s�   € ô\ .IØó.Ðô 
��ô ˜*¤e§k¡k×&;Ñ&;×<Ò<ð ‰Mä�zÓ"ð 	ô
 ØÐ.>Ô*>Èô
ˆô 	Ð+¸tÔDÜ˜uÐ2BÔIÜ% e¸tÓD÷ 
��‹ús   žA/BÂB)	c                óV   € V ^8„  d   QhR\         P                  R\        R,          RR/# )r6   rg   r©   Nrk   )ra   rb   r%   )rA   s   "r1   rC   rC     s1   € ÷ 12ñ 12Ü�9‰9ð12ô  Õ$ð12ð 
ñ	12rE   c               ó  aa€ \         P                  ! \        V VR7      oV3R lpW n        \         P                  ! \        V VR7      oR V3R llpW0n        \        P                  V4       \        P                  V4       R# )a·  Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model``.

Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model`` to
be a partial function to call ``get_state_dict`` and ``set_state_dict``.

Example:
    from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
    from torch.distributed.checkpoint.state_dict import patch_model_state_dict

    model = fsdp(model)
    patch_model_state_dict(model)

Args:
    model (nn.Module): the nn.Module to the model.
    options (StateDictOptions): the options to control how
        model state_dict and optimizer state_dict should be loaded. See
        `StateDictOptions` for the details.
Returns:
    None
)rg   r©   c                  ó   <€ S ! 4       # r+   rF   ©Ú_state_dict_calls   €r1   Ústate_dict_callÚ0_patch_model_state_dict.<locals>.state_dict_call,  ó   ø€ ÙÓ!Ð!rE   c                óF   € V ^8„  d   QhR\         \        \        3,          /# ©r6   rñ   ©r[   r@   r
   )rA   s   "r1   rC   Ú-_patch_model_state_dict.<locals>.__annotate__7  ó   € ÷ ;ñ ;¬¬c´3¨h­ñ ;rE   c                 ó   <€ S! V R 7       R# ))rØ   NrF   ©rñ   Ú_load_state_dict_calls   &€r1   Úload_state_dict_callÚ5_patch_model_state_dict.<locals>.load_state_dict_call7  ó   ø€ Ù¨z×:rE   N)rÐ   rÑ   r’  rñ   r¤  r  ræ   r›   )rg   r©   r°  r»  rº  r¯  s   &$  @@r1   Ú_patch_model_state_dictr¾    su   ù€ ô6 !×(Ò(ÜØØôÐõ"ð 'Ôä%×-Ò-ÜØØôÐ÷;ð ;ð 1Ôä×Ñ˜OÔ,Ü×ÑÐ0Ö1rE   c                ó¤   € V ^8„  d   QhR\         P                  R\        \        P                  P
                  R3,          R\        R,          RR/# )r6   rg   rb  .r©   Nrk   )ra   rb   r­   r\   r®   r¯   r%   )rA   s   "r1   rC   rC   C  sO   € ÷ ;5ñ ;5Ü�9‰9ð;5ô ”e—k‘k×+Ñ+¨SÐ0Õ1ð;5ô  Õ$ð	;5ð
 
ñ;5rE   c               óŽ  aa€ \         P                  ! \        V VVR7      oV3R lp\         P                  ! \        V VVR7      oR V3R llp\        P                  V4       \        P                  V4       \        V\        P                  P                  4      '       d   V3M
\        V4      pV F  pW5n        WEn        K  	  R# )a`  Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers``.

Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers`` to
be a partial function to call ``get_state_dict`` and ``set_state_dict``.

Note that if there are multiple optimizers, all of the optimizers will be patched.
So users only need to call one of the state_dict() to get the full result.

Example:
    from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
    from torch.distributed.checkpoint.state_dict import patch_model_state_dict

    model = fsdp(model)
    patch_model_state_dict(model)

Args:
    model (nn.Module): the nn.Module to the model.
    options (StateDictOptions): the options to control how
        model state_dict and optimizer state_dict should be loaded. See
        `StateDictOptions` for the details.
Returns:
    None
)rg   rb  r©   c                  ó   <€ S ! 4       # r+   rF   r®  s   €r1   r°  Ú4_patch_optimizer_state_dict.<locals>.state_dict_callh  r²  rE   c                óF   € V ^8„  d   QhR\         \        \        3,          /# r´  rµ  )rA   s   "r1   rC   Ú1_patch_optimizer_state_dict.<locals>.__annotate__r  r·  rE   c                 ó   <€ S! V R 7       R# ))rÙ   NrF   r¹  s   &€r1   r»  Ú9_patch_optimizer_state_dict.<locals>.load_state_dict_callr  r½  rE   N)rÐ   rÑ   r–  r§  ræ   r›   ru   r\   r®   r¯   r­   rñ   r  )rg   rb  r©   r°  r»  r®   rº  r¯  s   &$$   @@r1   Ú_patch_optimizer_state_dictrÇ  B  s´   ù€ ô> !×(Ò(Ü ØØØô	Ðõ"ô &×-Ò-Ü ØØØô	Ð÷;ð ;ô ×Ñ˜OÔ,Ü×ÑÐ0Ô1ô �j¤%§+¡+×"7Ñ"7×8Ò8ð 
‰ä�:Óð ó
 ˆØ*ÔØ 4Öó rE   c                óV   € V ^8„  d   Qh/ ^ \         9   d   \        \        ,          ;R&   # )r6   ræ   )Ú__conditional_annotations__r_   r   )rA   s   "r1   rC   rC      s    € × Ñ ÷Z +Ò *”Sœ•]Ñ *ò[ rE   )r^   ÚPrimitiveTyper$   rB  rD  rÛ   r%   r’  r–  r™  r¤  r§  rª  )r4   TT)r4   )orÉ  rd   rÐ   r,   rÁ   Úcollections.abcr   r   r   Údataclassesr   r   r   Ú	itertoolsr	   Útypingr
   r   r   r\   Útorch.distributedrô   rÝ   Útorch.nnra   Ú'torch.distributed._shard.sharded_tensorr   Ú#torch.distributed._state_dict_utilsr   r   r   r   r   r   Ú;torch.distributed.algorithms._checkpoint.checkpoint_wrapperr   Útorch.distributed.fsdpr   r   r   ry   r   r   r   r   r   Ú$torch.distributed.fsdp._common_utilsr   r   Ú"torch.distributed.fsdp._init_utilsr   Útorch.distributed.tensorr    Útorch.nn.modules.moduler!   Útorch.nn.parallelr"   rv   Útorch.utils._pytreer#   Ú__all__r{   rE  r&  rC  r_   r@   r^   r]   r<  r=  rÊ  r`   r­   r[   r$   rB  rD  rÛ   ræ   rÏ   r2   r%   rQ   r�   r’   r§   rÖ   rá   rè   rï   rö   Úno_gradr  r   r.  rI  r`  rp  rz  rŽ  r’  r–  r™  r   r¤  r§  rª  r¾  rÇ  rC   )rÉ  s   @r1   Ú<module>rÝ     s  øðç Ð Û Û 	Û ß 9Ñ 9ß 0Ñ 0Ý ß +Ñ +ã Ý  Ý Ý A÷÷ õ÷	÷ 	ó 	÷õ JÝ ,Ý 5Ý <Ý -ò€ð" €Ø€Ø
€Ø	€à	ˆS�€Ø˜-Õ'¨%¯,©,Õ6¸Õ<¸uÕDÀsÕJ€à�D˜Õ'Õ'¨%°Õ*>Õ>ÀÀcÈ;ÐFVÕAWÕWð 
ð �S˜)�^Õ$€Ø˜Õ'Ð Ø˜#˜}Ð/@Õ@Ð@ÕAÐ ñ &)£UÐ Ó *ð ×Ññó ðð ÷,.ð ,.ó ð,.ð^ ô@Ð%ó @ó ð@÷ FE÷R	ñ 	ô%"ðP}ð
 )-ð}ð (,÷}õ@+õ\õõ&ð$ ‡‚ƒô?;ó ð?;ðD ‡‚ƒôA
ó ðA
õH'&õT_õDEðP ‡‚ƒôIAó ðIAõXaðH ‡‚ƒôgNó ðgNðT" ð )-ð" ð (,÷	" ðJ* ð )-ð	* ð
 (,÷* ðZX2ð )-ð	X2ð
 (,÷X2õv"6ðJ$Eð (,÷	$EðN(Jð
 (,÷(JðV=Eð (,÷=EðD ð12ð (,ö12ó ð12ðl ð;5ð (,ö	;5ó ò;5rE   