+
    –°³jÜ6  ã                   ó„  € ^ RI t ^ RIHtHt ^ RIt^ RIHt ^ RIHu H	u H
t ^ RIHu Ht ^ RIHtHtHtHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ R	IHt ^ R
IH t H!t!H"t"Ht# ^ RI$H%t%H&t& R.t'R R lt(R R lt)R R lt*R R lt+R R lt,R R lt-R R lt.R R lt/R R lt0R R  lt1R! R" lt2 ! R# R]4      t3R# )$é    N)ÚAnyÚcast)ÚShardÚShardedTensorÚShardedTensorMetadataÚTensorProperties)ÚShardMetadata)ÚChunkShardingSpec)Ú_set_fsdp_flattened)ÚFSDPExtensions)Ú_create_chunk_sharded_tensor)Ú_remote_device)Ú
DeviceMeshÚDTensorÚ	Replicater   )Ú_flatten_tensorÚ_unflatten_tensorÚDTensorExtensionsc                óz   € V ^8„  d   QhR\         R\        \        P                  \        P                  3,          /# ©é   ÚtensorÚreturn©r   ÚtupleÚtorchÚSize)Úformats   "Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/distributed/tensor/parallel/fsdp.pyÚ__annotate__r       s,   € ÷ >ñ >”Wð >¤¤u§z¡z´5·:±:Ð'=Õ!>ñ >ó    c                 óø  € V P                   pVP                  ^8w  d   \        R4      hV P                  ^ ,          p^ .\	        V P                  4       4      ,          pVP                  ^ R7      pV P                  ^ ,          P                  4       '       d7   \        \        V4      P                  pV P                  V4      V,          pWcV&   \        P                  ! V4      V P                  P                  4       3# )é   ú&Only 1D DeviceMeshes currently handled)Úmesh_dim)Údevice_meshÚndimÚAssertionErrorÚ
placementsÚlenÚsizeÚis_shardr   ÚDShardÚdimr   r   Ú_local_tensor)r   r&   Ú	placementÚoffsetsÚ
num_chunksÚ	shard_dimÚ
chunk_sizes   &      r   Ú_get_boxr5      sÍ   € Ø×$Ñ$€KØ×Ñ˜1ÔÜÐEÓFÐFà×!Ñ! !Õ$€IØˆc”C˜Ÿ™›Ó&Õ&€GØ×!Ñ!¨1Ð!Ó-€Jð ×Ñ˜Õ×$Ñ$×&Ò&Üœ Ó+×/Ñ/ˆ	Ø—[‘[ Ó+¨zÕ9ˆ
Ø'�	Ñä�JŠJ�wÓ ×!5Ñ!5×!:Ñ!:Ó!<Ð=Ð=r!   c                ó†   € V ^8„  d   QhR\         R\        R\        \        P                  \        P                  3,          /# )r   r   Úidxr   )r   Úintr   r   r   )r   s   "r   r    r    1   s3   € ÷ >ñ >œð >¤sð >¬u´U·Z±ZÄÇÁÐ5KÕ/Lñ >r!   c                 ó‚   € \        V 4      w  r#\        P                  ! V Uu. uF  qDV,          NK  	  up4      V3# u upi ©N)r5   r   r   )r   r7   r1   r+   Úvals   &&   r   Ú_get_box_forr<   1   s8   € Ü˜VÓ$�M€GÜ�JŠJ©WÓ5©W c˜cŸ	˜	©WÑ5Ó6¸Ð=Ð=ùÒ5s   ¢<c                óz   € V ^8„  d   QhR\         R\        \        P                  \        P                  3,          /# r   r   )r   s   "r   r    r    6   s,   € ÷ *ñ *œ7ð *¤u¬U¯Z©Z¼¿¹Ð-CÕ'Dñ *r!   c                 ór   € V P                   pVP                  4       pVf   \        h\        W^ ,          4      # r:   )r&   Úget_coordinater(   r<   )r   r&   Úcoords   &  r   Ú_get_local_boxrA   6   s5   € Ø×$Ñ$€KØ×&Ñ&Ó(€EØ‚}ÜÐÜ˜ a¥Ó)Ð)r!   c                ó<   € V ^8„  d   QhR\         R\        R\        /# )r   ÚdtÚcurrent_rankr   )r   r8   r	   )r   s   "r   r    r    >   s!   € ÷ 
ñ 
¤ð 
¼ð 
Äñ 
r!   c                 óà   € V P                   pVP                  ^8w  d   \        R4      h\        V 4      w  r4\	        \        V4      \        V4      RV RV P                  P                   2R7      # )r#   r$   úrank:Ú/©Úshard_offsetsÚshard_sizesr0   )r&   r'   r(   rA   r	   Úlistr/   Údevice)rC   rD   Úmeshr1   Úsizess   &&   r   Ú_create_shard_md_from_dtrO   >   sf   € Ø�>‰>€DØ‡y�y�A„~ÜÐEÓFÐFä# BÓ'�N€GÜÜ˜7“mÜ˜“KØ˜,˜ q¨×)9Ñ)9×)@Ñ)@Ð(AÐBôð r!   c                óP   € V ^8„  d   QhR\         R\        P                  R\        /# )r   rC   Údt_pgr   )r   Úc10dÚProcessGroupr   )r   s   "r   r    r    K   s*   € ÷ %ñ %Üð%Ü×)Ñ)ð%äñ%r!   c                 ó*  € . p\         P                  ! V4      pV^ 8”  d   ^ M^pV P                  ^ ,          P                  4       '       d   VP	                  4       pM^p\        V4       Fb  p\        W4      w  rxTP                  \        \        V4      \        V4      RV^ 8”  d   TMT RV P                  P                   2R7      4       Kd  	  \        VV P	                  4       \        V P                  V P                  V P                   R7      R7      # )r   rF   rG   rH   )ÚdtypeÚlayoutÚrequires_grad)Úshards_metadatar+   Útensor_properties)ÚdistÚget_rankr)   r,   r+   Úranger<   Úappendr	   rK   r/   rL   r   r   rU   rV   rW   )	rC   rQ   Ú	shards_mdÚmy_rankÚscapegoat_rankÚshard_countÚir1   rN   s	   &&       r   Ú!_create_sharded_tensor_md_from_dtrc   K   sí   € ð €IÜ�mŠm˜EÓ"€GØ! Aœ+‘Q¨1€Nð 
‡}�}�QÕ× Ñ ×"Ò"Ø—j‘j“l‰àˆä�;ÖˆÜ% bÓ,‰ˆØ×ÑÜÜ" 7›mÜ  ›Kà¨a°!¬e™N¸ÐAÀÀ2×CSÑCS×CZÑCZÐB[Ð\ô	ö	
ñ  ô !Ø!Ø�W‰W‹YÜ*Ø—(‘(Ø—9‘9Ø×*Ñ*ô
ô	ð 	r!   c                óD   € V ^8„  d   QhR\         R\        P                  /# )r   rC   r   )r   rR   rS   )r   s   "r   r    r    s   s    € ÷ ñ ”7ð œt×0Ñ0ñ r!   c                 ór   € V P                   pVP                  ^8w  d   \        R4      hVP                  4       # )r#   r$   )r&   r'   r(   Ú	get_group)rC   rM   s   & r   Ú
_get_dt_pgrg   s   s/   € Ø�>‰>€DØ‡y�y�A„~ÜÐEÓFÐFØ�>‰>ÓÐr!   c                ó„   € V ^8„  d   QhR\         P                  R\        P                  R\        R\         P                  /# )r   Úspecr   Úrankr   )Ú
shard_specÚShardingSpecr   ÚTensorr8   )r   s   "r   r    r    z   s<   € ÷ ñ Ü
×
!Ñ
!ðÜ+0¯<©<ðÜ?Bðä×Ññr!   c                óL  € \        V \        4      '       g   V # RpV P                   FM  p\        \        V4      pVP                  4       V8X  g   K*  VP                  4       VP                  8w  g   KK  Rp M	  V'       d¥   \        P                  ! V 4      p \        V P                  4       Fu  w  rV\        \        V4      pVP                  4       V8X  g   K,  VP                  4       VP                  8w  g   KM  \	        RV RVP                   24      V P                  V&   Kw  	  V # )zÉ
Rewrite ``spec`` to match the device of ``tensor``.

FSDP.sharded_optim_state_dict sneakily ships optimizer state to CPU so if the original ShardingSpec
produces CUDA metadata, ST construction bombs.
FTrF   rG   )
Ú
isinstancer
   r)   r   r   rj   rL   ÚcopyÚdeepcopyÚ	enumerate)ri   r   rj   ÚrewriteÚprb   r0   s   &&&    r   Ú_rewrite_spec_if_neededru   z   sá   € ô �dÔ-×.Ò.Øˆð €GØ�_Œ_ˆÜ” Ó#ˆØ�6‰6‹8�tÖ §¡£
¨f¯m©mÖ ;ØˆGÙñ	 ÷
 Ü�}Š}˜TÓ"ˆä% d§o¡oÖ6‰LˆAÜœ^¨YÓ7ˆIØ�~‰~Ó 4Ö'¨I×,<Ñ,<Ó,>À&Ç-Á-Ö,Oä%3°e¸D¸6ÀÀ6Ç=Á=À/Ð4RÓ%S�—‘ Ó"ñ	 7ð €Kr!   c                óœ   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        P                  R\         P                  /# )r   r   rj   Ú
world_sizeÚnum_devices_per_nodeÚpgr   )r   rm   r8   rZ   rS   )r   s   "r   r    r    ™   sZ   € ÷ I
ñ I
Ü�L‰LðI
ä
ðI
ô ðI
ô ð	I
ô
 	×ÑðI
ô ‡\�\ñI
r!   c           	      óÆ  € \        V 4      \        J dá   \        V P                  4       4      ^8w  d   \        hV P                  4       p\        VVVVV4      pV P                  4       ^ ,          p\        V\        P                  ! VP                  4      4      .p\        P                  ! V P                  4       4      p	RV	P                  n        \        P                  ! VV	V P                  RR7      p
V
# \        V 4      \        J dÎ   V P                   pVP"                  ^8w  d   \	        R4      hV P$                  p\        VVV\&        P(                  P+                  4       V4      p\-        V 4      p\        V\/        V \0        P2                  ! V4      4      4      .p\5        W4      p	RV	P                  n        \        P                  ! VV	VRR7      p
V
# \        V VVVV4      # )r#   F)Úsharded_tensor_metadataÚprocess_groupÚ
init_rrefsr$   )Útyper   r*   Úlocal_shardsr(   Úlocal_tensorr   r   rp   rq   ÚmetadatarY   rW   Ú+_init_from_local_shards_and_global_metadataÚ_process_groupr   r&   r'   r/   r   ÚacceleratorÚdevice_countrg   rO   rZ   r[   rc   )r   rj   rw   rx   ry   Úinner_paramÚinner_stÚouter_local_shardÚshardsÚst_metaÚst_outerr&   rQ   s   &&&&&        r   Ú_chunk_tensorrŒ   ™   sÅ  € ô ˆFƒ|”}Ó$Üˆv×"Ñ"Ó$Ó%¨Ô*Ü Ð à×)Ñ)Ó+ˆÜ/ØØØØ Øó
ˆð #×/Ñ/Ó1°!Õ4Ðä�(œDŸMšMÐ*;×*DÑ*DÓEÓFð
ˆô —-’- §¡Ó 1Ó2ˆØ27ˆ×!Ñ!Ô/ä ×LÒLØØ$+Ø ×/Ñ/Øô	
ˆð ˆÜ	ˆf‹œÓ	 Ø×(Ñ(ˆØ×Ñ˜qÔ Ü Ð!IÓJÐJà×*Ñ*ˆä/ØØØÜ×Ñ×*Ñ*Ó,Øó
ˆô ˜6Ó"ˆô �(Ô4°V¼T¿]º]È5Ó=QÓRÓSð
ˆô 4°FÓBˆØ27ˆ×!Ñ!Ô/ä ×LÒLØØ$+ØØô	
ˆð ˆä+ØØØØ Øó
ð 	
r!   c                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# ©r   r   rj   r&   r   )r   rm   r8   r   r   )r   s   "r   r    r    å   s4   € ÷ >
ñ >
Ü�L‰Lð>
ä
ð>
ô ð>
ô ñ	>
r!   c                ó  € Ve   VP                  4       MRpVf   \        R4      hVP                  ^8  d   \        RVP                   R2R4      hV P                  4       P	                  4       p \        V \        P                  4      '       d§   \        V \        4      '       g‘   \        VP                  4       Uu. uF  p\        4       NK  	  pp\        VP                  4       Uu. uF  p\        4       NK  	  pp\        ^ 4      V^ &   \        P                  ! WVRR7      P                  VVR7      # V P                  pV^ ,          pV P                  4       p \        VP                  4       Uu. uF  p\        4       NK  	  ppW…R	&   \        VP                  4       U	u. uF  p	\        4       NK  	  pp	\        ^ 4      VR
&   W†R	&   \        P                  ! WVRR7      P                  VVR7      # u upi u upi u upi u up	i )z�
Shard a tensor to chunks along the first dimension.

The local rank will gets its corresponding chunk as the local tensor to create a DTensor.
Nz4No parent device_mesh is found for FSDP device_mesh.z!Found parent device_mesh of ndim=Ú,zbut meshes must be at least 2D.F)Ú	run_check©r&   r)   éÿÿÿÿéþÿÿÿ)Ú_get_root_meshÚRuntimeErrorr'   ÚdetachÚclonero   r   rm   r   r\   r   r-   Ú
from_localÚredistributer)   Úto_local)
r   rj   r&   Ú	root_meshÚ_Úreplicate_placementsÚshard_placementsÚtp_placementsÚtp_placementrb   s
   &&&       r   Ú_chunk_dtensorr¢   å   sÜ  € ð 1<Ò0G�×*Ñ*Ô,ÈT€IØÒÜÐQÓRÐRØ‡~�~˜ÔÜØ/°	·±Ð/?¸qÐAØ-ó
ð 	
ð �]‰]‹_×"Ñ"Ó$€Fô
 �&œ%Ÿ,™,×'Ò'´
¸6Ä7×0KÒ0Kô 6;¸9¿>¹>Ô5JÓKÑ5J°¤	¦Ñ5JÐÐKÜ16°y·~±~Ô1FÓGÑ1F¨AœIžKÑ1FÐÐGÜ$ Q›iÐ˜Ñä×!Ò!ØÐ3¸uô
ç
‰,Ø!Ø'ð ó 
ð	
ð ×)Ñ)ˆØ$ QÕ'ˆà—‘Ó"ˆô 6;¸9¿>¹>Ô5JÓKÑ5J°¤	¦Ñ5JÐÐKØ#/˜RÑ Ü16°y·~±~Ô1FÓGÑ1F¨AœIžKÑ1FÐÐGÜ% a›yÐ˜ÑØ+˜Ñä×!Ò!ØÐ3¸uô
ç
‰,Ø!Ø'ð ó 
ð	
ùò9  LùÚGùò*  LùâGs   Â<G2Ã(G7Å1G<Æ!Hc                ó�   € V ^8„  d   QhR\         P                  R\        \         P                  \        \        ,          3,          /# r   ©r   rm   r   rK   r   )r   s   "r   r    r    &  s2   € ÷ 	7ñ 	7Ü�L‰Lð	7ä
Œ5�<‰<œœe�Ð$Õ%ñ	7r!   c                 ó  € \        \        V 4      P                  4       p\        V4      ^8X  dJ   \	        V^ ,          P
                  4      \        J d&   V^ ,          P
                  pVP                  4       pTp T \        V4      ^ 8”  d   V3# . 3# )r#   )r   r   r   r*   r~   r   )r   r‰   Úinner_tensors   &  r   Ú_pre_load_state_dictr§   &  sz   € ô ”- Ó(×5Ñ5Ó7€FÜ
ˆ6ƒ{�aÔœD ¨¥×!1Ñ!1Ó2´mÓCØ˜a•y×'Ñ'ˆØ×*Ñ*Ó,ˆØˆàœc &›k¨Aœo�FÐ6Ð6°2Ð6Ð6r!   c                ó^   € V ^8„  d   QhR\         R\        R,          R\        P                  /# ©r   r   Úparent_meshNr   ©r   r   r   rm   )r   s   "r   r    r    2  s/   € ÷ ñ Üðä˜dÕ"ðô ‡\�\ñr!   c                ó6  € WP                   8w  d   \        h\        \        P                  ! V P
                  4      4      p\        \        V4      ^,
          4       F  p\        4       W#&   K  	  V P                  V P                   VR7      p V P                  4       # )zGAll gather a DTensor in its FSDP dimension and return the local tensor.r’   )r&   r(   rK   rp   rq   r)   r\   r*   r   rš   r›   )r   rª   r)   rb   s   &&  r   Ú_all_gather_dtensorr­   2  s‚   € ð
 ×(Ñ(Ô(ÜÐä”d—m’m F×$5Ñ$5Ó6Ó7€Jô ”3�z“? QÕ&Ö'ˆÜ!›ˆ
‹ñ (à× Ñ Ø×&Ñ&Øð !ó €Fð
 �?‰?ÓÐr!   c                   ó²   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltV3R lR ltRV3R	 lR
 lltV3R lR lt	V3R lR lt
V3R lR ltRtVtV ;t# )r   iG  zÞ
DTensorExtension is the TensorFlattener extension needed for 2D FSDP + TP.

This is the implementation for FSDPExtensions defined in
https://github.com/pytorch/pytorch/blob/main/torch/distributed/fsdp/_fsdp_extensions.py
c                ó   <€ V ^8„  d   QhRR/# )r   r   N© )r   Ú__classdict__s   "€r   r    ÚDTensorExtensions.__annotate__O  s   ø€ ÷ 
ñ 
¨ñ 
r!   c                óš   <€ \         SV `  4        R V n        Wn        \        P
                  P                  V P                  4      V n        R # r:   )ÚsuperÚ__init__Úcompute_streamÚdevice_handler   Ú_dynamoÚdisableÚpost_unflatten_transform)Úselfr·   Ú	__class__s   &&€r   rµ   ÚDTensorExtensions.__init__O  s>   ø€ Ü‰ÑÔØ"ˆÔØ*Ôô ).¯©×(=Ñ(=Ø×)Ñ)ó)
ˆÖ%r!   c                ór   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[R,          3,          /# )r   r   r   N)r   rm   r   r   )r   r±   s   "€r   r    r²   Y  s5   ø€ ÷ 'ñ 'á—‘ð'ñ 
‰u�|‰|™S 4�ZÐ'Õ	(ñ'r!   c                ó   € \        V4      # r:   )r   ©r»   r   s   &&r   Úpre_flatten_transformÚ'DTensorExtensions.pre_flatten_transformY  s   € ô ˜vÓ&Ð&r!   c                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# )r   r   Úparam_extensionr   )r   rm   r   )r   r±   s   "€r   r    r²   _  s-   ø€ ÷ ñ Ù—l‘lðÙ58ðá	�‰ñr!   c           	     óB  € V P                   ;'       g    V P                  P                  4       pV P                  P                  V4      ;_uu_ 4        \	        VVV P                  V P                   R 7      p\        V4       VuuRRR4       #   + '       g   i     R# ; i))r·   r¶   N)r¶   r·   Úcurrent_streamÚstreamr   r   )r»   r   rÄ   rÇ   Úresults   &&&  r   rº   Ú*DTensorExtensions.post_unflatten_transform_  s}   € ð ×$Ñ$×KÐK¨×(:Ñ(:×(IÑ(IÓ(KˆØ×Ñ×&Ñ& v×.Õ.ô 'ØØØ"×0Ñ0Ø#×2Ñ2ô	ˆFô   Ô'Ø÷ /×.×.Ó.ús   Á0BÂB	c                ó¢   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[RS[P                  RS[ P
                  R,          RS[ P                  /# )	r   r   rj   rw   rx   ry   rL   Nr   )r   rm   r8   rZ   rS   rL   )r   r±   s   "€r   r    r²   r  sq   ø€ ÷ 	Qñ 	Qá—‘ð	Qñ ð	Qñ ð		Qñ
 "ð	Qñ ×Ñð	Qñ —‘˜tÕ#ð	Qñ 
�‰ñ	Qr!   c                ó   € \        WW4V4      # r:   )rŒ   )r»   r   rj   rw   rx   ry   rL   s   &&&&&&&r   Úchunk_tensorÚDTensorExtensions.chunk_tensorr  s   € ô ˜V¨:ÈRÓPÐPr!   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# rŽ   )r   rm   r8   r   )r   r±   s   "€r   r    r²   }  s;   ø€ ÷ 9ñ 9á—‘ð9ñ ð9ñ  ð	9ñ
 
�‰ñ9r!   c                ó   € \        WV4      # r:   )r¢   )r»   r   rj   r&   s   &&&&r   Úchunk_dtensorÚDTensorExtensions.chunk_dtensor}  s   € ô ˜f¨KÓ8Ð8r!   c                ót   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[S[,          3,          /# r   r¤   )r   r±   s   "€r   r    r²   …  s5   ø€ ÷ ,ñ ,á—‘ð,ñ 
‰u�|‰|™T¡%�[Ð(Õ	)ñ,r!   c                ó   € \        V4      # r:   )r§   rÀ   s   &&r   Úpre_load_state_dict_transformÚ/DTensorExtensions.pre_load_state_dict_transform…  s   € ô $ FÓ+Ð+r!   c                óN   <€ V ^8„  d   QhRS[ RS[R,          RS[P                  /# r©   r«   )r   r±   s   "€r   r    r²   ‹  s2   ø€ ÷ 8ñ 8áð8ñ   $Õ&ð8ñ 
�‰ñ	8r!   c                ó   € \        W4      # r:   )r­   )r»   r   rª   s   &&&r   Úall_gather_dtensorÚ$DTensorExtensions.all_gather_dtensor‹  s   € ô
 # 6Ó7Ð7r!   )r¶   r·   rº   r:   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rµ   rÁ   rº   rÌ   rÐ   rÔ   rØ   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r¼   r±   s   @@r   r   r   G  sW   ù‡ € ñ÷
ó 
÷'ð '÷ð ÷&	Qò 	Q÷9ð 9÷,ð ,÷8÷ 8ð 8r!   )4rp   Útypingr   r   r   Útorch.distributedÚdistributedrZ   Ú&torch.distributed._shard.sharding_specÚ_shardÚsharding_specrk   Ú"torch.distributed.distributed_c10dÚdistributed_c10drR   Ú'torch.distributed._shard.sharded_tensorr   r   r   r   r	   Ú:torch.distributed._shard.sharding_spec.chunk_sharding_specr
   Ú$torch.distributed.fsdp._common_utilsr   Ú'torch.distributed.fsdp._fsdp_extensionsr   Ú#torch.distributed.fsdp._shard_utilsr   Útorch.distributed.remote_devicer   Útorch.distributed.tensorr   r   r   r-   Ú6torch.distributed.tensor.parallel._data_parallel_utilsr   r   Ú__all__r5   r<   rA   rO   rc   rg   ru   rŒ   r¢   r§   r­   r   r°   r!   r   Ú<module>ró      sš   ðã ß ã Ý  ß ;Ó ;ß 1Ð 1÷ó õ AÝ XÝ DÝ BÝ LÝ :ß TÓ T÷ð Ð
€õ>õ$>õ
*õ
õ%õPõõ>I
õX>
õB	7õô*I8˜ö I8r!   