+
    "Ý³jð  ã                   ó   € ^ RI t ^ RIHt ^ RIt^ RIHt ^ RIHu Ht ^ RI	H
t
HtHt ^ RIHt ^ RIHtHtHtHt ^ RIHt ^ RIHt ^ RIHtHtHt ^ R	IHt ^ R
IHt  ! R R]P@                  4      t! ! R R]P@                  4      t"RR R llt#R# )é    N)ÚUnion)Ú	ReplicateÚShardÚdistribute_tensor)Ú
DeviceMesh)ÚColwiseParallelÚPrepareModuleInputÚRowwiseParallelÚparallelize_module)ÚFloat8LinearConfig)Úconvert_to_float8_training)ÚFloat8ColwiseParallelÚFloat8RowwiseParallelÚPrepareFloat8ModuleInput)ÚMXFP8TrainingOpConfig)Ú	quantize_c                   ó<   a a€ ] tR t^ t oRtV 3R ltR tRtVtV ;t	# )ÚFeedForwardzMLP based modelc                ó  <€ \         \        V `  4        \        P                  ! W^,          RR7      V n        \        P                  ! W^,          RR7      V n        \        P                  ! V^,          VRR7      V n        R# )é   F)ÚbiasN)Úsuperr   Ú__init__ÚnnÚLinearÚw1Úw2Úout_proj©ÚselfÚsizeÚ	__class__s   &&€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torchao/testing/training/dtensor_utils.pyr   ÚFeedForward.__init__#   sT   ø€ ÜŒk˜4Ñ)Ô+Ü—)’)˜D¨¥(°Ô7ˆŒÜ—)’)˜D¨¥(°Ô7ˆŒÜŸ	š	 $¨¥(¨D°uÔ=ˆŽó    c                óž   € \         P                  ! V P                  V4      4      V P                  V4      ,          pV P	                  V4      pV# ©N)ÚFÚsilur   r   r   ©r    Úxs   &&r#   ÚforwardÚFeedForward.forward)   s8   € Ü�FŠF�4—7‘7˜1“:Ó §¡¨£Õ+ˆØ�M‰M˜!ÓˆØˆr%   )r   r   r   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r,   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r"   Ú__classdict__s   @@r#   r   r       s   ù‡ € Ùõ>÷ò r%   r   c                   ó8   a a€ ] tR t^/t oV 3R ltR tRtVtV ;t# )ÚToyModelc                óL   <€ \         \        V `  4        \        V4      V n        R # r'   )r   r9   r   r   Úffnr   s   &&€r#   r   ÚToyModel.__init__0   s   ø€ ÜŒh˜Ñ&Ô(Ü˜tÓ$ˆŽr%   c                ó$   € V P                  V4      # r'   ©r;   r*   s   &&r#   r,   ÚToyModel.forward4   s   € Ø�x‰x˜‹{Ðr%   r>   )	r.   r/   r0   r1   r   r,   r3   r4   r5   r6   s   @@r#   r9   r9   /   s   ù‡ € õ%÷ò r%   r9   c                ój   € V ^8„  d   QhR\         R\        \        \        3,          R\        R\        /# )r   ÚmeshÚconfigÚcompileÚallgather_in_lowp)r   r   r   r   Úbool)Úformats   "r#   Ú__annotate__rG   8   sB   € ÷ \ñ \Ü
ð\äÔ$Ô&;Ð;Õ<ð\ô ð	\ô
 ñ\r%   c                 ó  € V P                   p\        p\        V\        4      pV'       d   \        p\        V4      P                  V4      pV'       d    VP                  \        P                  4      p\        P                  ! V4      p	V! W‘R 7       V'       g   \        p
\        p\        pM\        p
\        p\         p\        P                  ! V4      pV'       g
   V! WÑR 7       \#        VV RV
! 4       RV
! 4       RV! 4       /4      pV'       d
   V! WÑR 7       \        P                  ! V4      pV'       g
   V! WáR 7       \#        VV RV! \%        ^4      \'        4       R7      RV
! 4       RV
! 4       RV! \%        ^4      RR7      /4      pV'       d
   V! WáR 7       \        P                  ! V4      pV'       g
   V! WñR 7       V'       g   V! \%        ^4      \'        4       R7      pMV! \%        ^4      \'        4       RR	7      p\#        VV RVRV
! 4       RV
! 4       RV! \%        ^4      RR7      /4      pV'       d
   V! WñR 7       V'       dC   \        P(                  ! V4      p\        P(                  ! V4      p\        P(                  ! V4      pV'       d   \        P                  M\        P*                  p\        P,                  ! ^V^,          W%RVR
7      p\        P,                  ! ^V^,          W%RVR
7      pVP/                  4       pVP/                  4       p\1        VP/                  4       V \%        ^ 4      .4      p\1        VP/                  4       V \%        ^ 4      .4      pV! V4      pVP3                  V4       V! V4      pVP3                  V4       V	! V4      pVP3                  V4       \        P4                  P7                  VV4       \        P4                  P7                  VP9                  4       V4       \        P4                  P7                  VP:                  P<                  P>                  P@                  VP:                  P<                  P>                  P@                  4       \        P4                  P7                  VP:                  PB                  P>                  P@                  VP:                  PB                  P>                  P@                  4       V! V4      pVP3                  V4       \        P4                  P7                  VP9                  4       V4       \        P4                  P7                  VP:                  P<                  P>                  P@                  VP:                  P<                  P>                  P@                  4       \        P4                  P7                  VP:                  PB                  P>                  P@                  VP:                  PB                  P>                  P@                  4       R# ))rB   zffn.w1zffn.w2zffn.out_projr;   )Úinput_layoutsÚdesired_input_layoutsF)Úoutput_layoutsÚuse_local_outputr   )rI   rJ   Úfwd_config_submodule_fqn)ÚdeviceÚrequires_gradÚdtypeN)"Údevice_typer   Ú
isinstancer   r   r9   ÚtoÚtorchÚbfloat16ÚcopyÚdeepcopyr   r
   r	   r   r   r   r   r   r   rC   Úfloat32ÚrandÚcloner   ÚbackwardÚtestingÚassert_closeÚfull_tensorr;   r   ÚweightÚgradr   )rA   rB   r!   rC   rD   rN   Úconvert_model_funcÚis_mxfp8Ú	toy_modelÚtoy_model_fp8Úcolwise_parallel_clsÚrowwise_parallel_clsÚprepare_input_clsÚtp_modelÚsp_modelÚ	sp_model2Úprepare_inputÚinput_dtyper+   ÚgoÚ
x_tp_inputÚgo_tpÚ
x_sp_inputÚgo_spÚtp_outÚsp_outÚ
global_outÚsp_out2s   &&&&&                       r#   Ú&_test_lowp_mlp_tensor_parallelism_baserv   8   s�  € ð ×Ñ€Fô 4ÐÜ˜&Ô"7Ó8€HßÜ&Ðä˜“×!Ñ! &Ó)€IßØ—L‘L¤§¡Ó0ˆ	ô —M’M )Ó,€MÙ�}Õ4÷ Ü.ÐÜ.ÐÜ.Ñä4ÐÜ4ÐÜ4Ðô �}Š}˜YÓ'€HßÙ˜8Õ3Ü!ØØàÑ*Ó,ØÑ*Ó,ØÑ0Ó2ð	
ó€H÷ Ù˜8Õ3ô �}Š}˜YÓ'€HßÙ˜8Õ3Ü!ØØàÑ$Ü# A›h¼i»kôð Ñ*Ó,ØÑ*Ó,ØÑ0Ü$ Q›x¸%ôð		
ó€H÷ Ù˜8Õ3ô —’˜iÓ(€IßÙ˜9Õ4çÙ)Ü ›(Ü"+£+ô
‰ñ
 *Ü ›(Ü"+£+Ø%)ô
ˆô #ØØà�=ØÑ*Ó,ØÑ*Ó,ØÑ0Ü$ Q›x¸%ôð		
ó€I÷ Ù˜9Õ4çÜ—=’= Ó*ˆÜ—=’= Ó*ˆÜ—M’M )Ó,ˆ	ç$,”%—.’.´%·-±-€KÜ�
Š
Ø	ˆ4�!�8�T¸À[ô	€Aô 
�ŠØ	ˆ4�!�8�T¸À[ô
€Bð —‘“€JØ�H‰H‹J€EÜ" 1§7¡7£9¨d´U¸1³X°JÓ?€JÜ˜bŸh™h›j¨$´°q³°
Ó;€Eá�jÓ!€FØ
‡O�O�EÔá�jÓ!€FØ
‡O�O�EÔá˜qÓ!€JØ×Ñ˜Ôä	‡M�M×Ñ˜v zÔ2Ü	‡M�M×Ñ˜v×1Ñ1Ó3°ZÔ@Ü	‡M�M×Ñ˜xŸ|™|Ÿ™×5Ñ5×:Ñ:¸H¿L¹L¿O¹O×<RÑ<R×<WÑ<WÔXÜ	‡M�M×ÑØ�‰×Ñ×$Ñ$×)Ñ)Ø�‰×Ñ×$Ñ$×)Ñ)ôñ
 ˜
Ó#€GØ×Ñ�UÔÜ	‡M�M×Ñ˜w×2Ñ2Ó4°jÔAÜ	‡M�M×ÑØ�‰�‰×Ñ×#Ñ# Y§]¡]×%5Ñ%5×%<Ñ%<×%AÑ%Aôô 
‡M�M×ÑØ�‰×Ñ×$Ñ$×)Ñ)Ø�‰×Ñ×%Ñ%×*Ñ*ör%   )é    FF)$rV   Útypingr   rT   Útorch.nnr   Útorch.nn.functionalÚ
functionalr(   Útorch.distributed._tensorr   r   r   Útorch.distributed.device_meshr   Ú!torch.distributed.tensor.parallelr   r	   r
   r   Útorchao.float8r   Ú"torchao.float8.float8_linear_utilsr   Ú%torchao.float8.float8_tensor_parallelr   r   r   Ú%torchao.prototype.moe_training.configr   Útorchao.quantizationr   ÚModuler   r9   rv   © r%   r#   Ú<module>r†      ss   ðó Ý ã Ý ß Ð ß IÑ IÝ 4÷ó õ .Ý I÷ñ õ
 HÝ *ô�"—)‘)ô ôˆr�y‰yô ÷\ñ \r%   