+
    "Ý³jàa  ã                   óh  € ^ RI t ^ RIt^ RIt^ RIHt ^ RIt^ RIt^ RIHt ^ RIH	t	H
t
Ht ^ RIHt ^ RIHt ^ RIHt ^ RIHtHtHtHt ]P.                  f   R	]n        ^ R
IHt ^ RIHt ^ RIHtHtHt ^ RI H!t! ^ RI"H#t# ^ RI$H%t% ^ RI&H't' ^ RI(H)t)H*t*  ^ RI+t, ^ RI.t.]P^                  P`                  e   ]Pb                  ! RRR7       ])! 4       t2 ! R R]4      t3 ! R R]4      t4^t5 ! R R]4      t6]! ]34       ]! ]44       ]7R8X  d
   ]! 4        R# R#   ]- d    Rt, Lˆi ; i  ]- d    Rt. L’i ; i)é    N)ÚPath)Únn)ÚCPUOffloadPolicyÚOffloadPolicyÚfully_shard)Úcommon_utils)Úskip_if_lt_x_gpu)ÚFSDPTest)ÚTestCaseÚinstantiate_parametrized_testsÚparametrizeÚ	run_testsiÒ  )ÚVersion)Úoptim)Ú_fp32_to_bf16_srÚquantize_4bit_with_qmapÚquantize_8bit_with_qmap)ÚOptimState4bit)ÚOptimState8bit)ÚOptimStateFp8)Úskip_if_rocm)Úget_available_devicesÚtorch_version_at_leastzSkipping the test in ROCmT)Úallow_module_levelc                   ó  a € ] tR t^@t o ]! R]4      R 4       t]! R]4      R 4       t]! R]4      R 4       t]! R]4      R 4       t	]! R]4      ]! RRR.4      R	 4       4       t
]! R]4      ]! RRR.4      R
 4       4       tRtV tR# )ÚTestQuantizeÚdevicec                ó�  € \         P                  ! ^ RVR7      p\         P                  ! RVR7      P                  4       P                  pVP	                  R4      V,
          P                  4       P                  R4      P                  \         P                  4      p\        W#4      p\         P                  P                  WE4       R# )é    é   ©r   é   Néÿÿÿÿ)ÚtorchÚrandÚsortÚvaluesÚ	unsqueezeÚabsÚargminÚtoÚuint8r   ÚtestingÚassert_close©Úselfr   ÚxÚqmapÚactualÚexpecteds   &&    ÚX/home/luca/screen-checking/.venv/lib/python3.14/site-packages/test/test_low_bit_optim.pyÚ(test_quantize_8bit_with_qmap_correctnessÚ5TestQuantize.test_quantize_8bit_with_qmap_correctnessA   s„   € ä�JŠJ�r˜4¨Ô/ˆÜ�zŠz˜# fÔ-×2Ñ2Ó4×;Ñ;ˆà—+‘+˜b“/ DÕ(×-Ñ-Ó/×6Ñ6°rÓ:×=Ñ=¼e¿k¹kÓJˆÜ*¨1Ó3ˆä�‰×"Ñ" 6Ö4ó    c                ó4  € \         P                  ! ^ RVR7      p\         P                  ! RVR7      P                  4       P                  p\         P                  ! \
        RR7      pV! W#4      p\        W#4      p\         P                  P                  WV4       R# )r   r    r!   r"   T©Ú	fullgraphN)r$   r%   r&   r'   Úcompiler   r-   r.   ©r0   r   r1   r2   Ú
compiled_fr3   r4   s   &&     r5   Ú$test_quantize_8bit_with_qmap_compileÚ1TestQuantize.test_quantize_8bit_with_qmap_compileK   sj   € ä�JŠJ�r˜4¨Ô/ˆÜ�zŠz˜# fÔ-×2Ñ2Ó4×;Ñ;ˆä—]’]Ô#:ÀdÔKˆ
Ù˜AÓ$ˆÜ*¨1Ó3ˆä�‰×"Ñ" 6Ö4r8   c                ó�  € \         P                  ! ^ RVR7      p\         P                  ! ^VR7      P                  4       P                  pVP	                  R4      V,
          P                  4       P                  R4      P                  \         P                  4      p\        W#4      p\         P                  P                  WE4       R# )r   r    r!   Nr#   )r$   r%   r&   r'   r(   r)   r*   r+   r,   r   r-   r.   r/   s   &&    r5   Ú(test_quantize_4bit_with_qmap_correctnessÚ5TestQuantize.test_quantize_4bit_with_qmap_correctnessV   s„   € ä�JŠJ�r˜4¨Ô/ˆÜ�zŠz˜" VÔ,×1Ñ1Ó3×:Ñ:ˆà—+‘+˜b“/ DÕ(×-Ñ-Ó/×6Ñ6°rÓ:×=Ñ=¼e¿k¹kÓJˆÜ*¨1Ó3ˆä�‰×"Ñ" 6Ö4r8   c                ó4  € \         P                  ! ^ RVR7      p\         P                  ! ^VR7      P                  4       P                  p\         P                  ! \
        RR7      pV! W#4      p\        W#4      p\         P                  P                  WV4       R# )r   r    r!   Tr:   N)r$   r%   r&   r'   r<   r   r-   r.   r=   s   &&     r5   Ú$test_quantize_4bit_with_qmap_compileÚ1TestQuantize.test_quantize_4bit_with_qmap_compile`   sj   € ä�JŠJ�r˜4¨Ô/ˆÜ�zŠz˜" VÔ,×1Ñ1Ó3×:Ñ:ˆä—]’]Ô#:ÀdÔKˆ
Ù˜AÓ$ˆÜ*¨1Ó3ˆä�‰×"Ñ" 6Ö4r8   r<   FTc                óž  € \         P                  ! ^ VR7      ^d,          pVP                  R	^4      P                  ^R4      p\         P                  ! \
        RRV'       * R7      pV! V4      pVP                  \         P                  J g   Q h\         P                  P                  VP                  4       P                  ^4      VRRR7       R# )
r   r!   é † TF©r;   ÚdynamicÚdisablegiUMuÿ>)ÚatolÚrtolNr#   )r$   r%   ÚviewÚrepeatr<   r   ÚdtypeÚbfloat16r-   r.   ÚfloatÚmean)r0   r   r<   r1   Úx_repÚfuncÚ
x_rep_bf16s   &&&    r5   Útest_bf16_stochastic_roundÚ'TestQuantize.test_bf16_stochastic_roundk   s£   € ô �JŠJ�r &Ô)¨CÕ/ˆØ—‘�r˜1“×$Ñ$ Q¨Ó0ˆä�}Š}Ü¨°eÈÄ[ô
ˆñ ˜%“[ˆ
Ø×Ñ¤5§>¡>Ó1Ð1Ð1ô 	�‰×"Ñ" :×#3Ñ#3Ó#5×#:Ñ#:¸1Ó#=¸qÀtÐRVÐ"ÖWr8   c                ó˜  € \         P                  ! R 4       ^ RIHp ^ RIHp ^ RIHpHp RpVP                  4       '       dS   VP                  4       '       g=   VP                  \        P                  ! 4       ^4      pVP                  RV^ ^R7       Rp \        P                   ! \"        P$                  4       \        P&                  ! ^ VR7      ^d,          p	V	P)                  R^4      P+                  ^R	4      p
\        P,                  ! \.        RRV'       * R
7      pV! V
4      pV! VR4      pVP1                  W­V! 4       .RR7      pV! V4      p\3        Wõ4      '       g   Q h\        P4                  P7                  VP9                  4       V4       V'       d   VP;                  4        R# R#   T'       d   TP;                  4        i i ; i)ztorch.distributedN)Úinit_device_mesh)ÚDTensorÚ	ReplicateFÚgloo)ÚbackendÚstoreÚrankÚ
world_sizeTr!   rH   rI   )Ú	run_checkr#   )é   )ÚpytestÚimportorskipÚtorch.distributedÚdistributedÚtorch.distributed.device_meshrZ   Útorch.distributed.tensorr[   r\   Úis_availableÚis_initializedÚ	FileStoreÚtempfileÚmktempÚinit_process_groupr$   Úmanual_seedr   ÚSEEDr%   rN   rO   r<   r   Ú
from_localÚ
isinstancer-   r.   Úto_localÚdestroy_process_group)r0   r   r<   ÚdistrZ   r[   r\   Ú
created_pgr_   r1   rT   rU   Ú	out_plainÚmeshÚx_dtÚout_dts   &&&             r5   Ú"test_bf16_stochastic_round_dtensorÚ/TestQuantize.test_bf16_stochastic_round_dtensorz   sn  € ô 	×ÒÐ/Ô0Ý(ÝBß?àˆ
Ø×Ñ×Ò t×':Ñ':×'<Ò'<Ø—N‘N¤8§?¢?Ó#4°aÓ8ˆEØ×#Ñ#ØØØØð	 $ô ð ˆJð	-Ü×Òœl×/Ñ/Ô0Ü—
’
˜2 fÔ-°Õ3ˆAØ—F‘F˜2˜q“M×(Ñ(¨¨GÓ4ˆEä—=’=Ü ¨D¸%ÈWÌôˆDñ ˜U›ˆIá# F¨DÓ1ˆDØ×%Ñ% e±I³K°=ÈEÐ%ÓRˆDÙ˜$“ZˆFä˜f×.Ò.Ð.Ð.Ü�M‰M×&Ñ& v§¡Ó'8¸)ÔDçØ×*Ñ*Ö,ñ ø�zØ×*Ñ*Õ,ð ús   ÂC<F. Æ.G	© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Ú_DEVICESr6   r?   rB   rE   rW   r|   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r5   r   r   @   sÍ   ø‡ € Ù�˜8Ó$ñ5ó %ð5ñ �˜8Ó$ñ5ó %ð5ñ �˜8Ó$ñ5ó %ð5ñ �˜8Ó$ñ5ó %ð5ñ �˜8Ó$Ù�˜U D˜MÓ*ñXó +ó %ðXñ �˜8Ó$Ù�˜U D˜MÓ*ñ#-ó +ó %ö#-r8   r   c            	       ó´  a € ] tR t^¢t o ]! R. R O4      ]! R]P                  ]P                  .4      ]! R]4      ]	! R4      R	 4       4       4       4       t
]! R. R!O4      ]! R]4      R
 4       4       t]! R. R!O4      ]! R]4      R 4       4       t]! R]]].4      ]! RR"R#.4      ]! R]4      R 4       4       4       t]! R]]].4      ]! R]4      R 4       4       t]P&                  P)                  ]RJ RR7      ]P&                  P)                  ]P,                  P/                  4       '       * RR7      ]	! R4      ]P&                  P)                  ]! R4      RR7      ]! RRR.4      R 4       4       4       4       4       t]P&                  P)                  ]RJ RR7      ]P&                  P)                  ]P,                  P/                  4       '       * RR7      ]! RRR.4      R 4       4       4       t]P&                  P)                  ]P,                  P/                  4       '       * ;'       d     ]P8                  P/                  4       '       * RR7      ]! R. R$O4      R 4       4       t]P&                  P)                  ]P,                  P/                  4       '       * ;'       d     ]P8                  P/                  4       '       * RR7      R 4       t]! R]4      R 4       tRt V t!R# )%Ú	TestOptimÚ
optim_nameÚAdam8bitÚ	AdamW8bitÚAdam4bitÚ	AdamW4bitrP   r   úROCm enablement in progressc                ó:  € VP                  R 4      '       dA   VR8X  d:   \        P                  P                  4       R8  d   \        P
                  ! R4       \        P                  ! \        P                  ! ^ R4      \        P                  ! 4       \        P                  ! R^ 4      4      pVP                  W2R7       \        \        V4      ! VP                  4       4      p\        P                  ! ^^ W2R7      pV! V4      P                  4       pVP!                  4        VP#                  4        VP%                  4        \&        P(                  ! 4       ;_uu_ 4       p\        P*                  ! VP-                  4       VP.                  4       \        P0                  ! VP.                  RR7      p	RRR4       \2        P4                  ! V4      p
\        \        V4      ! V
P                  4       4      pVP7                  X	4       \9        ^4       F¤  p\        P                  ! ^^ W2R7      pV! V4      P                  4       P!                  4        VP#                  4        VP%                  4        V
! V4      P                  4       P!                  4        VP#                  4        VP%                  4        K¦  	  \;        VP                  4       V
P                  4       4       F$  w  rÞ\        P<                  P?                  Wí4       K&  	  R#   + '       g   i     ELa; i)	ÚFp8Úcudaú+FP8 CUDA requires compute capability >= 8.9r"   ©r   rP   Úcpu©Úmap_locationN©é   é	   ) Úendswithr$   r’   Úget_device_capabilityrd   Úskipr   Ú
SequentialÚLinearÚReLUr+   Úgetattrr   Ú
parametersÚrandnÚsumÚbackwardÚstepÚ	zero_gradrm   ÚNamedTemporaryFileÚsaveÚ
state_dictÚnameÚloadÚcopyÚdeepcopyÚload_state_dictÚrangeÚzipr-   r.   )r0   rŠ   rP   r   ÚmodelÚ	optimizerr1   ÚlossÚfrª   Úmodel2Úoptim2Ú_Úp1Úp2s   &&&&           r5   Útest_optim_smokeÚTestOptim.test_optim_smoke£   s  € ð ×Ñ˜u×%Ò%¨&°FÔ*:Ü�z‰z×/Ñ/Ó1°FÔ:Ü—’ÐIÔJä—’œbŸiši¨¨CÓ0´"·'²'³)¼R¿YºYÀsÈBÓ=OÓPˆØ�‰˜ˆÔ,ÜœE :Ô.¨u×/?Ñ/?Ó/AÓBˆ	ä�KŠK˜˜2 fÔ:ˆÙ�Q‹x�|‰|‹~ˆØ�‰ŒØ�‰ÔØ×ÑÔô ×(Ò(×*Ô*¨aÜ�JŠJ�y×+Ñ+Ó-¨q¯v©vÔ6ÜŸš A§F¡F¸Ô?ˆJ÷ +ô —’˜uÓ%ˆÜœ 
Ô+¨F×,=Ñ,=Ó,?Ó@ˆØ×Ñ˜zÔ*ä�q–ˆAÜ—’˜A˜r¨&Ô>ˆAá�!‹H�L‰L‹N×#Ñ#Ô%Ø�N‰NÔØ×ÑÔ!á�1‹I�M‰M‹O×$Ñ$Ô&Ø�K‰KŒMØ×ÑÖñ ô ˜%×*Ñ*Ó,¨f×.?Ñ.?Ó.AÖB‰FˆBÜ�M‰M×&Ñ& rÖ.ó C÷' +×*Ð*ús   ÅAL	Ì	L	c                ó^  € VP                  R 4      '       dA   VR8X  d:   \        P                  P                  4       R8  d   \        P
                  ! R4       \        P                  ! 4       p\        P                  ! \        P                  4        \        P                  ! \        P                  ! ^ R4      \        P                  ! 4       \        P                  ! R^ 4      4      pVP                  VR7       \        \        V4      ! VP!                  4       4      p\        P"                  ! ^^ VR7      pV! V4      P%                  4       pVP'                  4        VP)                  4        VP+                  4        \        P                  ! V4       R#   \        P                  ! T4       i ; i)r‘   r’   r“   r"   r!   Nr˜   )r›   r$   r’   rœ   rd   r�   Úget_default_dtypeÚset_default_dtyperQ   r   rž   rŸ   r    r+   r¡   r   r¢   r£   r¤   r¥   r¦   r§   )r0   rŠ   r   Ú	old_dtyper²   r³   r1   r´   s   &&&     r5   Útest_optim_default_dtype_bf16Ú'TestOptim.test_optim_default_dtype_bf16Ð   s  € ð ×Ñ˜u×%Ò%¨&°FÔ*:Ü�z‰z×/Ñ/Ó1°FÔ:Ü—’ÐIÔJä×+Ò+Ó-ˆ	Ü×Ò¤§¡Ô/ð	/Ü—M’M¤"§)¢)¨B°Ó"4´b·g²g³iÄÇÂÈ3ÐPRÓASÓTˆEØ�H‰H˜FˆHÔ#Ü¤ zÔ2°5×3CÑ3CÓ3EÓFˆIä—’˜A˜r¨&Ô1ˆAÙ˜“8—<‘<“>ˆDØ�M‰MŒOØ�N‰NÔØ×ÑÔ!ô ×#Ò# IÖ.øŒE×#Ò# IÕ.ús   ÂC*F ÆF,c                ó  € VP                  R 4      '       dA   VR8X  d:   \        P                  P                  4       R	8  d   \        P
                  ! R4       \        P                  ! \        P                  ! ^ R4      \        P                  ! 4       \        P                  ! R^ 4      4      pVP                  VR7       \        \        V^ ,          P                  4       4      RR7      \        \        V^,          P                  4       4      RR7      .p\        \        V4      ! V4      p\        P                   ! ^^ VR7      pV! V4      P#                  4       pVP%                  4        VP'                  4        VP)                  4        R# )
r‘   r’   r“   r"   r!   g-Cëâ6?)ÚparamsÚlrçñhãˆµøä>Nr˜   )r›   r$   r’   rœ   rd   r�   r   rž   rŸ   r    r+   ÚdictÚlistr¢   r¡   r   r£   r¤   r¥   r¦   r§   )r0   rŠ   r   r²   Úparam_groupsr³   r1   r´   s   &&&     r5   Útest_param_groupsÚTestOptim.test_param_groupsè   s  € ð ×Ñ˜u×%Ò%¨&°FÔ*:Ü�z‰z×/Ñ/Ó1°FÔ:Ü—’ÐIÔJä—’œbŸiši¨¨CÓ0´"·'²'³)¼R¿YºYÀsÈBÓ=OÓPˆØ�‰˜ˆÔäœ˜U 1�X×0Ñ0Ó2Ó3¸Ô=Üœ˜U 1�X×0Ñ0Ó2Ó3¸Ô=ð
ˆô œE :Ô.¨|Ó<ˆ	ä�KŠK˜˜2 fÔ-ˆÙ�Q‹x�|‰|‹~ˆØ�‰ŒØ�‰ÔØ×ÑÖr8   ÚsubclassÚshapec                ó   € V\         8X  dA   VR 8X  d:   \        P                  P                  4       R8  d   \        P
                  ! R4       VP                  W#R7      pV^ ,          ^,          p\        P                  P                  VP                  4       RV VRV P                  4       4       \        P                  P                  VP                  4       WU^,           WEV^,           P                  4       4       R# )r’   r“   r!   Nr˜   )
r   r$   r’   rœ   rd   r�   Úzerosr-   r.   Ú
dequantize)r0   rÌ   rÍ   r   ÚtensorÚoffsets   &&&&  r5   Útest_subclass_sliceÚTestOptim.test_subclass_slice  sÆ   € ð ”}Ô$Ø˜Ô¤E§J¡J×$DÑ$DÓ$FÈÔ$OÜ—’ÐIÔJà—‘ �Ó5ˆØ�q•˜Q•ˆä�‰×"Ñ"Ø×ÑÓ  Ð(¨&°°&¨/×*DÑ*DÓ*Fô	
ô 	�‰×"Ñ"Ø×ÑÓ °!­Ð4Ø˜F Q�JÐ'×2Ñ2Ó4ö	
r8   c                óÔ  € RpVP                  W2R7      pV P                  VP                  \        P                  4       VP                  \        P                  4      pV P                  VP                  \        P                  4       VP                  '       g;   VP                  4       pV P                  VP                  \        P                  4       VP                  \        P                  4      pV P                  VP                  \        P                  4       VP                  W2\        P                  R7      pV P                  VP                  \        P                  4       R# )r    r!   r”   N)r    )	rÏ   ÚassertEqualrP   r$   Úfloat32r+   rQ   Úis_cpur•   )r0   rÌ   r   rÍ   rÑ   Útensor_bf16Útensor_bf16_cpuÚtensor_fp32_recasts   &&&     r5   Útest_subclass_appearance_dtypeÚ(TestOptim.test_subclass_appearance_dtype  s÷   € ð ˆð —‘ �Ó5ˆØ×Ñ˜Ÿ™¤u§}¡}Ô5ð —i‘i¤§¡Ó/ˆØ×Ñ˜×*Ñ*¬E¯N©NÔ;ð ×!×!Ð!Ø)Ÿo™oÓ/ˆOØ×Ñ˜_×2Ñ2´E·N±NÔCð )Ÿ^™^¬E¯M©MÓ:ÐØ×ÑÐ+×1Ñ1´5·=±=ÔAð —n‘n UÄÇÁ�nÓPˆØ×Ñ˜×*Ñ*¬E¯N©NÖ;r8   Nzbitsandbytes is not available©Úreasonz+bitsandbytes 8-bit Adam only works for CUDAz2.7.0úFailing in CIc                ó  € R p\         P                  ! \         P                  ! ^ R4      \         P                  ! 4       \         P                  ! R^€4      4      pVP	                  V4       \
        P                  ! V4      p\        \        P                  4      \        R4      8¼  d   RMRp\        \        P                  V4      ! VP                  4       4      p\        \        V4      ! VP                  4       VR7      p\        ^4       F¨  p\        P                  ! ^^ VR7      p	V! V	4      P!                  4       p
V
P#                  4        VP%                  4        VP'                  4        V! V	4      P!                  4       pVP#                  4        VP%                  4        VP'                  4        Kª  	  \)        VP                  4       VP                  4       4       F'  w  rÍ\        P*                  P-                  WÜRRR7       K)  	  R	# )
r’   r    z0.44.0r"   i   )Ú
block_sizer!   rÆ   ©rM   rL   N)r   rž   rŸ   r    r+   r­   r®   r   ÚbnbÚ__version__r¡   r   r¢   r°   r$   r£   r¤   r¥   r¦   r§   r±   r-   r.   )r0   rŠ   r   Úmodel1r¶   râ   Úoptim1r·   r¸   r1   Úloss1Úloss2r¹   rº   s   &&            r5   Útest_optim_8bit_correctnessÚ%TestOptim.test_optim_8bit_correctness/  sf  € ð ˆÜ—’œrŸyšy¨¨TÓ2´B·G²G³I¼r¿yºyÈÈsÓ?SÓTˆØ�	‰	�&ÔÜ—’˜vÓ&ˆô $¤C§O¡OÓ4¼ÀÓ8IÔI‘SÈtˆ
äœŸ™ JÔ/°×0AÑ0AÓ0CÓDˆÜœ 
Ô+¨F×,=Ñ,=Ó,?ÈJÔWˆä�q–ˆAÜ—’˜A˜r¨&Ô1ˆAá˜1“I—M‘M“OˆEØ�N‰NÔØ�K‰KŒMØ×ÑÔá˜1“I—M‘M“OˆEØ�N‰NÔØ�K‰KŒMØ×ÑÖñ ô ˜&×+Ñ+Ó-¨v×/@Ñ/@Ó/BÖC‰FˆBÜ�M‰M×&Ñ& r°D¸tÐ&ÖDó Dr8   zlpmm is not availablez#lpmm 4-bit Adam only works for CUDAc                óT  € R p\         P                  ! \         P                  ! ^ R4      \         P                  ! 4       \         P                  ! R^€4      4      pVP	                  V4       \
        P                  ! V4      pVR8X  d1   \        P                  P                  VP                  4       ^ R7      pMDVR8X  d/   \        P                  P                  VP                  4       4      pM\        RV R24      h\        \        V4      ! VP                  4       4      p\        ^4       F¨  p\        P                  ! ^^ VR7      pV! V4      P!                  4       p	V	P#                  4        VP%                  4        VP'                  4        V! V4      P!                  4       p
V
P#                  4        VP%                  4        VP'                  4        Kª  	  \)        VP                  4       VP                  4       4       F'  w  r¼\        P*                  P-                  WËRRR	7       K)  	  R
# )r’   r    r�   )Úweight_decayrŽ   zUnsupported z optimizer for lpmmr!   rÆ   rã   N)r   rž   rŸ   r    r+   r­   r®   Úlpmmr   ÚAdamWr¢   Ú
ValueErrorr¡   r°   r$   r£   r¤   r¥   r¦   r§   r±   r-   r.   )r0   rŠ   r   ræ   r¶   rç   r·   r¸   r1   rè   ré   r¹   rº   s   &&           r5   Útest_optim_4bit_correctnessÚ%TestOptim.test_optim_4bit_correctnessV  sŽ  € ð ˆÜ—’œrŸyšy¨¨TÓ2´B·G²G³I¼r¿yºyÈÈsÓ?SÓTˆØ�	‰	�&ÔÜ—’˜vÓ&ˆð ˜Ô#Ü—Z‘Z×%Ñ% f×&7Ñ&7Ó&9ÈÐ%ÓJ‰FØ˜;Ô&Ü—Z‘Z×%Ñ% f×&7Ñ&7Ó&9Ó:‰Fä˜|¨J¨<Ð7JÐKÓLÐLÜœ 
Ô+¨F×,=Ñ,=Ó,?Ó@ˆä�q–ˆAÜ—’˜A˜r¨&Ô1ˆAá˜1“I—M‘M“OˆEØ�N‰NÔØ�K‰KŒMØ×ÑÔá˜1“I—M‘M“OˆEØ�N‰NÔØ�K‰KŒMØ×ÑÖñ ô ˜&×+Ñ+Ó-¨v×/@Ñ/@Ó/BÖC‰FˆBÜ�M‰M×&Ñ& r°D¸tÐ&ÖDó Dr8   z&optim CPU offload requires CUDA or XPUzoffload_grad,grad_accumc                óÒ  € \         R	,          p\        P                  ! \        P                  ! ^ R4      \        P                  ! 4       \        P                  ! R^@RR7      \        P                  ! 4       \        P                  ! ^@^@RR7      \        P                  ! 4       \        P                  ! ^@^€RR7      4      pVP                  V4       V^,          P                  R4       \        P                  ! V4      p\        P                  P                  VP                  4       4      p\        P                  ! VP                  4       \        P                  P                  VR7      p\        P                  P                  P                  V^d4      p\        P                  P                  P                  V^d4      p	\        P                   ! VR7      p
V
P#                  ^*4       \%        ^4       F‚  p\%        V4       F@  p\        P&                  ! ^^ W:R7      pV! V4      P)                  4       P+                  4        KB  	  VP-                  4        VP/                  4        VP-                  4        K„  	  V
P#                  ^*4       \%        ^4       F‚  p\%        V4       F@  p\        P&                  ! ^^ W:R7      pV! V4      P)                  4       P+                  4        KB  	  VP-                  4        VP/                  4        V	P-                  4        K„  	  \1        VP                  4       VP                  4       4       F$  w  rÞ\        P2                  P5                  Wí4       K&  	  R# )
rc   i   T©ÚbiasF)Úoffload_gradientsr!   )r   Ú	generatorNr#   )rƒ   r   rž   rŸ   r    r+   Úrequires_grad_r­   r®   r$   r   rï   r¢   ÚCPUOffloadOptimizerÚlr_schedulerÚCosineAnnealingLRÚ	Generatorrp   r°   r£   r¤   r¥   r¦   r§   r±   r-   r.   )r0   Úoffload_gradÚ
grad_accumr   ræ   r¶   rç   r·   Ú
scheduler1Ú
scheduler2Úrngr¸   r1   r¹   rº   s   &&&            r5   Ú"test_optim_cpu_offload_correctnessÚ,TestOptim.test_optim_cpu_offload_correctnessz  sY  € ô ˜"•ˆô —’Ü�IŠI�b˜&Ó!Ü�GŠG‹IÜ�IŠI�f˜b tÔ,Ü�GŠG‹IÜ�IŠI�b˜" 4Ô(Ü�GŠG‹IÜ�IŠI�b˜# DÔ)ó
ˆð 	�	‰	�&Ôð 	ˆq�	× Ñ  Ô'Ü—’˜vÓ&ˆä—‘×"Ñ" 6×#4Ñ#4Ó#6Ó7ˆÜ×*Ò*Ø×ÑÓÜ�K‰K×ÑØ*ô
ˆô —[‘[×-Ñ-×?Ñ?ÀÈÓLˆ
Ü—[‘[×-Ñ-×?Ñ?ÀÈÓLˆ
ä�oŠo VÔ,ˆØ�‰˜Ôô �q–ˆAÜ˜:Ö&�Ü—K’K  2¨fÔD�Ù�q“	—‘“×(Ñ(Ö*ñ 'ð �K‰KŒMØ×ÑÔØ�O‰OÖñ ð 	�‰˜ÔÜ�q–ˆAÜ˜:Ö&�Ü—K’K  2¨fÔD�Ù�q“	—‘“×(Ñ(Ö*ñ 'ð �K‰KŒMØ×ÑÔØ�O‰OÖñ ô ˜&×+Ñ+Ó-¨v×/@Ñ/@Ó/BÖC‰FˆBÜ�M‰M×&Ñ& rÖ.ó Dr8   c           
     ó0  € \         R,          p\        P                  ! \        P                  ! ^ RRR7      \        P                  ! 4       \        P                  ! R^€RR7      4      pVP                  V4       \        P                  ! VP                  4       \        P                  P                  4      p\        ^4       F`  p\        P                  ! ^^ VR7      pV! V4      P                  4       P                  4        VP                  4        VP!                  4        Kb  	  \"        P$                  ! 4       ;_uu_ 4       p\        P&                  ! VP)                  4       VP*                  4       \        P,                  ! VP*                  RR7      pRRR4       \.        P0                  ! V4      p\        P                  ! VP                  4       \        P                  P                  4      p	V	P3                  X4       \        ^4       F¤  p\        P                  ! ^^ VR7      pV! V4      P                  4       P                  4        VP                  4        VP!                  4        V! V4      P                  4       P                  4        V	P                  4        V	P!                  4        K¦  	  \5        VP                  4       VP                  4       4       F$  w  r«\        P6                  P9                  Wº4       K&  	  R#   + '       g   i     ELz; i)	rc   r    Trô   r!   r•   r–   Nr#   )rƒ   r   rž   rŸ   r    r+   r   rù   r¢   r$   rï   r°   r£   r¤   r¥   r¦   r§   rm   r¨   r©   rª   r«   r¬   r­   r®   r¯   r±   r-   r.   )r0   r   ræ   rç   r¸   r1   Úfilerª   r¶   r·   r¹   rº   s   &           r5   Ú test_optim_cpu_offload_save_loadÚ*TestOptim.test_optim_cpu_offload_save_load»  s	  € ô
 ˜"•ˆô —’Ü�IŠI�b˜$ TÔ*¬B¯GªG«I´r·y²yÀÀsÐQUÔ7Vó
ˆð 	�	‰	�&ÔÜ×*Ò*¨6×+<Ñ+<Ó+>ÄÇÁ×@QÑ@QÓRˆä�q–ˆAÜ—’˜A˜r¨&Ô1ˆAÙ�1‹I�M‰M‹O×$Ñ$Ô&Ø�K‰KŒMØ×ÑÖñ	 ô ×(Ò(×*Ô*¨dÜ�JŠJ�v×(Ñ(Ó*¨D¯I©IÔ6ÜŸš D§I¡I¸EÔBˆJ÷ +ô
 —’˜vÓ&ˆÜ×*Ò*¨6×+<Ñ+<Ó+>ÄÇÁ×@QÑ@QÓRˆØ×Ñ˜zÔ*ä�q–ˆAÜ—’˜A˜r¨&Ô1ˆAá�1‹I�M‰M‹O×$Ñ$Ô&Ø�K‰KŒMØ×ÑÔá�1‹I�M‰M‹O×$Ñ$Ô&Ø�K‰KŒMØ×ÑÖñ ô ˜&×+Ñ+Ó-¨v×/@Ñ/@Ó/BÖC‰FˆBÜ�M‰M×&Ñ& rÖ.ó D÷) +×*Ð*ús   ÅALÌL	c                ó`  a	€ \         P                  ! R 4       \        P                  ! \        P                  ! ^ R4      \        P
                  ! 4       \        P                  ! R^€4      4      pVP                  V4       \        P                  ! V4      P                  4       p\         P                  P                  VP                  4       RR7      p\        P                  ! VP                  4       RRR7      p\         P                  ! ^^ VR7      p\        ^4       Fû  o	\         P                   ! V\         P                  R7      ;_uu_ 4        V! V4      pRRR4       XP#                  4       pVP%                  4        VP'                  4        VP)                  4        V! VP                  4       4      P#                  4       pVP%                  4        VP'                  4        VP)                  4        \         P*                  P-                  WxV	3R	 lR
7       Ký  	  R#   + '       g   i     LÏ; i)iè  r    rÆ   ©rÅ   T)rÅ   Úbf16_stochastic_roundr!   )rP   Nc                 ó   <€ R S RV  2# )z
Iteration z. r~   )ÚmsgÚidxs   &€r5   Ú<lambda>ÚHTestOptim.test_optim_bf16_stochastic_round_correctness.<locals>.<lambda>
  s   ø€ °
¸3¸%¸rÀ#ÀÑ.Gr8   )r  )r$   rp   r   rž   rŸ   r    r+   r­   r®   rQ   r   rï   r¢   Ú_AdamWr£   r°   Úautocastr¤   r¥   r¦   r§   r-   r.   )
r0   r   ræ   r¶   rç   r·   r1   rè   ré   r  s
   &&       @r5   Ú,test_optim_bf16_stochastic_round_correctnessÚ6TestOptim.test_optim_bf16_stochastic_round_correctnessç  ss  ø€ ä×Ò˜$ÔÜ—’œrŸyšy¨¨TÓ2´B·G²G³I¼r¿yºyÈÈsÓ?SÓTˆØ�	‰	�&ÔÜ—’˜vÓ&×/Ñ/Ó1ˆô —‘×"Ñ" 6×#4Ñ#4Ó#6¸4Ð"Ó@ˆÜ—’Ø×ÑÓØØ"&ô
ˆô �KŠK˜˜2 fÔ-ˆä˜–8ˆCä—’ ¬e¯n©n×=Ö=Ù˜q›	�÷ >à—I‘I“KˆEØ�N‰NÔØ�K‰KŒMØ×ÑÔñ ˜1Ÿ:™:›<Ó(×,Ñ,Ó.ˆEØ�N‰NÔØ�K‰KŒMØ×ÑÔä�M‰M×&Ñ&ØÔ"Gð 'ö ó ç=×=ús   Å	HÈH-r~   )r‹   rŒ   r�   rŽ   ÚAdamFp8ÚAdamWFp8)r‹   r�   r  )i   )r"   r"   ))Frc   )Fé   )Trc   )"r   r€   r�   r‚   r   r$   r×   rQ   rƒ   r   r»   rÁ   rÊ   r   r   r   rÓ   rÜ   rd   ÚmarkÚskipifrä   r’   rj   r   rê   rî   rñ   Úxpur  r  r  r„   r…   r†   s   @r5   r‰   r‰   ¢   s1  ø‡ € ÙØÚQóñ �˜5Ÿ=™=¨%¯.©.Ð9Ó:Ù�˜8Ó$ÙÐ/Ó0ñ$/ó 1ó %ó ;ó	ð$/ñL �ÒBÓCÙ�˜8Ó$ñ/ó %ó Dð/ñ, �ÒBÓCÙ�˜8Ó$ñó %ó Dðñ0 �˜n¨n¸mÐLÓMÙ�˜7 JÐ/Ó0Ù�˜8Ó$ñ
ó %ó 1ó Nð
ñ  �˜n¨n¸mÐLÓMÙ�˜8Ó$ñ<ó %ó Nð<ð0 ‡[�[×Ñ˜˜t˜Ð,KÐÓLØ‡[�[×ÑØ�J‰J×#Ñ#Ó%Ô%Ø<ð ó ñ Ð/Ó0Ø‡[�[×ÑÙ˜wÓ'°ð ó ñ � 
¨KÐ8Ó9ñEó :óó 1ó	ó MðEð: ‡[�[×Ñ˜ ˜Ð-DÐÓEØ‡[�[×ÑØ�J‰J×#Ñ#Ó%Ô%Ð.Sð ó ñ � 
¨KÐ8Ó9ñEó :óó Fð
Eð> ‡[�[×ÑØ�J‰J×#Ñ#Ó%Ô%×FÐF¨e¯i©i×.DÑ.DÓ.FÔ*FØ7ð ó ñ Ð*Ò,OÓPñ:/ó Qó	ð
:/ðx ‡[�[×ÑØ�J‰J×#Ñ#Ó%Ô%×FÐF¨e¯i©i×.DÑ.DÓ.FÔ*FØ7ð ó ñ&/ó	ð&/ñP �˜8Ó$ñ#ó %ö#r8   r‰   c                   ó>  a € ] tR tRt o ]V 3R lR l4       t]! ]4      ]! R4      ]	P                  P                  ]! R4      RR7      R 4       4       4       tR	 t]! ]4      ]! R4      ]	P                  P                  ]! R4      RR7      R
 4       4       4       tRtV tR# )Ú	TestFSDP2i  c                ó    <€ V ^8„  d   QhRS[ /# )r  Úreturn)Úint)Úformatr‡   s   "€r5   Ú__annotate__ÚTestFSDP2.__annotate__  s   ø€ ÷  ñ  ™Cñ  r8   c                ó   € \         # )N)Ú_FSDP_WORLD_SIZE)r0   s   &r5   ra   ÚTestFSDP2.world_size  s   € äÐr8   r�   z2.11.0rà   rÞ   c                óT  € \         P                  \        3\         P                  \        3\         P                  \        3.p\
        P                  P                  4       R8¼  d&   VP                  \         P                  \        34       V P                  RV/V P                  4       R# )r™   ÚargsNr˜   )r   rŒ   r   rŽ   r   r$   r’   rœ   Úappendr  Úrun_subtestsÚ_test_fsdp2)r0   Ú	args_lists   & r5   Ú
test_fsdp2ÚTestFSDP2.test_fsdp2  s}   € ô �_‰_œmÐ,Ü�_‰_œmÐ,Ü�_‰_Ô.Ð/ð
ˆ	ô
 �:‰:×+Ñ+Ó-°Ô7Ø×ÑœeŸn™n¬mÐ<Ô=à×ÑØ�YÐØ×Ñö	
r8   c           	     ó0  € ^ RI Hp ^ RIHu Hp ^ RIHu Hp ^ RIHp ^ RI	H
pHpHp Vw  rš^pRp^@pV! ^^RVV^ R7      p\        P                  ! ^*4       \        P                  ! R4      ;_uu_ 4        V! V4      pRRR4       V	! XP!                  4       RR	7      p\"        P$                  ! V4      pVP'                  4        F$  p\)        VV4      '       g   K  \+        VV
R
7       K&  	  \+        VV
R
7       V	! VP!                  4       RR	7      p\        P                  ! ^*V P,                  ,           ^,           4       \/        ^4       EF-  p\        P0                  ! ^ WËV3RR7      pVP3                  V^,          ^ 8H  R7       V! V4      P5                  4       pVP7                  4        VP9                  4        VP3                  V^,          ^ 8H  R7       V! V4      P5                  4       pVP7                  4        VP!                  4        FD  pVP:                  f   K  VP=                  VP:                  VP>                  P@                  R7       KF  	  VP9                  4        V PC                  VV4       EK0  	  VPD                  ^ ,          R,          ^ ,          pVPF                  V,          R,          pVPD                  ^ ,          R,          ^ ,          pVPF                  V,          R,          pVPI                  4       pV PC                  VPK                  4       VPK                  4       4       RV	PL                   2p\O        V4      PQ                  4       '       d   \R        PT                  ! V4       VPW                  VPY                  4       VR7       V	! VP!                  4       RR	7      pVP!                  4        F  p \        PZ                  ! V 4      V n        K   	  VP9                  4        VP]                  VPY                  4       VR7       VP_                  4       ^ 8X  d   \R        PT                  ! V4       \`        \b        \d        3p!\g        VPi                  VPY                  4       4      VPi                  VPY                  4       4      4       Fì  w  p"p#V"Pj                  V#Pj                  8X  g   Q V"Pj                  V#Pj                  34       h\)        V"V4      '       dY   V"Pm                  4       p"V#Pm                  4       p#V"Pj                  V#Pj                  8X  g   Q V"Pj                  V#Pj                  34       h\)        V"V!4      '       d!   V"PK                  4       p"V#PK                  4       p#V PC                  V"V#4       Kî  	  R#   + '       g   i     EL; i)r   N)r[   )Ú	ModelArgsÚTransformerÚTransformerBlockr    é   )Ún_layersÚn_headsÚdimÚ
vocab_sizeÚmax_seq_lenÚ	dropout_pr’   g{®Gáz„?r	  )Úoffload_policyr!   )Úset_to_none)ÚoprÄ   Úexp_avgÚ_fsdp_low_bit_optim_)Úcheckpoint_id)7rf   rg   Útorch.distributed.checkpointÚ
checkpointÚtorch.utils._pytreeÚutilsÚ_pytreeri   r[   Ú:torch.testing._internal.distributed._tensor.common_dtensorr.  r/  r0  r$   rp   r   r¢   r­   r®   Úmodulesrs   r   r`   r°   Úrandintr§   rS   r¥   r¦   ÚgradÚ
all_reduceÚReduceOpÚAVGrÖ   rÉ   ÚstateÚfull_tensorrÐ   r   r   ÚexistsÚshutilÚrmtreer©   rª   Ú
zeros_liker¬   Úget_rankr   r   r   r±   Ú	tree_iterÚ	__class__rt   )$r0   r&  rv   ÚdcpÚpytreer[   r.  r/  r0  Ú	optim_clsr8  Ú
batch_sizer5  Úseq_lenÚ
model_argsÚ
base_modelÚ
base_optimÚ
fsdp_modelÚmÚ
fsdp_optimÚiter_idxÚinpÚ	fsdp_lossÚ	base_lossÚparamÚ
base_paramÚbase_exp_avgÚ
fsdp_paramÚfsdp_exp_avgÚfull_fsdp_exp_avgr=  Úresumed_fsdp_optimÚpÚ
subclassesÚv1Úv2s$   &&                                  r5   r)  ÚTestFSDP2._test_fsdp2(  sc  € Ý(ß2Ð2ß,Ð,Ý4÷	
ñ 	
ð %)Ñ!ˆ	àˆ
Øˆ
ØˆÙØØØØ!ØØô
ˆ
ô 	×Ò˜"ÔÜ�\Š\˜&×!Õ!Ù$ ZÓ0ˆJ÷ "á˜z×4Ñ4Ó6¸4Ô@ˆ
ä—]’] :Ó.ˆ
Ø×#Ñ#Ö%ˆAÜ˜!Ð-×.Ô.Ü˜A¨n×=ñ &ô 	�J¨~Õ>Ù˜z×4Ñ4Ó6¸4Ô@ˆ
ä×Ò˜"˜tŸy™y�.¨1Õ,Ô-Ü˜aŸˆHÜ—-’-  :¸GÐ/DÈVÔTˆCØ× Ñ ¨h¸­l¸aÑ.?Ð ÔAÙ" 3›×,Ñ,Ó.ˆIØ×ÑÔ Ø�O‰OÔà× Ñ ¨h¸­l¸aÑ.?Ð ÔAÙ" 3›×,Ñ,Ó.ˆIØ×ÑÔ Ø#×.Ñ.Ö0�Ø—:‘:Ô)Ø—O‘O E§J¡J°4·=±=×3DÑ3D�OÖEñ 1ð �O‰OÔØ×Ñ˜Y¨	×2ñ !ð   ×,Ñ,¨QÕ/°Õ9¸!Õ<ˆ
Ø!×'Ñ'¨
Õ3°IÕ>ˆà×,Ñ,¨QÕ/°Õ9¸!Õ<ˆ
Ø!×'Ñ'¨
Õ3°IÕ>ˆØ(×4Ñ4Ó6Ðà×Ñ˜×0Ñ0Ó2Ð4E×4PÑ4PÓ4RÔSð /¨y×/AÑ/AÐ.BÐCˆÜ�Ó×%Ñ%×'Ò'Ü�MŠM˜-Ô(Ø�‰�×&Ñ&Ó(¸ˆÔFñ
 ' z×'<Ñ'<Ó'>À4ÔHÐØ×&Ñ&Ö(ˆAÜ×%Ò% aÓ(ˆAŽFñ )ð 	×ÑÔ!à�‰Ð#×.Ñ.Ó0ÀˆÔNØ�=‰=‹?˜aÔÜ�MŠM˜-Ô(ä$¤n´mÐDˆ
äØ×ÑÐ/×:Ñ:Ó<Ó=Ø×Ñ˜Z×2Ñ2Ó4Ó5ö
‰FˆB�ð —<‘< 2§<¡<Ô/ÐM°"·,±,ÀÇÁÐ1MÓMÐ/Ü˜"˜g×&Ò&Ø—[‘[“]�Ø—[‘[“]�Ø—|‘| r§|¡|Ô3ÐQ°b·l±lÀBÇLÁLÐ5QÓQÐ3Ü˜"˜j×)Ò)Ø—]‘]“_�Ø—]‘]“_�Ø×Ñ˜R Ö$ó
÷w "×!Ð!ús   Á5	VÖV	c                óü  € R p\         ^,          ^,           p\        P                  ! WRR7      pVP                  P                  ^ ,          \         ,          ^ 8w  g   Q h\        V4       \        P                  ! VP                  4       4      p\        ^4       F`  p\        P                  ! ^VRR7      pV! V4      P                  4       P                  4        VP                  4        VP                  4        Kb  	  R# )r1  r’   r!   N)r#  r   rŸ   ÚweightrÍ   r   r   rŒ   r¢   r°   r$   r£   r¤   r¥   r¦   r§   )r0   Úin_dimÚout_dimr²   r³   r¸   Úinputss   &      r5   Útest_uneven_shardÚTestFSDP2.test_uneven_shardŠ  s½   € ð ˆÜ" RÕ'¨!Õ+ˆô —	’	˜&°&Ô9ˆØ�|‰|×!Ñ! !Õ$Ô'7Õ7¸1Ô<Ð<Ð<Ü�EÔô —O’O E×$4Ñ$4Ó$6Ó7ˆ	ä�q–ˆAÜ—[’[  F°6Ô:ˆFÙ�&‹M×ÑÓ×(Ñ(Ô*Ø�N‰NÔØ×ÑÖ!ó	 r8   r~   N)r   r€   r�   r‚   Úpropertyra   r	   r#  r   rd   r  r  r   r+  r)  rs  r„   r…   r†   s   @r5   r  r    s´   ø‡ € Ø÷ ó ð ñ Ð&Ó'ÙÐ/Ó0Ø‡[�[×ÑÑ.¨xÓ8ÀÐÓQñ
ó Ró 1ó (ð
ò`%ñD Ð&Ó'ÙÐ/Ó0Ø‡[�[×ÑÑ.¨xÓ8ÀÐÓQñ"ó Ró 1ó (ö"r8   r  Ú__main__)8r­   rM  rm   Úpathlibr   rd   r$   r   Ú"torch.distributed._composable.fsdpr   r   r   Útorch.testing._internalr   Ú*torch.testing._internal.common_distributedr	   Ú#torch.testing._internal.common_fsdpr
   Ú$torch.testing._internal.common_utilsr   r   r   r   rq   Úpackaging.versionr   Útorchaor   Útorchao.optim.quant_utilsr   r   r   Útorchao.optim.subclass_4bitr   Útorchao.optim.subclass_8bitr   Útorchao.optim.subclass_fp8r   Útorchao.testing.utilsr   Útorchao.utilsr   r   Úbitsandbytesrä   ÚImportErrorrî   ÚversionÚhipr�   rƒ   r   r‰   r#  r  r   r~   r8   r5   Ú<module>r‰     s4  ðó Û Û Ý ã Û Ý ÷ñ õ
 1Ý GÝ 8÷ó ð ×ÑÒØ€LÔå %Ý ÷ñ õ
 7Ý 6Ý 4Ý .÷ð
ÛðÛð 	‡=�=×ÑÒ Ø
‡K‚KÐ+ÀÕEá Ó"€ô_-�8ô _-ôDi�ô iðX Ð ôM"�ô M"ñ` ˜|Ô ,Ù ˜yÔ )ð ˆzÔÙ†Kñ øðg ô Ø
‚Cðûð
 ô Ø‚Dðús$   ÂD ÂD% Ä	D"Ä!D"Ä%	D1Ä0D1