+
     °³jSJ  ã                   ó  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^ RI	H
t
 ^ RIHt ^ RIHtHtHt ^ RIHt ^ RIHtHt ^ R	IHtHt R
 tRR ltRR ltR tR tR tRR ltRR lt R t!R t"R t#]$R8X  d
   ]"! 4        R# R# )é    )ÚdeepcopyN)ÚAdamW)ÚLambdaLR)Ú
DataLoader)ÚAcceleratorÚDataLoaderConfigurationÚGradientAccumulationPlugin)ÚGradientState)ÚRegressionDatasetÚRegressionModel)ÚDistributedTypeÚset_seedc           	      óö  € \        V P                  4       VP                  4       4       FÌ  w  rVVP                  '       g   K  V'       gW   \        P                  ! VP
                  VP
                  4      R J g&   Q RV RVP
                   RVP
                   R24       hKw  \        P                  ! VP
                  VP
                  3/ VB RJ d   K©  Q RV RVP
                   RVP
                   R24       h	  R# )	Fú7Gradients in sync when they should not be at iteration z:
model_a grad (z) == model_b grad (Ú)Tú7Gradients not in sync when they should be at iteration z) != model_b grad (N)ÚzipÚ
parametersÚrequires_gradÚtorchÚallcloseÚgrad)Úmodel_aÚmodel_bÚdid_stepÚ	iterationÚkwargsÚparamÚ
grad_params   &&&&,  Úh/home/luca/screen-checking/.venv/lib/python3.14/site-packages/accelerate/test_utils/scripts/test_sync.pyÚcheck_model_parametersr!      s  € ä  ×!3Ñ!3Ó!5°w×7IÑ7IÓ7KÖLÑˆØ×"×"Ð"Ùßä—>’> %§*¡*¨j¯o©oÓ>À%ÓGð ØIÈ)ÈÐTeÐfk×fpÑfpÐeqð  rEð  FP÷  FUñ  FUð  EVð  VWð  XóÒGô
 —>’> %§*¡*¨j¯o©oÑHÀÑHÈDÕPð ØIÈ)ÈÐTeÐfk×fpÑfpÐeqð  rEð  FP÷  FUñ  FUð  EVð  VWð  XóÐPó Mó    c                 ó  € V P                  4        V ! V4      p\        P                  ! WRP                  VP                  4      4      pV'       g%   WcP
                  ,          pVP                  4        R # VP                  V4       R # ©N)ÚtrainÚFÚmse_lossÚtoÚdeviceÚgradient_accumulation_stepsÚbackward)ÚmodelÚinputÚtargetÚacceleratorÚdo_backwardÚoutputÚlosss   &&&&&  r    Ú
step_modelr3   .   sW   € Ø	‡K�K„MÙ�5‹\€FÜ�:Š:�fŸi™i¨¯©Ó6Ó7€DßØ×7Ñ7Õ7ˆØ�‰Žà×Ñ˜TÖ"r"   c                óì  € \        ^*4       \        4       p\        V4      p\        ^PR7      p\	        V^R7      pVP                  V P                  4       V'       dS   \        VP                  4       RR7      p\        VP                  4       RR7      p\        VR R7      p\        VR R7      pV'       d   V P                  VXXV4      w  r7r…MV P                  W54      w  r5V'       d	   VXWVXX3# W#V3# )z3Returns everything needed to perform basic training©Úlength©Ú
batch_sizegü©ñÒMbP?)ÚparamsÚlrc                 ó   € V R ,          # ©gÍÌÌÌÌÌä?© ©Úepochs   &r    Ú<lambda>Ú$get_training_setup.<locals>.<lambda>D   s   € °e¸T¶kr"   )Ú	lr_lambdac                 ó   € V R ,          # r<   r=   r>   s   &r    r@   rA   E   s   € ¸eÀT¾kr"   )r   r   r   r   r   r(   r)   r   r   r   Úprepare)	r/   Úschedr,   Ú	ddp_modelÚdsetÚ
dataloaderÚoptÚddp_optÚ	ddp_scheds	   &&       r    Úget_training_setuprL   9   sà   € äˆR„LÜÓ€EÜ˜“€IÜ BÔ'€DÜ˜D¨RÔ0€JØ	‡H�Hˆ[×ÑÔ ßÜ˜5×+Ñ+Ó-°$Ô7ˆÜ˜y×3Ñ3Ó5¸$Ô?ˆÜ˜Ñ(AÔBˆÜ˜WÑ0IÔJˆ	çØ4?×4GÑ4GÈ	ÐSZÐ\eÐgqÓ4rÑ1ˆ	˜I zà +× 3Ñ 3°IÓ JÑˆ	ßØ�s˜E¨y¸'À9ÐMÐMØ˜ZÐ'Ð'r"   c           	      óÄ  € \        V 4      w  rp\        \        V4      4      P                  4       w  rE\	        ^4       EFŠ  pV P                  WE34      w  rxVP                  V P                  4      VP                  V P                  4      r‡\        WW€4       V^,          ^ 8X  d0   V P                  V4      ;_uu_ 4        \        W$WP4       RRR4       M\        W$WP4       \        WRV4       \        VP                  4       VP                  4       4       Fl  w  ršV	P                  '       g   K  \        P                  ! V	P                   V
P                   4      '       d   KL  Q RV	P                    RV
P                    R24       h	  \        P"                  ! RV,           4       V\        P$                  ! \'        V4      4      ,          pEK�  	  R#   + '       g   i     Lÿ; i)é   NTú7Gradients not in sync when they should be:
Model grad (ú) != DDP grad (r   é9  )rL   ÚnextÚiterÚvaluesÚrangeÚgatherr(   r)   r3   Úno_syncr!   r   r   r   r   r   r   Úmanual_seedÚrandpermÚlen©r/   r,   rF   rH   Ú	ddp_inputÚ
ddp_targetr   r-   r.   r   Ú	ddp_params   &          r    Útest_noop_syncr_   P   sx  € ä#5°kÓ#BÑ €E�jä ¤ jÓ!1Ó2×9Ñ9Ó;Ñ€IÜ˜1—Xˆ	à#×*Ñ*¨IÐ+BÓC‰ˆØŸ™ ×!3Ñ!3Ó4°f·i±iÀ×@RÑ@RÓ6Sˆvä�5 Ô5à�q�=˜AÔà×$Ñ$ Y×/Õ/Ü˜9°ÔI÷ 0Ð/ô �y¨ZÔEô 	˜u°°yÔAÜ # E×$4Ñ$4Ó$6¸	×8LÑ8LÓ8NÖ OÑˆEØ×&×&Ð&ÙÜ—>’> %§*¡*¨i¯n©n×=Ô=ð ØJÈ5Ï:É:È,ÐVeÐfo×ftÑftÐeuÐuvÐwóÐ=ñ !Pô 	×Ò˜$ Õ*Ô+ØœeŸnšn¬S°«^Ó<Õ=‹	ó5 ÷ 0×/ús   Â>GÇGc           	      ój  € \        V 4      w  rp\        \        V4      4      P                  4       w  rE\	        ^4       EFÜ  pV P                  WE34      w  rxVP                  V P                  4      VP                  V P                  4      r‡\        WW€4       V^,          ^ 8X  d0   V P                  V4      ;_uu_ 4        \        W$WP4       RRR4       M\        W$WP4       \        VP                  4       VP                  4       4       FË  w  ršV	P                  '       g   K  V^,          ^ 8X  dT   \        P                  ! V	P                  V
P                  4      RJ g#   Q RV	P                   RV
P                   R24       hKz  \        P                  ! V	P                  V
P                  4      RJ d   K«  Q RV	P                   RV
P                   R24       h	  \        P                   ! R	V,           4       V\        P"                  ! \%        V4      4      ,          pEKß  	  R#   + '       g   i     ELR; i)
rN   NFú7Gradients in sync when they should not be:
Model grad (ú) == DDP grad (r   TrO   rP   rQ   )rL   rR   rS   rT   rU   rV   r(   r)   r3   rW   r   r   r   r   r   r   rX   rY   rZ   r[   s   &          r    Útest_distributed_syncrc   r   sÈ  € ä#5°kÓ#BÑ €E�jä ¤ jÓ!1Ó2×9Ñ9Ó;Ñ€IÜ˜1—Xˆ	à#×*Ñ*¨IÐ+BÓC‰ˆØŸ™ ×!3Ñ!3Ó4°f·i±iÀ×@RÑ@RÓ6Sˆvä�5 Ô5à�q�=˜AÔà×$Ñ$ Y×/Õ/Ü˜9°ÔI÷ 0Ð/ô �y¨ZÔEô !$ E×$4Ñ$4Ó$6¸	×8LÑ8LÓ8NÖ OÑˆEØ×&×&Ð&ÙØ˜1�} Ô!ä—~’~ e§j¡j°)·.±.ÓAÀUÓJð ØNÈuÏzÉzÈlÐZiÐjs×jxÑjxÐiyÐyzÐ{óÒJô
 —~’~ e§j¡j°)·.±.ÓAÀTÕIð ØNÈuÏzÉzÈlÐZiÐjs×jxÑjxÐiyÐyzÐ{óÐIñ !Pô 	×Ò˜$ Õ*Ô+ØœeŸnšn¬S°«^Ó<Õ=‹	óA ÷ 0×/Ð/ús   Â>H!È!H2c           	      óš  € \        V 4      w  rp. p^p\        V4       Fæ  p\        \        V4      4      P	                  4       w  rxV P                  Wx34      w  ršV	P                  V P                  4      V
P                  V P                  4      r©\        WW 4       V P                  V4      ;_uu_ 4        V! V4      p\        P                  ! W¸P                  VP                  4      4      pVP                  V4       RRR4       Kè  	  \        V4       EF‹  pWF,          pWe^,
          8  dª   V P                  V4       \        VP                  4       VP                  4       4       Fj  w  rÞVP                   '       g   K  \"        P$                  ! VP&                  VP&                  4      RJ d   KJ  Q RVP&                   RVP&                   R24       h	  KÂ  V P)                  V4      ;_uu_ 4        V P                  V4       RRR4       \        VP                  4       VP                  4       4       Fj  w  rÞVP                   '       g   K  \"        P$                  ! VP&                  VP&                  4      RJ d   KJ  Q RVP&                   RVP&                   R24       h	  EKŽ  	  R#   + '       g   i     EK™  ; i  + '       g   i     LÂ; i)	rN   NFra   rb   r   TrO   rP   )rL   rU   rR   rS   rT   rV   r(   r)   r3   rW   r&   r'   Úappendr+   r   r   r   r   r   r   Útrigger_sync_in_backward)r/   r,   rF   rH   ÚlossesÚnum_iterationsr   r\   r]   r-   r.   Ú
ddp_outputr2   r   r^   s   &              r    Ú"test_distributed_sync_multiple_fwdrj   š   sJ  € ä#5°kÓ#BÑ €E�jà€FØ€NÜ˜>Ö*ˆ	Ü $¤T¨*Ó%5Ó 6× =Ñ =Ó ?Ñˆ	ð $×*Ñ*¨IÐ+BÓC‰ˆØŸ™ ×!3Ñ!3Ó4°f·i±iÀ×@RÑ@RÓ6Sˆvô 	�5 Ô5ð × Ñ  ×+Õ+Ù" 9Ó-ˆJÜ—:’:˜j¯-©-¸
×8IÑ8IÓ*JÓKˆDØ�M‰M˜$Ô÷ ,Ñ+ñ +ô" ˜>×*ˆ	ØÕ ˆà¨Õ)Ô)à× Ñ  Ô&ô %(¨×(8Ñ(8Ó(:¸I×<PÑ<PÓ<RÖ$SÑ �Ø×*×*Ð*Ùä—~’~ e§j¡j°)·.±.ÓAÀUÕJð ØNÈuÏzÉzÈlÐZiÐjs×jxÑjxÐiyÐyzÐ{óÐJó	 %Tð ×5Ñ5°i×@Õ@Ø×$Ñ$ TÔ*÷ Aô %(¨×(8Ñ(8Ó(:¸I×<PÑ<PÓ<RÖ$SÑ �Ø×*×*Ð*Ùä—~’~ e§j¡j°)·.±.ÓAÀTÕIð ØNÈuÏzÉzÈlÐZiÐjs×jxÑjxÐiyÐyzÐ{óÐIô	 %Tó- +÷ ,×+Ñ+ú÷0 A×@ús   Â3A	J%Ç.J:Ê%J7Ê:K
c           
      óæ  € \        ^VR7      p\        WR7      p\        VVR7      p\        V4      w  rgp\	        V4       EF  w  ršV
P                  4       w  r¼VP                  W¼34      w  rÞVP                  VP                  4      VP                  VP                  4      rí\        WmWåR4       VP                  V4      ;_uu_ 4        \        W{WÅ4       RRR4       \        VP                  4       VP                  4       4       Fø  w  ppVP                  '       g   K  V	^,           ^,          ^ 8X  g    V	\        V4      ^,
          8X  g	   V'       dW   \        P                   ! VP"                  VP"                  4      RJ g&   Q RV	 RVP"                   R	VP"                   R
24       hK¤  \        P                   ! VP"                  VP"                  4      RJ d   KÕ  Q RV	 RVP"                   RVP"                   R
24       h	  \        P$                  ! RV	,           4       V\        P&                  ! \        V4      4      ,          pEK  	  \(        P*                  ! 4        R#   + '       g   i     EL”; i)é   ©Ú	num_stepsÚsync_each_batch©Úsplit_batchesÚdispatch_batches©Údataloader_configÚgradient_accumulation_pluginFNTr   z:
Model grad (rP   r   r   rb   rQ   )r	   r   r   rL   Ú	enumeraterT   rV   r(   r)   r3   Ú
accumulater   r   r   rZ   r   r   r   rX   rY   r
   Ú_reset_state)rq   rr   ro   ru   rt   r/   r,   rF   rH   r   Úbatchr\   r]   r-   r.   r   r^   s   &&&              r    Útest_gradient_accumulationrz   Ð   sD  € Ü#=ÈÐ[jÔ#kÐ Ü/¸mÔoÐÜØ+Ø%Aô€Kô
 $6°kÓ#BÑ €E�jÜ% j×1Ñˆ	Ø %§¡£Ñˆ	à#×*Ñ*¨IÐ+BÓC‰ˆØŸ™ ×!3Ñ!3Ó4°f·i±iÀ×@RÑ@RÓ6Sˆvä�5 °eÔ<à×#Ñ# I×.Õ.Ü�y¨ZÔE÷ /ô !$ E×$4Ñ$4Ó$6¸	×8LÑ8LÓ8NÖ OÑˆE�9Ø×&×&Ð&ÙØ˜Q• !Õ# qÔ(¨i¼3¸z»?ÈQÕ;NÔ.N×Sbä—~’~ e§j¡j°)·.±.ÓAÀTÓIð ØMÈiÈ[ÐXgÐhm×hrÑhrÐgsð  tCð  DM÷  DRñ  DRð  CSð  STð  UóÒIô
 —~’~ e§j¡j°)·.±.ÓAÀUÕJð ØMÈiÈ[ÐXgÐhm×hrÑhrÐgsð  tCð  DM÷  DRñ  DRð  CSð  STð  UóÐJñ !Pô 	×Ò˜$ Õ*Ô+ØœeŸnšn¬S°«^Ó<Õ=‹	ñ9 2ô: ×ÒÖ ÷+ /×.Ð.ús   ÃIÉI0c           	      óØ  € \        ^VR7      p\        WR7      p\        VVR7      p\        VR4      w  rgr‰r«p\	        V	4       EFy  w  rÞVP                  4       w  ppVP                  VV34      w  ppVP                  VP                  4      VP                  VP                  4      ppVP                  4        V
P                  4        \        VVVVR4       VP                  4        V^,           ^,          ^ 8X  g   V^,           \        V	4      8X  dF   V '       d   VP                  4        M,\        VP                  4       F  pVP                  4        K  	  VP                  V
4      ;_uu_ 4        \        W¯VV4       VP                  4        VP                  4        RRR4       VP                   ^ ,          R,          VP                   ^ ,          R,          8X  g?   Q RVP                   ^ ,          R,           R	VP                   ^ ,          R,           R
24       hV^,           ^,          ^ 8H  ;'       g    V^,           \        V	4      8H  pVP                  ^8”  d   \#        TT
T;'       g    TVRR7       V'       d   VP%                  4        VP%                  4        \&        P(                  ! RV,           4       EK|  	  \*        P,                  ! 4        R#   + '       g   i     ELB; i)rl   rm   rp   rs   TFNr:   z:Learning rates found in each optimizer did not align
opt: z

DDP opt: Ú
g{®Gáz„?)ÚrtolrQ   )r	   r   r   rL   rv   rT   rV   r(   r)   r%   r3   ÚsteprZ   rU   Únum_processesrw   Úparam_groupsr!   Ú	zero_gradr   rX   r
   rx   )rq   rr   ro   ru   rt   r/   r,   rI   rE   rH   rF   rJ   rK   r   ry   r\   r]   r-   r.   Ú_r   s   &&&                  r    Ú1test_gradient_accumulation_with_opt_and_schedulerrƒ   ù   sŽ  € ô $>ÈÐ[jÔ#kÐ Ü/¸mÔoÐÜØ+Ø%Aô€Kô
 DVÐVaÐcgÓChÑ@€E� 9°yÜ% j×1Ñˆ	Ø %§¡£Ñˆ	�:à#×*Ñ*¨I°zÐ+BÓC‰ˆˆvØŸ™ ×!3Ñ!3Ó4°f·i±iÀ×@RÑ@RÓ6Sˆvˆà�‰ŒØ�‰ÔÜ�5˜% ¨°eÔ<Ø�‰Œ
à˜�]˜aÕ 1Ô$¨9°q­=¼SÀ»_Ô*LßØ—
‘
•ä˜{×8Ñ8Ö9�AØ—J‘J–Lñ :ð ×#Ñ# I×.Õ.Ü�y¨Z¸ÔEØ�L‰LŒNØ�N‰NÔ÷ /ð ×Ñ Õ" 4Õ(¨G×,@Ñ,@ÀÕ,CÀDÕ,IÔIð 	
ØIÈ#×JZÑJZÐ[\ÕJ]Ð^bÕJcÐIdÐdoÐpw÷  qEñ  qEð  FGõ  qHð  IMõ  qNð  pOð  OQð  Ró	
ÐIð   !•m qÕ(¨QÑ.×WÐW°YÀµ]ÄsÈ:ÃÑ4VˆØ×$Ñ$ qÔ(Ü"ØØØ×+Ð+˜OØØõ÷ Ø�M‰MŒOØ×ÑÔô 	×Ò˜$ Õ*×+ñW 2ôX ×ÒÖ ÷3 /×.Ð.ús   Å1.KËK)c                  ó†  € \        4       p \        ^PR7      p\        V^R7      p\        ^`R7      p\        V^R7      pV P                  W$4      w  r$V P                  P
                  e   Q h\        V4       EF-  w  rV\        V P                  P
                  4      \        V4      8X  g   Q hV\        V4      ^,
          8  dÄ   V P                  P                  '       d   Q hV^8X  dœ   \        V4       F‰  w  rv\        V P                  P
                  4      \        V4      8X  g   Q hV\        V4      ^,
          8  d!   V P                  P                  '       d   Q hKk  V P                  P                  '       d   K‰  Q h	  EK  EK  V P                  P                  '       d   EK.  Q h	  V P                  P
                  e   Q hR# )éP   r5   r7   N)
r   r   r   rD   Úgradient_stateÚactive_dataloaderrv   ÚidrZ   Úend_of_dataloader)r/   Ú
first_dsetÚfirst_dataloaderÚsecond_dsetÚsecond_dataloaderr   r‚   Ú	batch_nums           r    Útest_dataloader_breakr�   3  s„  € Ü“-€KÜ"¨"Ô-€JÜ! *¸Ô<ÐÜ#¨2Ô.€KÜ" ;¸2Ô>ÐØ*5×*=Ñ*=Ð>NÓ*bÑ'Ðà×%Ñ%×7Ñ7Ò?Ð?Ð?Ü!Ð"2×3‰ˆ	Ü�+×,Ñ,×>Ñ>Ó?Ä2ÐFVÓCWÔWÐWÐWØ”sÐ+Ó,¨qÕ0Ô0Ø"×1Ñ1×C×CÐCÐCÐCØ˜AŒ~Ü$-Ð.?Ö$@‘L�IÜ˜k×8Ñ8×JÑJÓKÌrÐRcÓOdÔdÐdÐdØ ¤3Ð'8Ó#9¸AÕ#=Ô=Ø#.×#=Ñ#=×#O×#OÐ#OÐOÐOÑ#Oà*×9Ñ9×K×KÒKÐKÐKô %Aò ð ×-Ñ-×?×?Ó?Ð?Ð?ñ 4ð ×%Ñ%×7Ñ7Ò?Ð?Ò?r"   c                  ó¸  € \        4       p V P                  pVP                  ^ 8X  d   \        R4       VP                  \
        P                  8w  d   \        4        VP                  \
        P                  8X  d(   VP                  ^ 8X  d   \        R4       \        V 4       VP                  \
        P                  \
        P                  \
        P                  \
        P                  \
        P                  \
        P                  \
        P                   \
        P"                  39   dO   VP                  ^ 8X  d   \        R4       \%        V 4       VP                  ^ 8X  d   \        R4       \'        V 4       VP                  \
        P                  \
        P                  \
        P                  \
        P                  \
        P                  \
        P                   \
        P"                  39   dO   R FH  pR F?  pR F6  pVP                  ^ 8X  d   \        RRV RV RV R	24       \)        W#V4       K8  	  KA  	  KJ  	  VP                  ^ 8X  d   \        R
R4       \+        4        VP                  \
        P                  \
        P                  \
        P                  \
        P                  \
        P                  \
        P                   \
        P"                  39   di   R Fb  pR FY  pR FP  pV'       g   V'       g   V'       g   K  VP                  ^ 8X  d   \        R
RV RV RV R	24       \+        W#V4       KR  	  K[  	  Kd  	  VP-                  4        R# )r   zA**Test `accumulate` gradient accumulation with dataloader break**z'**Test NOOP `no_sync` context manager**z.**Test Distributed `no_sync` context manager**zE**Test Distributed `no_sync` context manager with multiple forwards**z+**Test `accumulate` gradient accumulation, z`split_batches=z` and `dispatch_batches=z` and `sync_each_batch=z`**zH**Test `accumulate` gradient accumulation with optimizer and scheduler, zJ`split_batches=False`, `dispatch_batches=False`, `sync_each_batch=False`**N)TF)r   ÚstateÚlocal_process_indexÚprintÚdistributed_typer   ÚXLAr�   ÚNOr_   Ú	MULTI_GPUÚ	MULTI_NPUÚ	MULTI_MLUÚ
MULTI_SDAAÚ
MULTI_MUSAÚ	MULTI_CPUÚ	MULTI_HPUÚMULTI_NEURONrc   rj   rz   rƒ   Údestroy_process_group)r/   r‘   Úsplit_batchrr   ro   s        r    Úmainr¡   L  sà  € Ü“-€KØ×Ñ€EØ× Ñ  AÔ%ÜÐQÔRØ×Ñ¤×!4Ñ!4Ô4ÜÔØ×Ñ¤×!3Ñ!3Ô3Ø×$Ñ$¨Ô)ÜÐ;Ô<Ü�{Ô#Ø×ÑÜ×!Ñ!Ü×!Ñ!Ü×!Ñ!Ü×"Ñ"Ü×"Ñ"Ü×!Ñ!Ü×!Ñ!Ü×$Ñ$ð	"ô 	ð ×$Ñ$¨Ô)ÜÐBÔCÜ˜kÔ*Ø×$Ñ$¨Ô)ÜÐYÔZÜ*¨;Ô7Ø×ÑÜ×!Ñ!Ü×!Ñ!Ü×!Ñ!Ü×"Ñ"Ü×"Ñ"Ü×!Ñ!Ü×$Ñ$ð"ô ó )ˆKÛ$1Ð Û'4�OØ×0Ñ0°AÔ5ÜØIØ-¨k¨]Ð:RÐScÐRdÐd{ð  }Lð  |Mð  MPð  Qôô /¨{ÈoÖ^ó (5ó %2ñ )ð × Ñ  AÔ%ÜØVØXô	
ô 6Ô7Ø×ÑÜ×!Ñ!Ü×!Ñ!Ü×!Ñ!Ü×"Ñ"Ü×"Ñ"Ü×!Ñ!Ü×$Ñ$ð"ô ó )ˆKÛ$1Ð Û'4�Oß&×/?ÏÙ Ø×0Ñ0°AÔ5ÜØfØ-¨k¨]Ð:RÐScÐRdÐd{ð  }Lð  |Mð  MPð  Qôô FÀkÐetÖuó (5ó %2ñ )ð 
×ÑÖ!r"   c                 ó   € \        4        R # r$   )r¡   )Úindexs   &r    Ú_mp_fnr¤   ˜  s   € ä†Fr"   Ú__main__)T)F)FFF)%Úcopyr   r   Útorch.nn.functionalÚnnÚ
functionalr&   Útorch.optimr   Útorch.optim.lr_schedulerr   Útorch.utils.datar   Úaccelerate.acceleratorr   r   r	   Úaccelerate.stater
   Úaccelerate.test_utilsr   r   Úaccelerate.utilsr   r   r!   r3   rL   r_   rc   rj   rz   rƒ   r�   r¡   r¤   Ú__name__r=   r"   r    Ú<module>r²      s~   ðõ ã ß Ð Ý Ý -Ý 'ç cÑ cÝ *ß Dß 6òô"#ô(ò.>òD%>òP3ôl&!ôR7!òt@ò2I"òXð
 ˆzÔÙ†Fñ r"   