+
    "Ý³jü(  ã            
       óZ  € ^ RI t ^ RIHtHtHtHtHt ^ RIt^ RIH	t	 ^ RI
Hu Ht ^ RIHt ^ RIHt ^ RIHtHtHtHt ^ RIHt ]P2                  ! 4       R R l4       t]P6                  P8                  P:                  P<                  ]P6                  P8                  P>                  P<                  ]P6                  P8                  P@                  PB                  ]P6                  P8                  PD                  P<                  ]P6                  P8                  PF                  P<                  ]P6                  P8                  PH                  P<                  ]P6                  P8                  PJ                  P<                  ]P6                  P8                  PL                  P<                  ]P6                  P8                  PN                  PB                  ]P6                  P8                  PP                  P<                  0
t) ! R	 R
]PB                  4      t*]PV                  PY                  ]*.4       R# )é    N)ÚAnyÚListÚOptionalÚSetÚTuple)Úsuggest_memory_format)Úhp_tensor_to_float8_dynamic)ÚFloat8TrainingTensorÚGemmInputRoleÚLinearMMConfigÚhp_tensor_and_scale_to_float8)ÚEPSc                ó<   € V ^8„  d   QhR\         P                  RR/# )é   ÚmoduleÚreturnN)ÚnnÚModule)Úformats   "ÚZ/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torchao/float8/fsdp_utils.pyÚ__annotate__r      s"   € ÷ 9Vñ 9V´R·Y±Yð 9VÀ4ñ 9Vó    c                óæ  € ^ RI Hp ^ RIHp \        P
                  P                  R4       V P                  4        Uu. uFb  p\        W24      '       g   K  \        VP                  V4      '       g   K4  \        VP                  P                  \        4      '       g   K`  VNKd  	  ppV Uu. uF  qUP                  NK  	  ppV Uu0 uF#  pVP                  P                  P                  kK%  	  ppV'       g   R# Vw  p\        P                  ! V\         P"                  R7      p	\        P$                  ! V	4      p
\        P&                  ! V
\(        4      p
V
P*                  pV
P-                  \        P.                  4      p
\        P0                  ! V4      P2                  V
,          pV\        P4                  J dE   \        P&                  ! V\        P0                  ! \        P4                  4      P2                  R7      pVP7                  4       P-                  \        P8                  4      p\;        V4       F&  w  råWÞ,          VP                  P                  n        K(  	  R# u upi u upi u upi )ac  
Calculate scale dynamically for all float8 parameters.

This should be run after the optimizer step. It performs a single all-reduce
to compute the scales for all float8 weights.

Args:
    module: The module containing float8 parameters.

Example::

    model(input).sum().backward()
    optim.step()
    precompute_float8_dynamic_scale_for_fsdp(model)
©ÚDTensor)ÚFloat8Linearz7torchao.float8.precompute_float8_dynamic_scale_for_fsdpN)Úord)Úmax)Útorch.distributed._tensorr   Útorchao.float8.float8_linearr   ÚtorchÚ_CÚ_log_api_usage_onceÚmodulesÚ
isinstanceÚweightÚ_local_tensorÚ!WeightWithDynamicFloat8CastTensorÚconfigÚcast_config_weightÚtarget_dtypeÚ_foreach_normÚmathÚinfÚstackÚclampr   ÚdtypeÚtoÚfloat64Úfinfor   Úfloat16Úto_localÚfloat32Ú	enumerateÚ_precomputed_scale)r   r   r   ÚmÚfloat8_linearsÚfloat8_linearÚweightsÚtarget_dtypesr+   Úmax_weightsÚamax_tensorÚorigin_dtypeÚscale_tensorÚlocal_scale_tensorÚis   &              r   Ú(precompute_float8_dynamic_scale_for_fsdprE      sÙ  € õ" 2å9ä	‡H�H× Ñ ØAôð —‘Ô!ó*á!ˆAÜ�a×&ô 	
ô �q—x‘x ×)ô 	
ô �q—x‘x×-Ñ-Ô/P×Q÷	 	
ˆÙ!ð ð *ñ IWÓWÉ°}×2Ô2É€GÐWñ ,ó'á+ˆMð 	×Ñ×/Ñ/×<Ô<Ù+ð ð '÷
 ÙØ#�O€\ô ×%Ò% g´4·8±8Ô<€KÜ—+’+˜kÓ*€Kô —+’+˜k¬3Ó/€Kð ×$Ñ$€LØ—.‘.¤§¡Ó/€KÜ—;’;˜|Ó,×0Ñ0°;Õ>€LØ”u—}‘}Ó$Ü—{’{ <´U·[²[ÄÇÁÓ5O×5SÑ5SÔTˆØ%×.Ñ.Ó0×3Ñ3´E·M±MÓBÐÜ% nÖ5ÑˆØ@RÕ@Uˆ×Ñ×*Ñ*Ö=ó 6ùòA*ùò Xùò's#   ¾I$ÁI$Á4'I$Â I$Â,I)Ã)I.c                   ó¦   a € ] tR t^‹t o ]RV 3R lR ll4       tRV 3R lR llt]RR l4       tR t	]R 4       t
R	 tR
 tRR/V 3R lR lltRtV tR# )r(   Nc          	      ó~   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  RS[S[ P                  ,          /# ©r   ÚtensorÚlinear_mm_configr1   Úprecomputed_scale©r!   ÚTensorr   r1   r   )r   Ú__classdict__s   "€r   r   Ú.WeightWithDynamicFloat8CastTensor.__annotate__�   sD   ø€ ÷ 
ñ 
á—‘ð
ñ )ð
ñ �{‰{ð	
ñ
 $¡E§L¡LÕ1ñ
r   c                ó&  € \         P                  P                  V VP                  4       VP	                  4       VP                  4       \        V4      VP                  VP                  VP                  VP                  4       VP                  R 7
      # ))ÚstridesÚstorage_offsetÚmemory_formatr1   ÚlayoutÚdeviceÚ
pin_memoryÚrequires_grad)r!   rM   Ú_make_wrapper_subclassÚsizeÚstriderR   r   r1   rT   rU   Ú	is_pinnedrW   )ÚclsrI   rJ   r1   rK   s   &&&&&r   Ú__new__Ú)WeightWithDynamicFloat8CastTensor.__new__Œ   ss   € ô �|‰|×2Ñ2ØØ�K‰K‹MØ—M‘M“OØ!×0Ñ0Ó2Ü/°Ó7Ø—,‘,Ø—=‘=Ø—=‘=Ø×'Ñ'Ó)Ø ×.Ñ.ð 3ó 
ð 	
r   c          	      ó~   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  RS[S[ P                  ,          /# rH   rL   )r   rN   s   "€r   r   rO   ¡   sD   ø€ ÷ 4ñ 4á—‘ð4ñ )ð4ñ �{‰{ð	4ñ
 $¡E§L¡LÕ1ñ4r   c                ó6   € Wn         W n        W0n        W@n        R # ©N)Ú_tensorÚ_linear_mm_configÚ_dtyper9   )ÚselfrI   rJ   r1   rK   s   &&&&&r   Ú__init__Ú*WeightWithDynamicFloat8CastTensor.__init__¡   s   € ð ŒØ!1ÔØŒð #4Ör   c                óÔ  aa€ V\         P                  P                  P                  P                  8X  dA   \        V^ ,          P                  V^ ,          P                  V^ ,          P                  4      # RoRoVV3R lp\        P                  ! \
        YST;'       g    / 34      w  r4V! V/ VB pV\        9  d   V# \        P                  ! \         P                  VV3R lV4      # )r   Nc                 óÂ   <€ Sf   V P                   oMV P                   S8X  g   Q hSf   V P                  oV P                  # V P                  S8X  g   Q hV P                  # ra   )rc   rd   rb   )Útr1   Ú	mm_configs   &€€r   ÚunwrapÚDWeightWithDynamicFloat8CastTensor.__torch_dispatch__.<locals>.unwrap¹   s`   ø€ àÒ Ø×/Ñ/‘	à×*Ñ*¨iÔ7Ð7Ð7àŠ}ØŸ™�ð —9‘9Ðð —x‘x 5Ô(Ð(Ð(Ø—9‘9Ðr   c                 ó   <€ \        V SS4      # ra   )r(   )Úxr1   rk   s   &€€r   Ú<lambda>ÚFWeightWithDynamicFloat8CastTensor.__torch_dispatch__.<locals>.<lambda>Î   s   ø€ Ô7¸¸9ÀeÔLr   )r!   ÚopsÚatenÚdetachÚdefaultr(   rb   rc   rd   ÚpytreeÚtree_map_onlyÚ_ops_to_preserve_subclassrM   )	r\   ÚfuncÚtypesÚargsÚkwargsrl   Úoutr1   rk   s	   &&&&&  @@r   Ú__torch_dispatch__Ú4WeightWithDynamicFloat8CastTensor.__torch_dispatch__°   sÁ   ù€ à”5—9‘9—>‘>×(Ñ(×0Ñ0Ô0Ü4Ø�Q•—‘  a¥×!:Ñ!:¸DÀ½G¿N¹Nóð ð /3ˆ	Ø'+ˆö	ô ×+Ò+Ü-¨v¸f¿l¸lÈÐ7Kó
‰ˆñ �DÐ#˜FÑ#ˆØÔ0Ô0ØˆJÜ×#Ò#Ü�L‰LÝLØó
ð 	
r   c                ó†   € R .pV P                   '       d   VP                  R4       VRV P                  RV P                  /3# )rb   r9   rk   r1   )r9   Úappendrc   rd   )re   Útensorss   & r   Ú__tensor_flatten__Ú4WeightWithDynamicFloat8CastTensor.__tensor_flatten__Ò   s@   € Ø�+ˆØ×"×"Ð"Ø�N‰NÐ/Ô0Ø˜ d×&<Ñ&<¸gÀtÇ{Á{ÐSÐSÐSr   c           
     ó^   € \        V R ,          VR,          VR,          \        V RR4      4      # )rb   rk   r1   r9   N)r(   Úgetattr)Úinner_tensorsÚflatten_specÚ
outer_sizeÚouter_strides   &&&&r   Ú__tensor_unflatten__Ú6WeightWithDynamicFloat8CastTensor.__tensor_unflatten__Ø   s4   € ä0Ø˜)Õ$Ø˜Õ%Ø˜Õ!Ü�MÐ#7¸Ó>ó	
ð 	
r   c                óV   € R V P                    RV P                   RV P                   R2# )z)WeightWithDynamicFloat8CastTensor(tensor=z, linear_mm_config=z, dtype=Ú))rb   rc   rd   )re   s   &r   Ú__repr__Ú*WeightWithDynamicFloat8CastTensor.__repr__á   sN   € Ø:¸4¿<¹<¸.ÐH[Ð\`×\rÑ\rÐ[sÐs{ð  }A÷  }Hñ  }Hð  |Ið  IJð  Kð  	Kr   c           	     óH  € V P                   pVe=   \        V P                  VV P                  V P                  \
        P                  4      pM=\        V P                  V P                  V P                  R\
        P                  VR7      pVP                  3VP                  33# )NT)Úreduce_amaxÚgemm_input_roleÚdevice_mesh)
r9   r   rb   rd   rc   r   ÚWEIGHTr	   Ú_dataÚ_scale)re   ÚmeshrK   Úfloat8_training_tensors   &&  r   Úfsdp_pre_all_gatherÚ5WeightWithDynamicFloat8CastTensor.fsdp_pre_all_gatherä   s˜   € ð
 !×3Ñ3ÐØÒ(Ü%BØ—‘Ø!Ø—‘Ø×&Ñ&Ü×$Ñ$ó&Ñ"ô &AØ—‘Ø—‘Ø×&Ñ&Ø Ü -× 4Ñ 4Ø ô&Ð"ð '×,Ñ,Ð.Ð1G×1NÑ1NÐ0PÐPÐPr   r}   c          	      ó’   <€ V ^8„  d   QhRS[ S[P                  R3,          RS[RS[P                  RS[S[P                  ,          /# )r   Úall_gather_outputs.ÚmetadataÚparam_dtyper}   )r   r!   rM   r   r1   r   )r   rN   s   "€r   r   rO   ý   sN   ø€ ÷ ñ á!¡%§,¡,°Ð"3Õ4ðñ ðñ —[‘[ð	ñ ‘e—l‘lÕ#ñr   c               óZ  € Vw  pVw  pVev   ^ RI Hp \        V\        4      '       d	   Wdn        R # \        WG4      '       d3   \        VP
                  \        4      '       d   WdP
                  n        R # \        RV 24      h\        VVVV P                  \        P                  R7      V33# )Nr   z[out must be a Float8TrainingTensor or DTensor(_local_tensor=Float8TrainingTensor), but got )r“   )
r   r   r%   r
   r—   r'   ÚRuntimeErrorrc   r   r•   )re   r�   rž   rŸ   r}   ÚdataÚscaler   s   &&&&$   r   Úfsdp_post_all_gatherÚ6WeightWithDynamicFloat8CastTensor.fsdp_post_all_gatherý   s»   € ð %‰ˆØ‰ˆØŠ?Ý9ä˜#Ô3×4Ò4Ø"”
ñ ô ˜C×)Ò)¬jØ×!Ñ!Ô#7÷/ò /ð ,1×!Ñ!Ô(ñ
 ô #ØqÐruÐqvÐwóð ô $ØØØØ×"Ñ"Ü)×0Ñ0ô
ð ˆ7ðð 	r   )rd   rc   r9   rb   ra   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr]   rf   Úclassmethodr~   rƒ   r‹   r�   rš   r¤   Ú__static_attributes__Ú__classdictcell__)rN   s   @r   r(   r(   ‹   sz   ø‡ € Ø÷
ñ 
ó ð
÷(4ò 4ð ó
ó ð
òBTð ñ
ó ð
òKòQð2ð '+÷÷ ð r   r(   )-r-   Útypingr   r   r   r   r   r!   Útorch.nnr   Útorch.utils._pytreeÚutilsÚ_pytreerv   Útorch._prims_commonr   Ú#torchao.float8.float8_scaling_utilsr	   Ú%torchao.float8.float8_training_tensorr
   r   r   r   Útorchao.float8.float8_utilsr   Úno_gradrE   rr   rs   Ú
empty_likeru   Ú	new_zerosÚslicerM   Úcopy_ÚviewÚ
as_stridedÚ_to_copyÚ_pin_memoryÚsplitÚclonerx   r(   ÚserializationÚadd_safe_globals© r   r   Ú<module>rÅ      sn  ðó ß 2Õ 2ã Ý ß $Ð $Ý 5õ÷ó õ ,ð ‡‚ƒô9Vó ð9Vð@ 
‡I�I‡N�N×Ñ×%Ñ%Ø	‡I�I‡N�N×Ñ×$Ñ$Ø	‡I�I‡N�N×Ñ×ÑØ	‡I�I‡N�N×Ñ× Ñ Ø	‡I�I‡N�N×Ñ×ÑØ	‡I�I‡N�N×Ñ×%Ñ%Ø	‡I�I‡N�N×Ñ×#Ñ#Ø	‡I�I‡N�N×Ñ×&Ñ&Ø	‡I�I‡N�N×Ñ×ÑØ	‡I�I‡N�N×Ñ× Ñ ðÐ ô`P¨¯©ô Pðh × Ñ × $Ñ $Ð&GÐ%HÖ Ir   