+
    "Ý³j±Ö ã                  ó|  € R t ^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	H
t
 ^ RIHtHt ^ RIHtHtHtHtHtHtHtHtHt ^ RIt^ RIHu Ht ^ RIHt ^ RIHt ^ RIH t! ^R	I"H#t#H$t$H%t%H&t& ^R
I'H(t( ^RI)H*t* ^RI+H,t,H-t-H.t.H/t/H0t0H1t1H2t2H3t3H4t4 ^RI5H6t6 ^RI7H8t8 ^RI9H:t:H;t;H<t<H=t= ^RI>H?t?H@t@HAtAHBtBHCtCHDtDHEtEHFtFHGtGHHtHHItIHJtJHKtK ^RILHMtM ^RINHOtOHPtP ^RIQHRtRHStSHTtTHUtUHVtVHWtWHXtX ^RIYHZtZ ^RI[H\t\ ^RI]H^t^H_t_ ]P! 4       t`] ! R R4      4       ta] ! R R4      4       tbR R ltc ! R R4      td ! R R ]&4      teR# )!aH  ParoQuant looper integration.

This processor keeps ParoQuant separate from the AWQ lifecycle:
1. capture calibration activations for each module
2. run ParoQuant's transformed-domain optimization per layer
3. export packed runtime tensors plus learned rotation state
4. replace the float modules with ParoQuant runtime kernels
)ÚannotationsN)Únullcontext)Ú	dataclassÚfield)	ÚAnyÚCallableÚDictÚIteratorÚListÚOptionalÚSequenceÚSetÚTuple)Únn)ÚModule)Ú
checkpoint)ÚDTYPE_SIZE_COLUMNÚExecutionConfigÚMODULE_FEATURE_COLUMNÚLoopProcessor)ÚNamedModule)Ú
BaseQModel)	ÚPROCESS_LOG_FWD_TIMEÚPROCESS_LOG_LAYERÚPROCESS_LOG_MODULEÚPROCESS_LOG_NAMEÚPROCESS_LOG_TIMEÚPROCESS_USED_MEMORYÚQUANT_LOG_LOSSÚQUANT_LOG_NSAMPLESÚQUANT_LOG_DAMP)ÚHookedLinear)Ú
ParoLinear)ÚFORMATÚMETHODÚQuantizeConfigÚresolve_quant_format)Ú_ParoQuantOptimLinearÚ_activate_stage_paramsÚ_normalize_group_sizeÚ_normalize_opt_implÚ_normalize_opt_optimizerÚ_normalize_quantizer_implÚ_quantizer_sym_for_implÚ"_resolve_best_state_snapshot_dtypeÚ_result_from_modelÚbuild_paroquant_optimizerÚbuild_random_rotation_buffersÚ'build_random_rotation_buffers_referenceÚoptimize_paroquant_linear)Únormalize_fallback)Úlog_time_blockÚsetup_logger)Úcreate_quant_moduleÚget_module_by_name_prefixÚmove_toÚnested_move_toÚpack_moduleÚrecurse_getattrÚrecurse_setattr)Úparent_module_lock)Ú$prewarm_paroquant_rotation_extension)ÚCPUÚtorch_empty_cachec                  óH  € ] tR t^Vt$ Rt]! ]R7      tR]R&   Rt	R]R&   Rt
R]R&   RtR]R	&   RtR
]R&   RtR]R&   RtR]R&   RtR]R&   RtR]R&   RtR]R&   RtR]R&   RtR]R&   ]! ]R7      tR]R&   ]! ]R7      tR]R&   RtR]R&   ]! ]P2                  R7      tR]R &   R!tR# )"Ú_ParoQuantLayerStatezGPer-layer bookkeeping for activation capture and deferred quantization.)Údefault_factoryzDict[str, NamedModule]ÚmodulesNzOptional[torch.nn.Module]Úlayer_moduleÚpristine_layer_moduleÚprepared_group_source_modulez$Optional[Dict[str, torch.nn.Module]]Ú&prepared_group_source_module_by_devicez"Optional[List[List[torch.Tensor]]]Úlayer_inputsz'Optional[List[Dict[str, torch.Tensor]]]Úlayer_input_kwargsÚlayer_outputszOptional[Any]Úgrouped_datasetzOptional[Dict[str, Any]]Úgrouped_dataset_by_deviceÚreplay_batchesúOptional[int]Úsubset_totalzSet[int]Úprocessed_subsetszSet[str]Úpending_modulesFÚboolÚ	quantizedzthreading.LockÚlock© )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   ÚdictrE   Ú__annotations__rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rQ   ÚsetrR   rS   rU   Ú	threadingÚLockrV   Ú__static_attributes__rW   ó    Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/gptqmodel/looper/paroquant_processor.pyrC   rC   V   sÌ   ‡ áQá&+¸DÔ&A€GÐ#ÓAØ.2€LÐ+Ó2Ø7;ÐÐ4Ó;Ø>BÐ Ð";ÓBØSWÐ*Ð,PÓWØ7;€LÐ4Ó;ØBFÐÐ?ÓFØ8<€MÐ5Ó<Ø%)€O�]Ó)Ø:>ÐÐ7Ó>Ø$(€N�MÓ(Ø"&€L�-Ó&Ù"'¸Ô"<Ð�xÓ<Ù %°cÔ :€O�XÓ:Ø€IˆtÓÙ °·±Ô@€Dˆ.×@rc   rC   c                  óX   € ] tR t^lt$ RtR]R&   R]R&   R]R&   R]R	&   R]R
&   R]R&   RtR# )Ú_ParoQuantReplayBatchzGOne CPU-owned layer replay batch used by streamed grouped optimization.úList[torch.Tensor]ÚinputsúDict[str, Any]Úinput_kwargsútorch.TensorÚtargetúOptional[torch.Tensor]Úposition_idsÚattention_maskÚintÚ	row_countrW   N)rX   rY   rZ   r[   r\   r^   rb   rW   rc   rd   rf   rf   l   s(   ‡ áQàÓØ Ó ØÓØ(Ó(Ø*Ó*Ø‡Nrc   rf   c               ó    € V ^8„  d   QhRRRR/# )é   Úvaluer   ÚreturnrT   rW   )Úformats   "rd   Ú__annotate__rw   x   s   € ÷ ñ  sð ¨tñ rc   c                óÒ  € \        V \        P                  4      '       d   V P                  4       # \        V \        4      '       dQ   \
        ;QJ d*    R V P                  4        4       F  '       g   K   R# 	  R# ! R V P                  4        4       4      # \        V \        \        34      '       d5   \
        ;QJ d    R V  4       F  '       g   K   R# 	  R# ! R V  4       4      # R# )zPDetect nested inference-mode tensors so caches can rebuild autograd-safe values.c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5i©N©Ú_value_has_inference_tensor©Ú.0Úinners   & rd   Ú	<genexpr>Ú._value_has_inference_tensor.<locals>.<genexpr>}   s   é € ÐRÁ>¸%Ô.¨u×5Ð5Ã>ùó   ‚TFc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   r{   r}   s   & rd   r€   r�      s   é € ÐIÁ5¸%Ô.¨u×5Ð5Ã5ùr‚   )	Ú
isinstanceÚtorchÚTensorÚis_inferencer]   ÚanyÚvaluesÚlistÚtuple©rt   s   &rd   r|   r|   x   s¡   € ä�%œŸ™×&Ò&Ø×!Ñ!Ó#Ð#Ü�%œ×Òß‹sÑRÀ5Ç<Á<Ä>ÓR�sŒsÐRŠsÐRˆsÑRÀ5Ç<Á<Ä>ÓRÓRÐRÜ�%œ$¤˜×'Ò'ß‹sÑIÁ5ÓI�sŒsÐIŠsÐIˆsÑIÁ5ÓIÓIÐIÙrc   c                  ó~   € ] tR t^ƒtRtRR/R R llt]R R l4       tR R	 lt]R
 R l4       t	R R lt
R R ltRtR# )Ú_LayerShardLoaderz=Stream replay batches from CPU to one device shard at a time.Úmetadata_cacheNc          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )rs   Úbatchesúlist[_ParoQuantReplayBatch]Útarget_deviceútorch.deviceÚshard_batchesrp   r�   ú-Optional[dict[tuple[int, str], torch.Tensor]]ru   ÚNonerW   )rv   s   "rd   rw   Ú_LayerShardLoader.__annotate__†   s=   € ÷ -ñ -à,ð-ð $ð	-ð
 ð-ð Fð-ð 
ñ-rc   c               	óˆ   € Wn         \        P                  ! V4      V n        \	        ^\        V4      4      V n        W@n        R# )é   N)r‘   r…   Údevicer“   Úmaxrp   r•   r�   )Úselfr‘   r“   r•   r�   s   &&$$$rd   Ú__init__Ú_LayerShardLoader.__init__†   s4   € ð ŒÜ"Ÿ\š\¨-Ó8ˆÔÜ  ¤C¨Ó$6Ó7ˆÔØ,Örc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   rt   rk   r›   r”   ru   rW   )rv   s   "rd   rw   r˜   ”   s&   € ÷ 	Bñ 	B ð 	B°|ð 	BÈñ 	Brc   c                	óÂ  € V P                   P                  \        P                  8H  ;'       d(    V P                  4       ;'       d    VP                  R 8H  pV P	                  4       '       d=   \
        P                  ! R4      ;_uu_ 4        V P                  WRR7      uuRRR4       # V P                   V8X  d   V # V P                  WR7      #   + '       g   i     L5; i)ÚcudaFT)r›   Únon_blockingÚcopyN)r›   r£   )r›   Útyper@   Ú	is_pinnedr‡   r…   Úinference_modeÚto)rt   r›   r£   s   && rd   Ú_tensor_to_deviceÚ#_LayerShardLoader._tensor_to_device“   s    € à—|‘|×(Ñ(¬C¯H©HÑ4×dÐd¸¿¹Ó9J×dÐdÈvÏ{É{Ð^dÑOdˆØ×Ñ×Òô ×%Ò% e×,Õ,Ø—x‘x vÈt�xÓT÷ -Ò,à�<‰<˜6Ô!ØˆLØ�x‰x˜vˆxÓAÐA÷	 -×,ús   ÂCÃC	c               ó    € V ^8„  d   QhRRRR/# )rs   rt   rm   ru   rW   )rv   s   "rd   rw   r˜   Ÿ   s   € ÷ ñ Ð0Fð ÐKañ rc   c                	óv  € Vf   R # VP                   V P                  8X  d   VP                  4       '       g   V# V P                  pVf   V P	                  WP                  4      # \        V4      \        V P                  4      3pVP                  V4      pVf    V P	                  WP                  4      pWBV&   V# rz   )r›   r“   r‡   r�   r©   ÚidÚstrÚget)r�   rt   ÚcacheÚ	cache_keyÚcacheds   &&   rd   Ú_metadata_tensor_to_deviceÚ,_LayerShardLoader._metadata_tensor_to_deviceŸ   s¥   € ØŠ=ÙØ�<‰<˜4×-Ñ-Ô-°e×6HÑ6H×6JÒ6JØˆLØ×#Ñ#ˆØŠ=Ø×)Ñ)¨%×1CÑ1CÓDÐDÜ˜“Y¤ D×$6Ñ$6Ó 7Ð8ˆ	Ø—‘˜9Ó%ˆØŠ>Ø×+Ñ+¨E×3EÑ3EÓFˆFØ%�)ÑØˆrc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   rt   r   r›   r”   ru   rW   )rv   s   "rd   rw   r˜   ¯   s!   € ÷ 	ñ 	 Sð 	°,ð 	À3ñ 	rc   c           	     	ó0  a€ \        V \        P                  4      '       d   \        P	                  V S4      # \        V \
        4      '       d:   V P                  4        UUu/ uF  w  r#V\        P                  VS4      bK  	  upp# \        V \        4      '       d'   V  Uu. uF  p\        P                  VS4      NK  	  up# \        V \        4      '       d3   \        ;QJ d    . V3R  lV  4       F  NK  	  5# ! V3R  lV  4       4      # V # u uppi u upi )c              3  óP   <"  € T F  p\         P                  VS4      x € K  	  R # 5irz   )rŽ   Ú_move_value_to_device)r~   r   r›   s   & €rd   r€   Ú:_LayerShardLoader._move_value_to_device.<locals>.<genexpr>·   s%   øé € ÐcÑ]bÐTYÔ*×@Ñ@ÀÈ×OÐOÓ]bùs   ƒ#&)
r„   r…   r†   rŽ   r©   r]   Úitemsr¸   rŠ   r‹   )rt   r›   Úkeyr   s   &f  rd   r¸   Ú'_LayerShardLoader._move_value_to_device®   sÜ   ø€ ä�eœUŸ\™\×*Ò*Ü$×6Ñ6°u¸fÓEÐEÜ�eœT×"Ò"Øbg×bmÑbmÔboÔpÑboÑT^ÐTW�CÔ*×@Ñ@ÀÈÓOÒOÑboÒpÐpÜ�eœT×"Ò"ÙX]Ó^ÑX]ÈuÔ%×;Ñ;¸EÀ6ÖJÑX]Ñ^Ð^Ü�eœU×#Ò#ß”5ÔcÑ]bÓc—5Ðc�5ÔcÑ]bÓcÓcÐcØˆùó qùâ^s   Á!"DÂ!Dc               ó    € V ^8„  d   QhRRRR/# )rs   Úbatchrf   ru   rW   )rv   s   "rd   rw   r˜   º   s   € ÷ 
ñ 
Ð(=ð 
ÐBWñ 
rc   c                	óè  € \        VP                   Uu. uF  q P                  W P                  4      NK  	  upVP                  P                  4        UUu/ uF   w  r4W0P                  W@P                  4      bK"  	  uppV P                  VP                  V P                  4      V P                  VP                  4      V P                  VP                  4      VP                  R 7      # u upi u uppi )©rh   rj   rl   rn   ro   rq   )rf   rh   r©   r“   rj   rº   r¸   rl   r³   rn   ro   rq   )r�   r¾   Útensorr»   rt   s   &&   rd   Ú_materialize_batchÚ$_LayerShardLoader._materialize_batchº   sÍ   € Ü$ØUZ×UaÒUaÓbÑUaÈ6×*Ñ*¨6×3EÑ3EÖFÑUaÑbð #(×"4Ñ"4×":Ñ":Ô"<ôá"<‘J�Cð ×/Ñ/°×7IÑ7IÓJÒJÙ"<òð ×)Ñ)¨%¯,©,¸×8JÑ8JÓKØ×8Ñ8¸×9KÑ9KÓLØ×:Ñ:¸5×;OÑ;OÓPØ—o‘oô

ð 
	
ùÚbùós   ”#C)Á&C.c               ó   € V ^8„  d   QhRR/# )rs   ru   z%Iterator[list[_ParoQuantReplayBatch]]rW   )rv   s   "rd   rw   r˜   Ç   s   € ÷ ñ ÐBñ rc   c              #  	ó6  "  € \        ^ \        V P                  4      V P                  4       Fa  p\	        \        V P                  4      WP                  ,           4      pV P                  W  Uu. uF  q0P                  V4      NK  	  ppVx € Kc  	  R# u upi 5i©é    N)ÚrangeÚlenr‘   r•   ÚminrÂ   )r�   ÚstartÚendr¾   Úshards   &    rd   Úiter_shardsÚ_LayerShardLoader.iter_shardsÇ   sv   é € Ü˜1œc $§,¡,Ó/°×1CÑ1CÖDˆEÜ”c˜$Ÿ,™,Ó'¨×1CÑ1CÕ)CÓDˆCØAEÇÁÈeÑAXÓYÑAX¸×,Ñ,¨UÖ3ÑAXˆEÐYØŒKó EùâYùs   ‚A-BÁ/BÂB)r‘   r�   r•   r“   )rX   rY   rZ   r[   r\   rž   Ústaticmethodr©   r³   r¸   rÂ   rÎ   rb   rW   rc   rd   rŽ   rŽ   ƒ   sT   † ÙGð-ð IM÷-ð ô	Bó ð	Bõð ô	ó ð	õ
÷ñ rc   rŽ   c                  óÜ  a € ] tR t^ÍtRtRÅR V 3R llltV 3R ltV 3R ltR tV 3R	 lt	R
 R lt
R R ltRÆR R lltR R ltR R ltR R ltR R ltR R lt]R R l4       tR R ltR R ltR  R! lt]R" R# l4       tR$ R% ltR& R' ltR( R) ltR* R+ ltR, R- ltR. R/ ltR0 R1 ltR2 R3 ltR4 R5 lt R6 R7 lt!R8 R9 lt"R: R; lt#R< R= lt$]R> R? l4       t%R@ RA lt&]RB RC l4       t']RD RE l4       t(]RF RG l4       t)]RH RI l4       t*RJ RK lt+RL RM lt,]RÆRN RO ll4       t-RP RQ lt.]RR RS l4       t/RT RU lt0RV RW lt1RXR/RY RZ llt2]R[ R\ l4       t3]R] R^ l4       t4R_ R` lt5Ra Rb lt6]Rc Rd l4       t7RXR/Re Rf llt8]Rg Rh l4       t9]Ri Rj l4       t:Rk Rl lt;]Rm Rn l4       t<Ro Rp lt=Rq Rr lt>Rs Rt lt?Ru Rv lt@Rw Rx ltARy Rz ltBR{ R| ltC]R} R~ l4       tDR R€ ltER� R‚ ltFRƒ R„ ltG]R… R† l4       tH]R‡Rˆ/R‰ RŠ ll4       tI]R‹ RŒ l4       tJ]R� RŽ l4       tKR�RR�R/R‘ R’ lltL]R“ R” l4       tMR• R– ltNR—R/R˜ R™ lltOR—R/Rš R› lltPRœ R� ltQ]Rž RŸ l4       tRR  R¡ ltS]R¢ R£ l4       tT]R¤ R¥ l4       tUR¦ R§ ltVR¨ R© ltWRÆRª R« lltXR¬ R­ ltYR® R¯ ltZRÇR° R± llt[R²RR³R/R´ Rµ llt\R¶ R· lt]R¸ R¹ lt^Rº R» lt_R¼ V 3R½ llt`R¾ R¿ lta]bRÀ RÁ l4       tcRÂ RÃ ltdRÄteV ;tf# )ÈÚParoQuantProcessorzEStandalone ParoQuant lifecycle: capture, optimize, export, then pack.TNc               ó4   € V ^8„  d   QhRRRRRRRRR	R
RR
RR/# )rs   Úqcfgr%   Úcalibration_concat_sizerP   Úcalibration_sortúOptional[str]Ú
batch_sizerp   Úrequire_fwdrT   Úcalculate_w_wq_diffÚcalibration_concat_separatorrW   )rv   s   "rd   rw   ÚParoQuantProcessor.__annotate__Ð   sR   € ÷ 0&ñ 0&ð ð0&ð "/ð0&ð (ð0&ð ð0&ð ð0&ð "ð0&ð '4ñ0&rc   c                óJ  <€ \        \        VRR4      4      P                  4       P                  4       R8g  p\        SV `  VVVVVVVV\        V
RRRVR7      R7	       W°n        . V n        W€n	        W�n
        \        VP                  VP                  4      V n        V P                  V P                  4      V n        / V n        \"        P$                  ! 4       V n        \"        P$                  ! 4       V n        / V n        RV n        RV n        RV n        VP2                  V n        R# )zHConfigure a looper that captures activations and quantizes after replay.Ú	opt_scopeÚmoduleT)rÙ   Úfwd_replay_after_processÚsubset_forward_early_stopÚenable_activation_captureÚcapture_layer_forward_context)	Ú	tokenizerrÔ   ÚcalibrationrÕ   rÖ   rÛ   Úprepare_dataset_funcrØ   Úexecution_configNF)r®   ÚgetattrÚstripÚlowerÚsuperrž   r   rÚ   Ú
avg_lossesÚ
gptq_modelÚmodelr&   rv   ÚmethodÚ!_select_qlinear_kernel_for_formatÚqlinear_kernelÚ_layer_statesr`   ra   Ú_layer_states_lockÚ_rotary_lockÚ_rotary_cacheÚ_rotary_source_idÚ_clean_group_layer_inputsÚ_runtime_prewarmedÚfallback)r�   rä   rÔ   rå   ræ   rÕ   rÖ   rØ   rí   rî   rÙ   rÚ   rÛ   Úcapture_group_layer_contextÚ	__class__s   &&&&&&&&&&&&& €rd   rž   ÚParoQuantProcessor.__init__Ð   s	  ø€ ô  '*¬'°$¸ÀXÓ*NÓ&O×&UÑ&UÓ&W×&]Ñ&]Ó&_ÐckÑ&kÐ#Ü‰ÑØØØ#Ø$;Ø-Ø)EØ!5Ø!Ü,Ø'Ø)-Ø*.Ø*.Ø.Iôð 	ô 	
ð$ $7Ô Ø')ˆŒØ$ŒØŒ
Ü*¨4¯;©;¸¿¹ÓDˆŒØ"×DÑDÀTÇ[Á[ÓQˆÔØ>@ˆÔÜ"+§.¢.Ó"2ˆÔÜ%ŸNšNÓ,ˆÔØ35ˆÔØ04ˆÔØMQˆÔ&Ø"'ˆÔØŸ™ˆŽrc   c                	óB   <€ \         SV `  4       pV P                  VR &   V# )Úclean_group_layer_inputs)rë   Úcontinuation_state_dictr÷   ©r�   Ústaterû   s   & €rd   rÿ   Ú*ParoQuantProcessor.continuation_state_dict  s&   ø€ Ü‘Ñ/Ó1ˆØ,0×,JÑ,JˆÐ(Ñ)Øˆrc   c                	óB   <€ \         SV `  V4       VR ,          V n        R# )rþ   N)rë   Úload_continuation_state_dictr÷   r   s   &&€rd   r  Ú/ParoQuantProcessor.load_continuation_state_dict  s   ø€ Ü‰Ñ,¨UÔ3Ø).Ð/IÕ)JˆÖ&rc   c                ó   € \        R4      h)zLReject runtime dataset swaps because capture state is tied to the processor.z;ParoQuantProcessor's calibration_dataset cannot be modified)ÚNotImplementedError)r�   Úcalibration_datasets   &&rd   Úset_calibration_datasetÚ*ParoQuantProcessor.set_calibration_dataset  s   € ä!Ð"_Ó`Ð`rc   c                ó†   <€ \         SV `  V4       V P                  4       '       d   VP                  V n        R# RV n        R# )zASeed grouped calibration with the clean first-layer input stream.N)rë   Úreceive_input_cacheÚ_train_on_noisy_inputs_enabledrJ   r÷   )r�   Úinput_cacherû   s   &&€rd   r  Ú&ParoQuantProcessor.receive_input_cache  s4   ø€ ä‰Ñ# KÔ0ØEI×EhÑEh×EjÒEj¨×)AÑ)AˆÖ&ÐptˆÖ&rc   c               ó   € V ^8„  d   QhRR/# )rs   Úformat_valuer#   rW   )rv   s   "rd   rw   rÜ     s   € ÷ ñ ¸fñ rc   c                óš   € \        V\        4      '       g   \        V4      MTpV\        P                  8w  d   \        RV 24      h\        # )z>Resolve the only supported runtime kernel class for ParoQuant.z*METHOD.PARO does not support this FORMAT: )r„   r#   Ú	PAROQUANTÚ
ValueErrorr"   )r�   r  Úfmts   && rd   rð   Ú4ParoQuantProcessor._select_qlinear_kernel_for_format  sB   € ä*4°\Ä6×*JÒ*JŒf�\Ô"ÐP\ˆØ”&×"Ñ"Ô"ÜÐIÈ,ÈÐXÓYÐYÜÐrc   c               ó   € V ^8„  d   QhRR/# )rs   ru   r—   rW   )rv   s   "rd   rw   rÜ     s   € ÷ Cñ C ñ Crc   c                óª  € \        V RR4      '       d   R# RV n        \        \        V P                  RR4      4      p\	        \        V P                  R^€4      4      p\	        \        V P                  R^4      4      pV'       d$   VR9   d   VR9   d   \
        P                  R4       \        VVVR	7      '       g   R# \
        P                  R
4       R# )zMBuild optional fused ParoQuant runtime pieces before timed layer work starts.rø   FNTÚopt_fused_rotationÚ
group_sizeÚkrotz1ParoQuant: prewarming fused rotation extension...)Úfused_rotationr  r  z.ParoQuant: prewarmed fused rotation extension.>   é€   >   rš   é   )rè   rø   rT   rÔ   rp   ÚlogÚinfor?   )r�   r  r  r  s   &   rd   Úprewarm_runtimeÚ"ParoQuantProcessor.prewarm_runtime  s¨   € ä�4Ð-¨u×5Ò5Ùà"&ˆÔÜœg d§i¡iÐ1EÀuÓMÓNˆÜœ §¡¨L¸#Ó>Ó?ˆ
Ü”7˜4Ÿ9™9 f¨aÓ0Ó1ˆß˜j¨EÔ1°d¸f´nÜ�H‰HÐHÔIÜ3Ø)Ø!Ø÷
ó 
ñ
 Ü�‰ÐAÖBrc   c               ó   € V ^8„  d   QhRR/# )rs   Úmodule_namer×   rW   )rv   s   "rd   rw   rÜ   .  s   € ÷ ñ °=ñ rc   c                ó   € V'       d   V P                   P                  VRR4      MRp\        T;'       g    V P                   P                  V P                   P                  4      pV\
        P                  8w  d   \        RV R24      h\        # )zFResolve per-module dynamic overrides while enforcing ParoQuant format.rv   Nz6METHOD.PARO does not support dynamic format override `ú`.)	rÔ   Údynamic_getr&   rv   rï   r#   r  r  r"   )r�   r$  Úformat_overrideÚtarget_formats   &&  rd   Ú_resolve_qlinear_kernelÚ*ParoQuantProcessor._resolve_qlinear_kernel.  sv   € çP[˜$Ÿ)™)×/Ñ/°¸XÀtÔLÐaeˆÜ,¨_×-PÐ-PÀÇ	Á	×@PÑ@PÐRV×R[ÑR[×RbÑRbÓcˆØœF×,Ñ,Ô,ÜÐUÐVcÐUdÐdfÐgÓhÐhÜÐrc   c               ó    € V ^8„  d   QhRRRR/# )rs   Úlayer_indexrp   ru   rC   rW   )rv   s   "rd   rw   rÜ   6  s   € ÷ ñ ¨Cð Ð4Hñ rc   c                óÖ   € V P                   ;_uu_ 4        V P                  P                  V4      pVf   \        4       pW P                  V&   RRR4       V#   + '       g   i     X# ; i)zBFetch or create the shared state bucket for one transformer layer.N)ró   rò   r¯   rC   )r�   r-  r  s   && rd   Ú_get_layer_stateÚ#ParoQuantProcessor._get_layer_state6  s[   € à×$×$Ó$Ø×&Ñ&×*Ñ*¨;Ó7ˆEØŠ}Ü,Ó.�Ø27×"Ñ" ;Ñ/÷	 %ð
 ˆ÷ %Ö$ð
 ˆús   •8AÁA(	c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r$  r®   Úfeaturerk   ru   r—   rW   )rv   s   "rd   rw   rÜ   ?  s"   € ÷ ;ñ ;°ð ;¸|ð ;ÐPTñ ;rc   c                óN  € VP                  4       ^8:  d   VP                  ^ 4      pVP                  P                  R8w  d    VP	                  4       P                  4       pMVP	                  4       pV P                  ;_uu_ 4        V P                  P                  V4      pVf   R. /pW0P                  V&   V P                  4       pVP                  R. 4      P                  V4       VP                  R. 4      P                  V4       RRR4       R#   + '       g   i     R# ; i)z>Store one batch of calibration activations for a named module.ÚcpuNrh   Úinput_batch_indices)ÚdimÚ	unsqueezer›   r¥   Údetachr4  rV   Útasksr¯   Úcurrent_batch_indexÚ
setdefaultÚappend)r�   r$  r2  ÚentryÚbatch_indexs   &&&  rd   Ú_record_input_featureÚ(ParoQuantProcessor._record_input_feature?  sÙ   € à�;‰;‹=˜AÔØ×'Ñ'¨Ó*ˆGà�>‰>×Ñ %Ô'Ø—n‘nÓ&×*Ñ*Ó,‰Gà—n‘nÓ&ˆGà�Y�Y‹YØ—J‘J—N‘N ;Ó/ˆEØŠ}Ø! 2˜�Ø*/—
‘
˜;Ñ'Ø×2Ñ2Ó4ˆKØ×ÑÐ2°BÓ7×>Ñ>¸{ÔKØ×Ñ˜X rÓ*×1Ñ1°'Ô:÷ �Y�YŠYús   ÂBDÄD$	c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r$  r®   r-  rp   ru   r—   rW   )rv   s   "rd   rw   rÜ   R  s!   € ÷ 
+ñ 
+¨sð 
+Àð 
+Èñ 
+rc   c                ó2  € V P                   ;_uu_ 4        V P                  P                  V4      pVe   VP                  R4      V8w  d   R. RV/V P                  V&    RRR4       R# VP                  R. 4       RRR4       R#   + '       g   i     R# ; i)zOReset repeated relative module names when quantization advances to a new layer.Nr-  rh   )rV   r9  r¯   r;  )r�   r$  r-  r=  s   &&& rd   Ú_ensure_task_bucketÚ&ParoQuantProcessor._ensure_task_bucketR  sv   € à�Y�Y‹YØ—J‘J—N‘N ;Ó/ˆEØŠ} §	¡	¨-Ó 8¸KÔ Gà˜bØ! ;ð+�—
‘
˜;Ñ'ð ÷ ‰Yð ×Ñ˜X rÔ*÷ �Y�YŠYús   •A
BÁ)BÂB	c               ó    € V ^8„  d   QhRRRR/# )rs   r  rC   ru   zDict[str, torch.Tensor]rW   )rv   s   "rd   rw   rÜ   ^  s   € ÷ ñ Ð+?ð ÐD[ñ rc   c                óÐ  € / p\        VP                  4       EF%  pV P                  P                  V4      ;'       g    / pVP                  R. 4      pVP                  R. 4      p\	        V4      \	        V4      8X  de   \
        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       d+   \        \        We4      R R7       UUu. uF  w  rxVNK	  	  pppV'       g   \        P                  ! ^ 4      W#&   Kì   \        P                  ! V^ R7      W#&   W#,          .VR&   VP                  RR	4       EK(  	  V# u uppi   \         d    T^ ,          Y#&    EKL  i ; i)
zIMaterialize concatenated calibration features for all modules in a layer.rh   r5  c              3  ó(   "  € T F  qR Jx € K
  	  R # 5irz   rW   )r~   Úindexs   & rd   r€   Ú;ParoQuantProcessor._layer_input_features.<locals>.<genexpr>e  s   é € Ð3[ÑSZÈ%ÀÕ4EÓSZùs   ‚FTc                ó   € V ^ ,          # )rÇ   rW   )Úitems   &rd   Ú<lambda>Ú:ParoQuantProcessor._layer_input_features.<locals>.<lambda>f  s   € ÐbfÐghÖbirc   ©r»   )r6  N)rŠ   rE   r9  r¯   rÉ   ÚallÚsortedÚzipr…   ÚemptyÚcatÚpopÚRuntimeError)	r�   r  ÚfeaturesÚnamer=  ÚtensorsÚindicesÚ_rÁ   s	   &&       rd   Ú_layer_input_featuresÚ(ParoQuantProcessor._layer_input_features^  s  € à,.ˆÜ˜Ÿ™×'ˆDØ—J‘J—N‘N 4Ó(×.Ð.¨BˆEØ*/¯)©)°H¸bÓ*AˆGØ—i‘iÐ 5°rÓ:ˆGÜ�7‹|œs 7›|Ô+·³Ñ3[ÑSZÓ3[··²Ñ3[ÑSZÓ3[×0[Ò0[Ü39¼#¸gÓ:OÑUiÕ3jÔkÑ3j¡i a›6Ñ3j�ÑkßÜ!&§¢¨Q£�‘Ùð,Ü!&§¢¨7¸Ô!:�‘Ø#+¥>Ð"2��h‘Ø—	‘	Ð/°×6ñ (ð ˆùó løô  ô ,Ø!(¨¥�•ð,ús   ÃEÄ8E
Å
E%Å$E%c               ó    € V ^8„  d   QhRRRR/# )rs   r$  r®   ru   ztuple[int, int, bool]rW   )rv   s   "rd   rw   rÜ   r  s   € ÷ %ñ %°ð %Ð8Mñ %rc   c                óf  € \        V P                  P                  VRV P                  P                  4      4      p\        V P                  P                  VRV P                  P                  4      4      p\        V P                  P                  VRV P                  P                  4      4      pW#V3# )zBRead effective bit-width, group size, and symmetry for one module.Úbitsr  Úsym)rp   rÔ   r'  Úruntime_bitsr  rT   r`  )r�   r$  r_  r  r`  s   &&   rd   Ú_module_quant_paramsÚ'ParoQuantProcessor._module_quant_paramsr  s€   € ä�4—9‘9×(Ñ(¨°f¸d¿i¹i×>TÑ>TÓUÓVˆÜ˜Ÿ™×.Ñ.¨{¸LÈ$Ï)É)×J^ÑJ^Ó_Ó`ˆ
Ü�4—9‘9×(Ñ(¨°e¸T¿Y¹Y¿]¹]ÓKÓLˆØ Ð$Ð$rc   c               ó    € V ^8„  d   QhRRRR/# )rs   rß   r   ru   rk   rW   )rv   s   "rd   rw   rÜ   z  s   € ÷ ñ  kð °lñ rc   c                óº   € V P                   P                  pVP                  4       ^8w  d0   \        R\	        VP
                  4       RV P                   R24      hV# )z@Return the 2D weight matrix expected by the ParoQuant optimizer.z7ParoQuant currently expects rank-2 module weights, got z for `r&  )ÚweightÚdatar6  r  r‹   ÚshapeÚ	full_name)rß   rf  s   & rd   Ú_module_weight_matrixÚ(ParoQuantProcessor._module_weight_matrixy  s^   € ð —‘×#Ñ#ˆØ�:‰:‹<˜1ÔÜØIÌ%ÐPV×P\ÑP\ÓJ]ÐI^Ð^dÐek×euÑeuÐdvÐvxÐyóð ð ˆrc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   rß   r   Úoriginal_weightrk   ru   r—   rW   )rv   s   "rd   rw   rÜ   ƒ  s#   € ÷  +ñ  +°ð  +ÐWcð  +Ðhlñ  +rc   c                óz  € V P                  V4      pVP                  P                  VP                  VP                  R7      pVP
                  P                  VP                  R7      P                  4       pVP                  P                  VP                  R7      P                  4       pVP                  P                  4       pVP                  P                  \        P                  R7      P                  4       p	VP                  P                  VP                  R7      P                  4       p
VP                  P                  VP                  R7      P                  4       pV P                  ;_uu_ 4        VP                  P!                  RVRVRVRV	RV
RV/4       R	R	R	4       V P"                  '       dË   VP                  \        P$                  8X  d
   W5,
          pMEVP                  \        P&                  R7      VP                  \        P&                  R7      ,
          pV P                  ;_uu_ 4        WÁP                  R
&   VP)                  4       P                  \*        RR7      VP                  R&   R	R	R	4       WQP,                  n        R	#   + '       g   i     Lþ; i  + '       g   i     L5; i)zLStore one optimization result into the wrapped module and its scratch state.©r›   Údtype)rp  Úpack_weightÚq_scalesÚq_zerosÚpairsÚthetaÚchannel_scalesNÚ	w_wq_diffT©r›   r¤   Úwq)rj  Úpseudo_weightr¨   r›   rp  rq  r4  rr  rs  rt  r…   Úint16ru  rv  rV   r  ÚupdaterÚ   Úfloat16Úfloat32r8  r@   rf  rg  )r�   rß   Úresultrm  rf  rz  rq  rr  rs  rt  ru  rv  rw  s   &&&&         rd   Ú_apply_optimization_resultÚ-ParoQuantProcessor._apply_optimization_resultƒ  só  € à×+Ñ+¨FÓ3ˆØ×,Ñ,×/Ñ/°v·}±}ÈFÏLÉLÐ/ÓYˆØ×(Ñ(×+Ñ+°&·,±,Ð+Ó?×CÑCÓEˆØ—?‘?×%Ñ%¨F¯L©LÐ%Ó9×=Ñ=Ó?ˆØ—.‘.×$Ñ$Ó&ˆØ—‘—‘¤e§k¡k�Ó2×6Ñ6Ó8ˆØ—‘—‘ f§l¡l�Ó3×7Ñ7Ó9ˆØ×.Ñ.×1Ñ1¸¿¹Ð1ÓE×IÑIÓKˆà�Y�Y‹YØ�L‰L×Ñà! ;Ø Ø˜wØ˜UØ˜UØ$ nðô	÷ ð ×#×#Ð#Ø×$Ñ$¬¯©Ô5Ø+Õ;‘	à+×.Ñ.´U·]±]Ð.ÓCÀm×FVÑFVÔ]b×]jÑ]jÐFVÓFkÕk�	Ø——“Ø,5—‘˜[Ñ)Ø%2×%9Ñ%9Ó%;×%>Ñ%>ÄcÐPTÐ%>Ó%U�—‘˜TÑ"÷ ð +�‰Ö÷+ �Yú÷" —ús   Å9(JÈ<AJ*ÊJ'	Ê*J:	c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
rs   rß   r   Úfeatrk   Úval_lossÚfloatÚdurationru   r—   rW   )rv   s   "rd   rw   rÜ   ¥  s2   € ÷ ñ ¨ð ¸<ð ÐSXð Ðdið Ðnrñ rc   c                ót  € VP                  4       ^ 8X  d   ^ M3VP                  RVP                  R,          4      P                  ^ ,          p\        V P	                  4       \
        VP                  \        VP                  \        V P                  V4      \        V P                  V4      \        VR \        V \        R\        VR \         V P#                  4       \$        V P'                  4       /pV P(                  ;_uu_ 4        V P*                  P-                  V4       V P.                  P-                  V4       RRR4       V P0                  P-                  RVP                   RVP                   24       V P2                  P-                  V4       V P5                  V4       R#   + '       g   i     Ls; i)zJAppend one quantization log row using the same format as other processors.z.10fÚ z.3fNzlayer-Ú-éÿÿÿÿ)ÚnumelÚreshaperh  r   rW  r   r-  r   r   Úmodule_feature_summaryr   Úmodule_dtype_size_summaryr   r   r    r   r   Úformatted_fwd_timer   Údevice_memory_reportrV   Ú	durationsr<  rì   Úmodule_namesr  Úlog_new_row)r�   rß   rƒ  r„  r†  Ú	n_samplesÚstats   &&&&&  rd   Ú_log_quant_resultÚ$ParoQuantProcessor._log_quant_result¥  s?  € àŸ™›¨Ô*‘A°·±¸RÀÇÁÈBÅÓ0P×0VÑ0VÐWXÕ0Yˆ	ô ˜dŸi™i›kÜ˜v×1Ñ1Ü §¡Ü! 4×#>Ñ#>¸vÓ#FÜ˜t×=Ñ=¸fÓEÜ˜x¨˜oÜ 9 +Ü˜BÜ ¨# Ü  $×"9Ñ"9Ó";Ü ×!:Ñ!:Ó!<ð
ˆð �Y�Y‹YØ�N‰N×!Ñ! (Ô+Ø�O‰O×"Ñ" 8Ô,÷ ð 	×Ñ× Ñ  6¨&×*<Ñ*<Ð)=¸Q¸v¿{¹{¸mÐ!LÔMØ�‰�‰˜Ôà×Ñ˜Ö÷ �Yús   Ä7F'Æ'F7	c               ó$   € V ^8„  d   QhRRRRRR/# )rs   rß   r   rh   rk   ru   útuple[float, float]rW   )rv   s   "rd   rw   rÜ   ¿  s(   € ÷ 52ñ 52àð52ð ð52ð 
ñ	52rc   c                óÊ  € V P                  VP                  4      w  r4pV P                  V4      p\        VRR4      e   VP                  P
                  MRpVP                  4       P                  4       pVP                  4       ^ 8X  dA   \        P                  ! ^ VP                  ^,          3VP                  VP                  R7      pV P                  VP                  VP                  4      p	\        P                   ! R4      ;_uu_ 4        \        P"                  ! 4       ;_uu_ 4        \%        R5/ RVbRVbRVbRVbRVbR	VbR
V P&                  P(                  bRV P&                  P*                  bRV P&                  P,                  bRV P&                  P.                  bRV P&                  P0                  bRV P&                  P2                  bRV P&                  P4                  bRV P&                  P6                  bRV P&                  P8                  bRV P&                  P:                  bRV	bR\        V P&                  RR4      bR\=        \        V P&                  RR4      4      bR\?        \        V P&                  RR64      4      bR\=        \        V P&                  RR4      4      bR \A        \        V P&                  R!R4      4      bR"\=        \        V P&                  R#R$4      4      bR%\=        \        V P&                  R&R$4      4      bR'\A        \        V P&                  R(R4      4      bR)V P&                  PB                  bR*\A        \        V P&                  R+R4      4      bR,V PE                  4       bR-\        V P&                  R.R/4      bR0V P&                  PF                  bR1V P&                  PH                  bR2V P&                  PJ                  bR3V P&                  PL                  bR4V P&                  PN                  b p
RRR4       RRR4       V PQ                  VX
V4       V
PR                  V
PT                  3#   + '       g   i     LC; i  + '       g   i     LN; i)7zKOptimize one module and stash its packed runtime tensors in `module.state`.ÚbiasN)rp  r›   Frf  rh   r_  r  r`  r  Ú
pair_ratioÚ
train_rowsÚval_rowsrØ   Úrotation_epochsÚfinetune_epochsÚrotation_lrÚ	weight_lrÚquantizer_lrÚseedÚoptimizer_nameÚopt_optimizerÚadamwÚoptimizer_weight_decayÚopt_weight_decayç{®Gáz„?Úoptimizer_betasÚ	opt_betasÚoptimizer_epsÚopt_epsç»½×Ùß|Û=Úoptimizer_amsgradÚopt_amsgradÚsgd_momentumÚopt_sgd_momentumç        Úsgd_dampeningÚopt_sgd_dampeningÚsgd_nesterovÚopt_sgd_nesterovr  Úgradient_checkpointingÚopt_gradient_checkpointingÚstage_cudagraphÚbest_state_dtypeÚopt_best_state_dtypeÚfp32Ú
stage_implÚ	pair_implÚquantizer_implÚscale_clamp_minÚscale_clamp_maxrW   ©gÍÌÌÌÌÌì?gffffffî?)+rb  ri  rj  rè   r›  rg  r8  Úcloner‹  r…   rR  rh  rp  r›   Ú_module_seedr-  r§   Úenable_gradr3   rÔ   r  Úopt_pair_ratioÚopt_train_samplesÚopt_validation_samplesÚopt_batch_sizeÚopt_rotation_epochsÚopt_finetune_epochsÚopt_rotation_lrÚopt_weight_lrÚopt_quantizer_lrr…  r‹   rT   r  Ú%_module_scope_stage_cudagraph_enabledÚopt_stage_implÚopt_pair_implÚopt_quantizer_implÚopt_channel_scale_clamp_minÚopt_channel_scale_clamp_maxr€  Ú
train_lossr„  )r�   rß   rh   r_  r  r`  rf  r›  rm  Úmodule_seedr  s   &&&        rd   Ú_quantize_one_moduleÚ'ParoQuantProcessor._quantize_one_module¿  sÔ  € ð !%× 9Ñ 9¸&×:JÑ:JÓ KÑˆ˜#Ø×+Ñ+¨FÓ3ˆÜ#*¨6°6¸4Ó#@Ò#Lˆv�{‰{×ÒÐRVˆØ Ÿ-™-›/×/Ñ/Ó1ˆØ�<‰<‹>˜QÔÜ—[’[ ! V§\¡\°!¥_Ð!5¸V¿\¹\ÐRX×R_ÑR_Ô`ˆFØ×'Ñ'¨×(:Ñ(:¸F×<LÑ<LÓMˆä×!Ò! %×(Õ(¬%×*;Ò*;×*=Õ*=Ü.ò #Ùð#áð#ñ ð#ñ ð	#ñ
 &ð#ñ ð#ð —Y‘Y—^’^ð#ð  Ÿ9™9×3Ò3ð#ð  Ÿ9™9×6Ò6ð#ð Ÿ™×9Ò9ð#ð  Ÿ9™9×3Ò3ð#ð !%§	¡	× =Ò =ð#ð !%§	¡	× =Ò =ð#ð !ŸI™I×5Ò5ð#ð Ÿ)™)×1Ò1ð#ð  "ŸY™Y×7Ò7ð!#ñ" !ð##ô$  ' t§y¡y°/À7ÔKð%#ô& (-¬W°T·Y±YÐ@RÐTXÓ-YÔ'Zð'#ô( !&¤g¨d¯i©i¸ÀkÓ&RÔ Sð)#ô* $¤G¨D¯I©I°yÀ%Ó$HÔIð+#ô, #'¤w¨t¯y©y¸-ÈÓ'OÔ"Pð-#ô. #¤7¨4¯9©9Ð6HÈ#Ó#NÔOð/#ô0 $¤G¨D¯I©IÐ7JÈCÓ$PÔQð1#ô2 "¤'¨$¯)©)Ð5GÈÓ"OÔPð3#ð4  $Ÿy™y×;Ò;ð5#ô6 (,¬G°D·I±IÐ?[Ð]bÓ,cÔ'dð7#ð8 !%× JÑ JÔ Lð9#ô: ")¨¯©Ð4JÈFÔ!Sð;#ð<  Ÿ9™9×3Ò3ð=#ð> Ÿ)™)×1Ò1ð?#ð@  $Ÿy™y×;Ò;ðA#ðB !%§	¡	× EÒ EðC#ðD !%§	¡	× EÒ EðE#ˆF÷ +>×(ðL 	×'Ñ'¨°¸ÔHØ× Ñ  &§/¡/Ð1Ð1÷O +>×*=ú×(×(ús%   ÄQÄ,KP?	ÐQÐ?QÑ
QÑQ"	c               ó    € V ^8„  d   QhRRRR/# ©rs   ri  r®   ru   rW   )rv   s   "rd   rw   rÜ   ÷  s   € ÷ ,ñ , Sð ,¨Sñ ,rc   c                ó4   € V P                  R^4      R,          # )zBUse the terminal module name as the shared seed key across layers.Ú.rŠ  ©Úrsplit)ri  s   &rd   Ú_module_archetypeÚ$ParoQuantProcessor._module_archetypeö  s   € ð ×Ñ  QÓ'¨Õ+Ð+rc   c               ó    € V ^8„  d   QhRRRR/# rÜ  rW   )rv   s   "rd   rw   rÜ   û  s   € ÷ 1ñ 1¨#ð 1°#ñ 1rc   c                óR   € V P                  4       R8X  d   V# V P                  V4      # )aó  Keep module-scope seeds unique per full module while preserving grouped-scope behavior.

Module scope optimizes every linear independently across the full model.
Reusing only the terminal archetype name correlates rotations across layers
(`...layers.0.self_attn.q_proj`, `...layers.1.self_attn.q_proj`, etc.),
which materially hurts end-to-end recovery on full-model module runs.
Grouped scopes keep the existing archetype behavior to avoid disturbing the
separately tuned layer/compute-block path.
rß   )Ú_opt_scope_moderá  )r�   ri  s   &&rd   Ú_module_seed_keyÚ#ParoQuantProcessor._module_seed_keyû  s,   € ð ×ÑÓ! XÔ-ØÐØ×%Ñ% iÓ0Ð0rc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r-  rp   ri  r®   ru   rW   )rv   s   "rd   rw   rÜ   	  s&   € ÷ Eñ E¨ð E¸ð EÀñ Erc   c                ó  € V P                  V4      p\        V P                  P                  4       R\        V4       RV 2P	                  R4      p\
        P                  ! V^R7      P                  4       p\        P                  VRRR7      # )zTDerive a deterministic per-module seed from base seed, layer index, and module name.Ú:zutf-8)Údigest_sizeÚbigF)Ú	byteorderÚsigned)	ræ  rp   rÔ   Úopt_seedÚencodeÚhashlibÚblake2bÚdigestÚ
from_bytes)r�   r-  ri  r$  Úseed_materialró  s   &&&   rd   rÆ  ÚParoQuantProcessor._module_seed	  sx   € à×+Ñ+¨IÓ6ˆÜ˜tŸy™y×1Ñ1Ó2Ð3°1´S¸Ó5EÐ4FÀaÈÀ}ÐU×\Ñ\Ð]dÓeˆÜ—’ ¸AÔ>×EÑEÓGˆÜ�~‰~˜f°¸eˆ~ÓDÐDrc   c               ó   € V ^8„  d   QhRR/# ©rs   ru   r®   rW   )rv   s   "rd   rw   rÜ     s   € ÷ Nñ N ñ Nrc   c                óz   € \        \        V P                  RR4      4      P                  4       P	                  4       # )z6Normalize the configured ParoQuant optimization scope.rÞ   rß   )r®   rè   rÔ   ré   rê   ©r�   s   &rd   rå  Ú"ParoQuantProcessor._opt_scope_mode  s,   € ä”7˜4Ÿ9™9 k°8Ó<Ó=×CÑCÓE×KÑKÓMÐMrc   c               ó   € V ^8„  d   QhRR/# ©rs   ru   rT   rW   )rv   s   "rd   rw   rÜ     s   € ÷  ñ  °ñ  rc   c                ót   € \        V P                  RR4      pVf   V P                  4       R8H  # \        V4      # )zPResolve grouped-stage checkpointing from config, defaulting to layer scope only.rº  NÚlayer)rè   rÔ   rå  rT   )r�   Ú
configureds   & rd   Ú_gradient_checkpointing_enabledÚ2ParoQuantProcessor._gradient_checkpointing_enabled  s:   € ä˜TŸY™YÐ(DÀdÓKˆ
ØÒØ×'Ñ'Ó)¨WÑ4Ð4Ü�JÓÐrc   c               ó   € V ^8„  d   QhRR/# rý  rW   )rv   s   "rd   rw   rÜ     s   € ÷ 2ñ 2¨4ñ 2rc   c                ó(   € V P                  4       R8g  # )zeReturn whether this layer should optimize compute_block/layer scopes instead of one linear at a time.rß   ©rå  rú  s   &rd   Úuses_grouped_optimizationÚ,ParoQuantProcessor.uses_grouped_optimization  s   € à×#Ñ#Ó%¨Ñ1Ð1rc   c               ó   € V ^8„  d   QhRR/# rý  rW   )rv   s   "rd   rw   rÜ     s   € ÷ 1ñ 1¨Dñ 1rc   c                ó(   € V P                  4       R8g  # )ae  Whether stage-layer orchestration must build a separate pristine layer copy.

Whole-layer scope replays clean targets before mutating the live layer, so
it can use the untouched live layer directly. ComputeBlock scope still needs
a dedicated pristine copy because later grouped clone construction may
happen after subset-time hooks and wrapper replacement.
rÿ  r  rú  s   &rd   Úneeds_pristine_layer_cloneÚ-ParoQuantProcessor.needs_pristine_layer_clone  s   € ð ×#Ñ#Ó%¨Ñ0Ð0rc   c               ó   € V ^8„  d   QhRR/# rý  rW   )rv   s   "rd   rw   rÜ   )  s   € ÷ ñ ¸Tñ rc   c                ó   € R# )zJParoQuant captures grouped pristine layer IO outside subset forwards only.FrW   rú  s   &rd   Ú+capture_layer_forward_context_during_subsetÚ>ParoQuantProcessor.capture_layer_forward_context_during_subset)  s   € árc   c               ó   € V ^8„  d   QhRR/# rý  rW   )rv   s   "rd   rw   rÜ   -  s   € ÷ lñ l°ñ lrc   c                óx   € \        V RR4      p\        \        VRR4      4      ;'       d    V P                  4       # )zVEnable official clean-target / noisy-input calibration only when explicitly requested.rÔ   NÚopt_train_on_noisy_inputsF)rè   rT   r  ©r�   rÔ   s   & rd   r  Ú1ParoQuantProcessor._train_on_noisy_inputs_enabled-  s6   € ä�t˜V TÓ*ˆÜ”G˜DÐ"=¸uÓEÓF×kÐkÈ4×KiÑKiÓKkÐkrc   c               ó   € V ^8„  d   QhRR/# rý  rW   )rv   s   "rd   rw   rÜ   2  s   € ÷ 
Fñ 
F°tñ 
Frc   c                óp   € V P                  4       R8X  d   R# \        \        V P                  RR4      4      # )aW  Disable per-linear stage CUDA graphs in the full quantization loop.

Module scope calls the ParoQuant optimizer once per linear across the
whole model. The captured train-step graph keeps CUDA graph private-pool
allocations alive across calls, which makes active VRAM ratchet upward
layer by layer. Grouped/layer scope does not use this path.
rß   FÚopt_stage_cudagraph)rå  rT   rè   rÔ   rú  s   &rd   rÑ  Ú8ParoQuantProcessor._module_scope_stage_cudagraph_enabled2  s1   € ð ×ÑÓ! XÔ-ÙÜ”G˜DŸI™IÐ'<¸eÓDÓEÐErc   c               ó   € V ^8„  d   QhRR/# )rs   ru   zDict[str, object]rW   )rv   s   "rd   rw   rÜ   >  s   € ÷ 
ñ 
Ð/@ñ 
rc   c                óT  € \        V RR4      pR\        \        VRR4      4      R\        \        VRR4      4      R\        \        VR	R
4      4      R\        \        VRR4      4      R\        \        VRR4      4      R\        \        VRR4      4      R\        \        VRR4      4      /# )zIReturn shared optimizer hyperparameters for ParoQuant stage param groups.rÔ   NÚweight_decayr©  rª  Úbetasr¬  Úepsr®  r¯  Úamsgradr±  FÚmomentumr³  r´  Ú	dampeningr¶  Únesterovr¸  rÄ  )rè   r…  r‹   rT   r  s   & rd   Ú_optimizer_param_group_kwargsÚ0ParoQuantProcessor._optimizer_param_group_kwargs>  s£   € ä�t˜V TÓ*ˆàœE¤'¨$Ð0BÀDÓ"IÓJØ”Uœ7 4¨°kÓBÓCØ”5œ  y°%Ó8Ó9Ø”tœG D¨-¸Ó?Ó@Øœœg dÐ,>ÀÓDÓEØœœw tÐ-@À#ÓFÓGØœœW TÐ+=¸uÓEÓFð
ð 	
rc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r-  rp   rJ   úList[List[torch.Tensor]]ru   rW   )rv   s   "rd   rw   rÜ   K  s/   € ÷ 
Pñ 
Pð ð
Pð /ð	
Pð
 
"ñ
Prc   c               ób   € ?V P                  4       '       g   V# \        V RR4      ;'       g    T# )zHReturn the clean calibration stream used to build grouped layer targets.r÷   N)r  rè   ©r�   r-  rJ   s   &$$rd   rþ   Ú+ParoQuantProcessor.clean_group_layer_inputsK  s4   € ð Ø×2Ñ2×4Ò4ØÐÜ�tÐ8¸$Ó?×OÐOÀ<ÐOrc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r-  rp   rJ   r%  ru   r—   rW   )rv   s   "rd   rw   rÜ   W  s*   € ÷ 	:ñ 	:ð ð	:ð /ð		:ð
 
ñ	:rc   c               óD   € ?V P                  4       '       d	   W n        R# R# )zRAdvance the clean calibration stream for later-layer train-on-noisy-inputs replay.N)r  r÷   r'  s   &$$rd   Úreceive_clean_layer_inputsÚ-ParoQuantProcessor.receive_clean_layer_inputsW  s!   € ð Ø×.Ñ.×0Ò0Ø-9Ö*ñ 1rc   c               ó    € V ^8„  d   QhRRRR/# )rs   rß   r   ru   r'   rW   )rv   s   "rd   rw   rÜ   b  s   € ÷ -8ñ -8°ð -8Ð@Uñ -8rc   c                ób  € V P                  VP                  4      w  r#pV P                  V4      p\        VRR4      e   VP                  P
                  MRp\        W5P                  ^,          4      p\        V P                  P                  RR7      p\        W@P                  P                  4      p	\        V P                  P                  4       V P                  VP                  VP                  4      p
VR8X  dX   \!        VP                  ^,          VV P                  P"                  V P                  P$                  V
VP&                  R7      w  r¼MV\)        VP                  ^,          VV P                  P"                  V P                  P$                  V
VP&                  R7      w  r¼\+        VP-                  4       P/                  VP&                  \0        P2                  R7      Vf   RM8VP-                  4       P/                  VP&                  \0        P2                  R7      TTT	TTV P                  P4                  V P                  P6                  V P9                  4       R8X  d   R	MV P                  P:                  R
7
      P/                  VP&                  \0        P2                  R7      # )zOMaterialize one ParoQuant optimizer wrapper from the current live module state.r›  NrÀ  ©r   Ú	reference)Úin_featuresr  r  rœ  r¤  r›   ro  rÿ  F)r_  r  Úquantizer_symrt  Ú
theta_maskrÂ  rÃ  r  )rb  ri  rj  rè   r›  rg  r)   rh  r*   rÔ   rÓ  r-   rÔ  r,   rÆ  r-  r2   r  rÈ  r›   r1   r'   r8  r¨   r…   r~  rÕ  rÖ  rå  r  )r�   rß   r_  r  r`  rf  r›  Únormalized_group_sizeÚnormalized_pair_implr2  rØ  rt  r3  s   &&           rd   Ú_build_group_optim_linearÚ,ParoQuantProcessor._build_group_optim_linearb  sþ  € à $× 9Ñ 9¸&×:JÑ:JÓ KÑˆ˜#Ø×+Ñ+¨FÓ3ˆÜ#*¨6°6¸4Ó#@Ò#Lˆv�{‰{×ÒÐRVˆÜ 5°jÇ,Á,ÈqÅ/Ó RÐÜ2°4·9±9×3JÑ3JÐR]Ô^ÐÜ/°·Y±Y×5QÑ5QÓRˆÜ! $§)¡)×">Ñ">Ô?Ø×'Ñ'¨×(:Ñ(:¸F×<LÑ<LÓMˆà ;Ô.Ü GØ"ŸL™L¨�OØ0Ø—Y‘Y—^‘^ØŸ9™9×3Ñ3Ø Ø—}‘}ô!ÑˆE�:ô !>Ø"ŸL™L¨�OØ0Ø—Y‘Y—^‘^ØŸ9™9×3Ñ3Ø Ø—}‘}ô!ÑˆEô %Ø�M‰M‹O×Ñ f§m¡m¼5¿=¹=ÐÓIØ’L‰D d§k¡k£m×&6Ñ&6¸f¿m¹mÔSX×S`ÑS`Ð&6Ó&aØØ,Ø'ØØ!Ø ŸI™I×AÑAØ ŸI™I×AÑAð
 %)×$8Ñ$8Ó$:¸gÔ$E™5È4Ï9É9×KgÑKgô
÷ ‰"�F—M‘M¬¯©ˆ"Ó
7ð	8rc   c               ó    € V ^8„  d   QhRRRR/# )rs   rß   r!   ru   ztorch.nn.LinearrW   )rv   s   "rd   rw   rÜ   ’  s   € ÷ ñ ¨Lð ¸_ñ rc   c                ób  € \         P                  P                  P                  \         P                  P                  4      p\         P                  P                  P                  V4       V P                  Vn        V P                  Vn        V P                  Vn        V P                  Vn	        V# )zYDrop HookedLinear's inference-only forward while preserving the shared parameter storage.)
r…   r   ÚLinearÚ__new__r   rž   r1  Úout_featuresrf  r›  )rß   Úrestoreds   & rd   Ú_restore_linear_from_hookedÚ.ParoQuantProcessor._restore_linear_from_hooked‘  sq   € ô —8‘8—?‘?×*Ñ*¬5¯8©8¯?©?Ó;ˆÜ�‰�‰× Ñ  Ô*Ø%×1Ñ1ˆÔØ &× 3Ñ 3ˆÔØ Ÿ-™-ˆŒØŸ™ˆŒØˆrc   c               ó    € V ^8„  d   QhRRRR/# )rs   rß   útorch.nn.Moduleru   rp   rW   )rv   s   "rd   rw   rÜ   œ  s   € ÷ 	ñ 	°Oð 	Èñ 	rc   c                óô   € ^ p\        VP                  4       4       FX  w  r4\        V\        4      '       d'   \	        WV P                  V4      4       V^,          pKA  W P                  V4      ,          pKZ  	  V# )z`Layer-scope training must not run through HookedLinear, which always forwards in inference mode.)rŠ   Únamed_childrenr„   r!   Úsetattrr>  Ú_strip_hooked_linear_wrappers)r�   rß   ÚreplacedÚ
child_nameÚchilds   &&   rd   rE  Ú0ParoQuantProcessor._strip_hooked_linear_wrappersœ  sj   € àˆÜ!% f×&;Ñ&;Ó&=Ö!>ÑˆJÜ˜%¤×.Ò.Ü˜¨D×,LÑ,LÈUÓ,SÔTØ˜A•�ÙØ×:Ñ:¸5ÓAÕAŠHñ "?ð ˆrc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   rÿ  rA  Únamed_modulesúlist[NamedModule]ru   r—   rW   )rv   s   "rd   rw   rÜ   ¨  s&   € ÷ ñ Øðà(ðð 
ñrc   c                óÐ   € V FG  p\        WP                  4      pW2n         \        VP	                  4       4      P
                  Vn        KI  	  R#   \        \        3 d     Ka  i ; i)a“  Retarget NamedModule handles after live-layer wrapper surgery.

Layer scope temporarily unwraps HookedLinear modules into plain
``nn.Linear`` so autograd can flow through the real decoder layer. The
corresponding ``NamedModule`` wrappers must follow those in-place
replacements; otherwise later result application and packing will update
detached wrapper objects instead of the real live layer modules.
N)	r<   rW  rß   ÚnextÚ
parametersrp  Úmodule_dtypeÚStopIterationÚAttributeError)rÿ  rK  Únamed_moduleÚlive_modules   &&  rd   Ú!_sync_named_modules_to_live_layerÚ4ParoQuantProcessor._sync_named_modules_to_live_layer§  s^   € ó *ˆLÜ)¨%×1BÑ1BÓCˆKØ"-ÔðÜ,0°×1GÑ1GÓ1IÓ,J×,PÑ,P�Ö)ó	 *øô
 "¤>Ð2ô Úðús   ¢(AÁA%Á$A%c               ó    € V ^8„  d   QhRRRR/# )rs   rÿ  rA  ru   ú list[tuple[object, str, object]]rW   )rv   s   "rd   rw   rÜ   ½  s   € ÷ ñ ¨Oð Ð@`ñ rc   c                ól  € . p\        4       p\        V RR4      \        \        V RR4      RR4      .pV F|  pVf   K	  \        V4      pWR9   d   K  VP                  V4       R FI  p\	        WF4      '       g   K  \        WF4      pVR8X  d   K*  \        WFR4       VP                  WFV34       KK  	  K~  	  V# )zOTemporarily force live-layer grouped optimization onto eager attention kernels.ÚconfigNÚ	self_attnÚeager)Ú_attn_implementationÚattn_implementation)r_   rè   r­   ÚaddÚhasattrrD  r<  )rÿ  Ú	overridesÚseen_configsÚcandidate_configsrZ  Ú	config_idÚattrÚoriginal_values   &       rd   Ú_force_layer_eager_attentionÚ/ParoQuantProcessor._force_layer_eager_attention¼  sÀ   € ð 79ˆ	Ü!$£ˆä�E˜8 TÓ*Ü”G˜E ;°Ó5°xÀÓFð
Ðó (ˆFØŠ~ÙÜ˜6›
ˆIØÔ(ÙØ×Ñ˜YÔ'ÛG�Ü˜v×,Ò,ÙÜ!(¨Ó!6�Ø! WÔ,ÙÜ˜ gÔ.Ø× Ñ  &°Ð!?Ö@ó Hñ (ð Ðrc   c               ó    € V ^8„  d   QhRRRR/# )rs   ra  rX  ru   r—   rW   )rv   s   "rd   rw   rÜ   ×  s   € ÷ 2ñ 2Ð1Qð 2ÐVZñ 2rc   c                óH   € \        V 4       F  w  rp\        WV4       K  	  R# )zURestore any attention implementation overrides after live-layer grouped optimization.N)ÚreversedrD  )ra  rZ  re  rf  s   &   rd   Ú_restore_layer_attention_implÚ0ParoQuantProcessor._restore_layer_attention_implÖ  s#   € ô -5°YÖ,?Ñ(ˆF˜.Ü�F .Ö1ó -@rc   c               ó    € V ^8„  d   QhRRRR/# )rs   rÿ  rA  ru   ztuple[int, int]rW   )rv   s   "rd   rw   rÜ   Ý  s   € ÷ 1ñ 1Øð1à	ñ1rc   c           	     ó²  € ^ p^ p\         P                  ! R4      ;_uu_ 4        V P                  4        EF   p\        VP                  P                  4       4       Fg  w  rEVf   K  \        P                  ! VP                  4       P                  4       VP                  R7      pWeJg   KP  WcP                  V&   V^,          pKi  	  \        VP                  P                  4       4       FG  w  rGVf   K  VP                  4       P                  4       pWgJg   K0  WcP                  V&   V^,          pKI  	  EK  	  RRR4       W3#   + '       g   i     Y3# ; i)aß  Replace inference-born params/buffers with fresh normal tensors for live autograd.

The layer object itself stays in place, but worker-side model loading can
create parameters and buffers under ``torch.inference_mode()``. Those
tensors keep inference-only bookkeeping even after ``module.to(...)``,
which breaks backward in the in-place layer optimizer. Rebuilding them
in place gives the live layer normal autograd-safe storage without
falling back to a full cloned layer path.
FN)Úrequires_grad)r…   r§   rE   rŠ   Ú_parametersrº   r   Ú	Parameterr8  rÅ  rp  Ú_buffers)rÿ  Úreplaced_paramsÚreplaced_buffersrß   rW  ÚparamÚrebuiltÚbuffers   &       rd   Ú(_materialize_live_layer_autograd_tensorsÚ;ParoQuantProcessor._materialize_live_layer_autograd_tensorsÜ  s  € ð ˆØÐÜ×!Ò! %×(Õ(ØŸ-™-Ÿ/�Ü#'¨×(:Ñ(:×(@Ñ(@Ó(BÖ#C‘K�DØ’}Ù Ü Ÿlšl¨5¯<©<«>×+?Ñ+?Ó+AÐQV×QdÑQdÔe�GØÕ+Ø3:×*Ñ*¨4Ñ0Ø'¨1Õ,šñ $Dô %)¨¯©×)>Ñ)>Ó)@Ö$A‘L�DØ’~Ù Ø$Ÿm™m›o×3Ñ3Ó5�GØÕ,Ø07Ÿ™¨Ñ-Ø(¨AÕ-Ò(ô %Bñ *÷ )ð  Ð0Ð0÷! )Ö(ð  Ð0Ð0ús   £BEÂ+A)EÄ EÅE	c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r  rC   Úgroup_modulesrL  ru   z8tuple[torch.nn.Module, dict[str, _ParoQuantOptimLinear]]rW   )rv   s   "rd   rw   rÜ   ý  s)   € ÷ :*ñ :*à#ð:*ð )ð:*ð 
Bñ	:*rc   c                óâ  € V'       g   \        R4      h\        VRR4      pVf¸   \        VRR4      ;'       g    VP                  pVf   \        R4      h\        P
                  ! V4      P                  \        \        P                  R7      pV P                  V4       \        VRR4      p\        VRR4      pVe   \        VR	4      '       d   R
Vn        W1n        V^ ,          P                  P                  pV P!                  4       R8X  dê   \        VRR4      pVf	   / pW�n        \%        V4      p	VP'                  V	4      p
V
fQ   V	\%        \        4      8X  d   Tp
M5\        P
                  ! V4      P                  V\        P                  R7      p
W¨V	&   \        P
                  ! V
4      p\)        VP+                  4       4      P                  V8w  d"   VP                  V\        P                  R7      pM7\        P
                  ! V4      pVP                  V\        P                  R7      p/ pV F8  pV P-                  V4      p\/        W½P0                  V4       WìVP0                  &   K:  	  W¼3# )zLClone the layer and swap one selected group to ParoQuant optimizer wrappers.úEParoQuantProcessor grouped optimization requires at least one module.rH   NrG   zIParoQuantProcessor grouped optimization requires the source layer module.ro  r[  rZ  r]  r\  Úcompute_blockrI   )r  rè   rF   rU  r¤   Údeepcopyr¨   r@   r…   r~  rE  r`  r]  rH   rf  r›   rå  rI   r®   r¯   rN  rO  r6  r=   rW  )r�   r  r|  Úprepared_sourceÚsource_layerÚ
layer_attnÚlayer_attn_configr“   Údevice_cacheÚ
device_keyÚprepared_source_for_deviceÚlayer_cloneÚoptim_modulesrS  Úoptim_modules   &&&            rd   Ú_build_group_optim_layerÚ+ParoQuantProcessor._build_group_optim_layerý  s#  € ÷ ÜÐdÓeÐeä! %Ð)GÈÓNˆØÒ"Ü" 5Ð*AÀ4ÓH×^Ð^ÈE×L^ÑL^ˆLØÒ#Ü"Ð#nÓoÐoä"Ÿmšm¨LÓ9×<Ñ<ÄCÌuÏ}É}Ð<Ó]ˆOð ×.Ñ.¨Ô?ô ! °+¸tÓDˆJÜ '¨
°H¸dÓ CÐØ Ò,´Ð9JÐLb×1cÒ1cØ9@Ð!Ô6à1@Ô.à% aÕ(×/Ñ/×6Ñ6ˆØ×ÑÓ! _Ô4Ü" 5Ð*RÐTXÓYˆLØÒ#Ø!�Ø?KÔ<ä˜]Ó+ˆJØ)5×)9Ñ)9¸*Ó)EÐ&Ø)Ò1Ø¤¤S£Ô)Ø1@Ñ.ä15·²¸Ó1O×1RÑ1RÐZgÔot×o|Ño|Ð1RÓ1}Ð.Ø+E˜ZÑ(äŸ-š-Ð(BÓCˆKÜ�K×*Ñ*Ó,Ó-×4Ñ4¸ÔEØ)Ÿn™n°MÌÏÉ˜nÓW�øäŸ-š-¨Ó8ˆKØ%Ÿ.™.°ÄUÇ]Á]˜.ÓSˆKà:<ˆÛ)ˆLØ×9Ñ9¸,ÓGˆLÜ˜K×):Ñ):¸LÔIØ/;˜,×+Ñ+Ó,ñ *ð
 Ð)Ð)rc   c               ó   € V ^8„  d   QhRR/# )rs   ru   úOptional[nn.Module]rW   )rv   s   "rd   rw   rÜ   9  s   € ÷ Kñ KÐ"5ñ Krc   c                óþ   € \        V RR4      pV P                  eF   VeB   \        V P                  RR4      '       d%   \        WP                  P                  .4      w  r#V# Vf   R# \        \        VRV4      RR4      # )zKReturn the model rotary module used to refresh grouped layer replay kwargs.rî   NÚrotary_embeddingÚ
rotary_emb)rè   rí   r8   r�  )r�   rî   ÚrotaryrZ  s   &   rd   Ú_get_root_rotaryÚ#ParoQuantProcessor._get_root_rotary9  sq   € ä˜˜g tÓ,ˆØ�?‰?Ò&¨5Ò+<ÄÈÏÉÐZlÐnr×AsÒAsÜ1°%¿/¹/×:ZÑ:ZÐ9[Ó\‰IˆFØˆMØŠ=ÙÜ”w˜u g¨uÓ5°|ÀTÓJÐJrc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r’  rŽ  rù   úOptional[torch.device]ru   rW   )rv   s   "rd   rw   rÜ   D  s$   € ÷ ñ Ð#6ð ÐBXð Ðdzñ rc   c                óÎ   € V f   V# \        \        V RR4      RR4      pVe   V#  \        V P                  4       4      P                  #   \        \
        \        3 d    Tu # i ; i)zCResolve the active device for a rotary module, falling back safely.NÚinv_freqr›   )rè   rN  rO  r›   rQ  rR  rU  )r’  rù   Úrotary_devices   && rd   Ú_get_rotary_deviceÚ%ParoQuantProcessor._get_rotary_deviceC  sk   € ð Š>ØˆOä¤¨°
¸DÓ AÀ8ÈTÓRˆØÒ$Ø Ð ð	Ü˜×)Ñ)Ó+Ó,×3Ñ3Ð3øÜœ~¬|Ð<ô 	ØŠOð	ús   ¦"A	 Á	A$Á#A$c               ó    € V ^8„  d   QhRRRR/# )rs   r“   r–  ru   rŽ  rW   )rv   s   "rd   rw   rÜ   R  s   € ÷ #ñ #Ð4Jð #ÐObñ #rc   c                óÜ  € V P                  4       pVe   Vf   V# \        P                  ! V4      pV P                  V4      V8X  d   V# \	        V4      pV P
                  ;_uu_ 4        V P                  4       pVf    RRR4       R# \        V4      pV P                  V8w  d!   V P                  P                  4        W@n        V P                  V4      V8X  d   VuuRRR4       # V P                  P                  V4      pVf7    \        P                  ! V4      p\        WQR7       WRJd   WPP                  V&   VuuRRR4       #   \         d    Tp L:i ; i  + '       g   i     R# ; i)zHReturn a rotary module materialized on the requested device when needed.N©r›   )r“  r…   r›   rš  r®   rô   r­   rö   rõ   Úclearr¯   r¤   r€  Ú	Exceptionr9   )r�   r“   r’  r±   Ú	source_idr²   s   &&    rd   Ú_get_rotary_for_deviceÚ)ParoQuantProcessor._get_rotary_for_deviceR  sC  € à×&Ñ&Ó(ˆØŠ>˜]Ò2ØˆMäŸš ]Ó3ˆØ×"Ñ" 6Ó*¨mÔ;ØˆMä˜Ó&ˆ	Ø××ÓØ×*Ñ*Ó,ˆFØŠ~Ø÷ Ñô
 ˜6›
ˆIØ×%Ñ%¨Ô2Ø×"Ñ"×(Ñ(Ô*Ø)2Ô&à×&Ñ& vÓ.°-Ô?Ø÷ Òð ×'Ñ'×+Ñ+¨IÓ6ˆFØŠ~ð$Ü!Ÿ]š]¨6Ó2�Fô ˜Õ5ØÓ'Ø4:×&Ñ& yÑ1à÷1 Òøô" !ô $Ø#’Fð$ú÷# ×Ðús=   Á(EÂAEÃ&EÄEÄ EÅEÅEÅEÅEÅE+	c               ó    € V ^8„  d   QhRRRR/# )rs   r’  rŽ  ru   rT   rW   )rv   s   "rd   rw   rÜ   x  s   € ÷ ~ñ ~Ð6Ið ~Èdñ ~rc   c                ó    € V f   R# \        V 4      pVP                  P                  R4      ;'       d    VP                  P	                  R4      # )zhAllow memoized rotary embeddings only for known HF rotary classes that depend on ids, device, and dtype.Fztransformers.models.ÚRotaryEmbedding)r¥   rY   Ú
startswithrX   Úendswith)r’  Úrotary_types   & rd   Ú%_can_cache_rotary_position_embeddingsÚ8ParoQuantProcessor._can_cache_rotary_position_embeddingsw  sG   € ð Š>ÙÜ˜6“lˆØ×%Ñ%×0Ñ0Ð1GÓH×}Ð}È[×MaÑMa×MjÑMjÐk|ÓM}Ð}rc   c               ó(   € V ^8„  d   QhRRRRRRRR/# )rs   r›   r”   Ú	batch_dimrp   Úseq_lenru   rk   rW   )rv   s   "rd   rw   rÜ     s4   € ÷ #ñ #ð ð#ð ð	#ð
 ð#ð 
ñ#rc   c               óÔ  € \        V RR4      pVf	   / pW@n        \        V4      \        V4      \        V4      3pVP	                  V4      pVe   VP                  4       '       dq   \        P                  ! R4      ;_uu_ 4        \        P                  ! W1\        P                  R7      P                  ^ 4      P                  VR4      pRRR4       WdV&   V#   + '       g   i     L; i)zUReuse deterministic generated position ids for repeated grouped layer replay batches.Ú_group_position_ids_cacheNFro  rŠ  )rè   r°  r®   rp   r¯   r‡   r…   r§   ÚarangeÚlongr7  Úexpand)r�   r›   r­  r®  Úposition_ids_cacher±   Úcached_position_idss   &$$$   rd   Ú_cached_group_position_idsÚ-ParoQuantProcessor._cached_group_position_ids  sÄ   € ô % TÐ+FÈÓMÐØÒ%Ø!#ÐØ-?Ô*ä˜“[¤# i£.´#°g³,Ð?ˆ	Ø0×4Ñ4°YÓ?ÐØÒ&Ð*=×*JÑ*J×*LÒ*LÜ×%Ò% e×,Õ,Ü&+§l¢l°7ÔQV×Q[ÑQ[Ô&\×&fÑ&fÐghÓ&i×&pÑ&pÐqzÐ|~Ó&Ð#÷ -à,?˜yÑ)Ø"Ð"÷ -×,ús   ÂACÃC'	c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
rs   r’  z	nn.ModuleÚxrk   rn   r™  r–  ru   r   rW   )rv   s   "rd   rw   rÜ   ”  s>   € ÷ *ñ *ð ð*ð ð	*ð
 #ð*ð .ð*ð 
ñ*rc   c               óø  € T;'       g    VP                   p\        V RR4      pVf	   / pW`n        \        V4      \        V4      \	        V4      \	        VP
                  4      3pVP                  V4      pVe   \        V4      '       d^   \        P                  W%4      p	\        P                  W54      p
\        P                  ! R4      ;_uu_ 4        V! Wš4      pRRR4       W†V&   V#   + '       g   i     L; i)z\Reuse rotary outputs when ids, dtype, and device are unchanged across replay batches/epochs.Ú'_group_rotary_position_embeddings_cacheNF)r›   rè   r»  r­   r®   rp  r¯   r|   rŽ   r©   r…   r§   )r�   r’  r¹  rn   r™  Útarget_rotary_deviceÚrotary_cacher±   Úcached_position_embeddingsÚx_for_rotaryÚpos_for_rotarys   &$$$$      rd   Ú(_cached_group_rotary_position_embeddingsÚ;ParoQuantProcessor._cached_group_rotary_position_embeddings”  sà   € ð  -×8Ð8°·±ÐÜ˜tÐ%NÐPTÓUˆØÒØˆLØ;GÔ8ô ˆv‹JÜˆ|ÓÜÐ$Ó%Ü�—‘‹Lð	
ˆ	ð &2×%5Ñ%5°iÓ%@Ð"Ø%Ò-Ô1LÐMg×1hÒ1hÜ,×>Ñ>¸qÓWˆLÜ.×@Ñ@ÀÓdˆNÜ×%Ò% e×,Õ,Ù-3°LÓ-QÐ*÷ -à&@˜Ñ#Ø)Ð)÷ -×,ús   Ã	C)Ã)C9	r°   c               ó4   € V ^8„  d   QhRRRRRRRRR	RR
RRR/# )rs   rÿ  rA  r¹  rk   rj   ri   ro   rm   rn   r°   rT   ru   rW   )rv   s   "rd   rw   rÜ   ²  sY   € ÷ Sñ SàðSð ð	Sð
 %ðSð /ðSð -ðSð ðSð 
ñSrc   c          
     ól
  € VP                   pRpRp	V'       d–   VP                  '       g„   \        V RR4      p	V	f	   / p	W�n        \	        V4      \	        V4      \	        V4      Ve   \	        V4      M^ Ve   \	        V4      M^ \        V4      3pV	P                  V4      p
V
e   \        V
4      # RR0pV'       d�   \        V RR4      pVf	   / pWÀn        \	        V4      \        V4      3pVP                  V4      pVf=   VP                  4        UUu/ uF  w  ppWû9  g   K  V\        VVR7      bK  	  pppWìV&   \        V4      pMAVP                  4        UUu/ uF%  w  ppWû9  g   K  V\        P                  VV4      bK'  	  ppp\        V RR4      pVf
   / pVV n        \        V4      pVP                  V4      pVfK   RpRpRp \        P                   ! VP"                  4      P$                  pR	V9   pR
V9   pRV9   pVVV3pVVV&   Vw  pppV'       d   Vf   RM\        P+                  WG4      VR&   VP-                  4       ^8X  d"   V'       g	   V'       d   VP/                  ^ 4      pVP-                  4       ^8¼  dP   VP0                  ^ ,          pVP-                  4       ^8¼  d   VP0                  ^,          MVP0                  ^ ,          pM^pVP0                  ^ ,          pV P3                  4       pVEeW   VEeR   V'       EdI   T P5                  T;'       g    VP                   4      pT P7                  TT;'       g    VP                   4      pV'       d   TMRpVe   VP0                  R,          V8w  d9   T P9                  T;'       g    VP                   VVR7      pV'       d   VVR	&   TpM8Ve(   VP                   V8w  d   \        P+                  VV4      pV'       d   VVR	&   V P;                  V4      '       d   V P=                  VVVVR7      VR
&   MŸVe   VP                   V8X  d   TMVP?                  V4      pV! VV4      VR
&   MjV'       dc   Ve   VP0                  R,          V8w  d/   T P9                  T;'       g    VP                   VVR7      pVVR	&   M\        P+                  WW4      VR	&   RVR&   \        V P@                  RR4      p V e   V ! VV.VVR7      pV PC                  V4      pVe   V	e   \        V4      W˜&   V# u uppi u uppi   \&        \(        3 d    Rp EL1i ; i)zLRefresh grouped replay kwargs so real decoder layers can be replayed safely.NÚ_group_forward_prepared_cacheÚpast_key_valuesÚpast_key_valueÚ_group_forward_kwargs_cacherž  Ú_group_forward_signature_cacheFrn   Úposition_embeddingsro   T)r›   r­  r®  )r’  r¹  rn   r™  Ú	use_cacheÚprepare_layer_replay_kwargs)rÿ  Úlayer_inputÚadditional_inputsr“   rŠ  )"r›   rp  rè   rÅ  r­   r®   r¯   r]   rÈ  rº   r:   rŽ   r¸   rÉ  r¥   ÚinspectÚ	signatureÚforwardrO  r  Ú	TypeErrorr©   r6  r7  rh  r“  r¢  rš  r¶  rª  rÁ  r¨   rí   Ú!_normalize_group_runtime_metadata)!r�   rÿ  r¹  rj   ro   rn   r°   r“   Úprepared_cache_keyÚprepared_cacheÚcached_preparedÚskip_kwargsÚkwargs_cacheÚkwargs_cache_keyÚcached_module_kwargsr»   rt   Úmodule_kwargsÚsignature_cacheÚ
layer_typeÚcached_signatureÚsupports_position_idsÚsupports_position_embeddingsÚsupports_attention_maskrÐ  r­  r®  r’  r™  rÀ  Ú
pos_valuesr¿  Úprepare_replay_kwargss!   &&$$$$$                          rd   Ú_prepare_group_forward_kwargsÚ0ParoQuantProcessor._prepare_group_forward_kwargs²  sZ  € ð Ÿ™ˆà!ÐØˆß˜ŸŸ˜Ü$ TÐ+JÈDÓQˆNØÒ%Ø!#�Ø5CÔ2ä�5“	Ü�1“Ü�<Ó Ø&4Ò&@”�>Ô"ÀaØ$0Ò$<”�<Ô À!Ü�MÓ"ð"Ðð -×0Ñ0Ð1CÓDˆOØÒ*Ü˜OÓ,Ð,à(Ð*:Ð;ˆçÜ" 4Ð)FÈÓMˆLØÒ#Ø!�Ø3?Ô0Ü " <Ó 0´#°mÓ2DÐEÐØ#/×#3Ñ#3Ð4DÓ#EÐ Ø#Ò+ð '3×&8Ñ&8Ô&:ô(á&:™
˜˜UØÑ-ô E�Cœ¨°mÔDÒDÙ&:ð %ñ (ð
 2FÐ-Ñ.Ü Ð!5Ó6‰Mð #/×"4Ñ"4Ô"6ôá"6‘J�C˜ØÑ)ô S�Ô&×<Ñ<¸UÀMÓRÒRÙ"6ð ñ ô " $Ð(HÈ$ÓOˆØÒ"Ø ˆOØ2AˆDÔ/Ü˜%“[ˆ
Ø*×.Ñ.¨zÓ:ÐØÒ#Ø$)Ð!Ø+0Ð(Ø&+Ð#ð/Ü#×-Ò-¨e¯m©mÓ<×GÑG�	Ø(6¸)Ñ(CÐ%Ø/DÈ	Ñ/QÐ,Ø*:¸iÑ*GÐ'ð !6Ð7SÐUlÐmÐØ*:ˆO˜JÑ'ØWgÑTÐÐ;Ð=Tç"Ø6DÒ6L©dÜ!×3Ñ3°NÓRð Ð*Ñ+ð �5‰5‹7�aŒ<×2×6RØ—‘˜A“ˆAð �5‰5‹7�aŒ<ØŸ™ �
ˆIØ$%§E¡E£G¨q¤L�a—g‘g˜a–j°a·g±g¸aµj‰GàˆIØ—g‘g˜a•jˆGà×&Ñ&Ó(ˆØÓ 6Ó#5×:VÐ:VØ×0Ñ0°×1JÐ1JÀ!Ç(Á(ÓKˆFØ ×3Ñ3°F¸M×<UÐ<UÈQÏXÉXÓVˆMß-B™\ÈˆNØÒ%¨×)=Ñ)=¸bÕ)AÀWÔ)LØ!×<Ñ<Ø(×4Ð4¨A¯H©HØ'Ø#ð =ó �
÷
 )Ø4>�M .Ñ1Ø!+‘à Ò,°×1FÑ1FÈ-Ô1WÜ%6×%HÑ%HÈÐYfÓ%g�Nß(Ø4B�M .Ñ1à×9Ñ9¸&×AÒAØ7;×7dÑ7dØ!ØØ!/Ø"/ð	 8eó 8�Ð3Ò4ð %2Ò$9¸Q¿X¹XÈÔ=V™qÐ\]×\`Ñ\`ÐanÓ\o�Ù7=¸lÈNÓ7[�Ð3Ò4ß"ØÒ# |×'9Ñ'9¸"Õ'=ÀÔ'HØ!×<Ñ<Ø(×4Ð4¨A¯H©HØ'Ø#ð =ó �
ð
 1;�˜nÒ-ä0A×0SÑ0SÐT`Ó0p�˜nÑ-à%*ˆ�kÑ"Ü '¨¯©Ð9VÐX\Ó ]ÐØ Ò,Ù1ØØ˜CØ"/Ø+ô	ˆMð ×>Ñ>¸}ÓMˆØÒ)¨nÒ.HÜ15°mÓ1DˆNÑ.ØÐùóS(ùóøô* ¤	Ð*ô /Ø*.Ó'ð/ús*   ÄTÄ!TÅTÅ)TÇ9T ÔT3Ô2T3c               ó    € V ^8„  d   QhRRRR/# ©rs   rt   r   ru   rW   )rv   s   "rd   rw   rÜ   H  s   € ÷ ñ ¨Sð °Sñ rc   c                óŠ  € \        V \        P                  4      '       d:   \        P                  ! R4      ;_uu_ 4        V P	                  4       uuRRR4       # \        V \
        4      '       d9   V P                  4        UUu/ uF  w  rV\        P                  V4      bK  	  upp# \        V \        4      '       d&   V  Uu. uF  p\        P                  V4      NK  	  up# \        V \        4      '       d-   \        ;QJ d    . R V  4       F  NK  	  5# ! R V  4       4      # V #   + '       g   i     LÝ; iu uppi u upi )a/  Rebuild replay/runtime tensor trees with fresh normal tensors.

This is stricter than the inference-detection path. It protects the
live layer optimizer from tensors that still alias inference-backed
storage even when `is_inference()` does not fire on the exact view that
reaches the layer entry point.
FNc              3  óL   "  € T F  p\         P                  V4      x € K  	  R # 5irz   )rÒ   Ú_clone_group_runtime_metadatar}   s   & rd   r€   ÚCParoQuantProcessor._clone_group_runtime_metadata.<locals>.<genexpr>X  s"   é € ÐdÑ^cÐUZÔ+×IÑIÈ%×PÐPÓ^cùó   ‚"$)r„   r…   r†   r§   rÅ  r]   rº   rÒ   rê  rŠ   r‹   ©rt   r»   r   s   &  rd   rê  Ú0ParoQuantProcessor._clone_group_runtime_metadataG  sñ   € ô �eœUŸ\™\×*Ò*Ü×%Ò% e×,Õ,Ø—{‘{“}÷ -Ò,ä�eœT×"Ò"Øch×cnÑcnÔcpÔqÑcpÑU_ÐUX�CÔ+×IÑIÈ%ÓPÒPÑcpÒqÐqÜ�eœT×"Ò"ÙY^Ó_ÑY^ÐPUÔ&×DÑDÀUÖKÑY^Ñ_Ð_Ü�eœU×#Ò#ß”5ÑdÑ^cÓd—5Ðd�5ÑdÑ^cÓdÓdÐdØˆ÷ -×,üó rùâ_s   ¿D'Â!D:ÃE Ä'D7	c               ó    € V ^8„  d   QhRRRR/# )rs   Úinput_batchrg   ru   rp   rW   )rv   s   "rd   rw   rÜ   \  s   € ÷ 	Añ 	AÐ,>ð 	AÀ3ñ 	Arc   c                ó:  € V '       g   ^ # V ^ ,          p\        V\        P                  4      '       d   VP                  4       ^ 8X  d   ^ # VP	                  4       ^ 8X  d   ^# \        VP                  4       \        ^VP                  R,          4      ,          4      # )z<Count flattened token rows for one cached layer-input batch.rŠ  )r„   r…   r†   r‹  r6  rp   rœ   rh  )rð  Úprimarys   & rd   Ú_layer_batch_row_countÚ)ParoQuantProcessor._layer_batch_row_count[  sk   € ÷ ÙØ˜a•.ˆÜ˜'¤5§<¡<×0Ò0°G·M±M³OÀqÔ4HÙØ�;‰;‹=˜AÔÙÜ�7—=‘=“?¤c¨!¨W¯]©]¸2Õ->Ó&?Õ?Ó@Ð@rc   c               ó$   € V ^8„  d   QhRRRRRR/# ©rs   Úinput_batchesr%  Ú
row_budgetrp   ru   rW   )rv   s   "rd   rw   rÜ   g  s(   € ÷ "ñ "à/ð"ð ð"ð 
ñ	"rc   c                ó°   € V'       g   ^ # V^ 8:  d   ^# ^ p\        V^R7       F&  w  rEW0P                  V4      ,          pW28¼  g   K$  Vu # 	  \        V4      # )zQChoose the smallest non-empty prefix whose cached rows meet the requested budget.)rË   )Ú	enumerateró  rÉ   )r�   r÷  rø  Ú
total_rowsrH  r¾   s   &&&   rd   Ú_prefix_batch_count_for_rowsÚ/ParoQuantProcessor._prefix_batch_count_for_rowsg  sW   € ÷ ÙØ˜Œ?ÙØˆ
Ü% m¸1×=‰LˆEØ×5Ñ5°eÓ<Õ<ˆJØÖ'Ø’ñ >ô �=Ó!Ð!rc   c               ó$   € V ^8„  d   QhRRRRRR/# rö  rW   )rv   s   "rd   rw   rÜ   x  s(   € ÷ "ñ "à/ð"ð ð"ð 
ñ	"rc   c                ó¾   € V'       g   ^ # V^ 8:  d   ^# ^ p^ p\        V4       F-  pW0P                  V4      ,          pV^,          pW28¼  g   K+  Vu # 	  \        V4      # )zQChoose the smallest non-empty suffix whose cached rows meet the requested budget.)rk  ró  rÉ   )r�   r÷  rø  rû  Úcountr¾   s   &&&   rd   Ú_suffix_batch_count_for_rowsÚ/ParoQuantProcessor._suffix_batch_count_for_rowsx  sa   € ÷ ÙØ˜Œ?ÙØˆ
ØˆÜ˜mÖ,ˆEØ×5Ñ5°eÓ<Õ<ˆJØ�Q�JˆEØÖ'Ø’ñ	 -ô
 �=Ó!Ð!rc   c               ó    € V ^8„  d   QhRRRR/# )rs   Útarget_batchr   ru   rk   rW   )rv   s   "rd   rw   rÜ   Œ  s   € ÷ ñ  cð ¨lñ rc   c                óú   € \        V \        \        34      '       d   V '       g   \        R4      hV ^ ,          p \        V \        P
                  4      '       g#   \        R\        V 4      P                   R24      hV # )zMNormalize cached layer outputs to the single tensor used for the loss target.z>ParoQuant grouped optimization received an empty target batch.z=ParoQuant grouped optimization expected tensor targets, got `r&  )	r„   rŠ   r‹   r  r…   r†   rÒ  r¥   rX   )r  s   &rd   Ú_target_primaryÚ"ParoQuantProcessor._target_primary‹  sh   € ô �l¤T¬5 M×2Ò2ßÜ Ð!aÓbÐbØ'¨�?ˆLÜ˜,¬¯©×5Ò5ÜÐ[Ô\`ÐamÓ\n×\wÑ\wÐ[xÐxzÐ{Ó|Ð|ØÐrc   c          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )rs   r  r   r›   r”   rp  ztorch.dtyper°   rT   ru   rk   rW   )rv   s   "rd   rw   rÜ   –  s<   € ÷ ñ àðð ð	ð
 ðð ðð 
ñrc   c               óþ  € V P                  V4      pVP                  4       '       dE   \        P                  ! R4      ;_uu_ 4        VP	                  W#RR7      pRRR4       V'       g   V# V'       g   VP	                  W#R7      # \        V RR4      pVf	   / pW`n        \        V4      \        V4      \        V4      3pVP                  V4      pVf   VP	                  W#R7      pW†V&   V#   + '       g   i     Lš; i)zZCache grouped loss targets after primary-tensor normalization and dtype/device conversion.FT)r›   rp  r¤   Nro  Ú_group_target_cache)
r  r‡   r…   r§   r¨   rè   r
  r­   r®   r¯   )	r�   r  r›   rp  r°   rl   Útarget_cacher±   Úcached_targets	   &&$$$    rd   Ú_prepare_group_targetÚ(ParoQuantProcessor._prepare_group_target–  sÜ   € ð ×%Ñ% lÓ3ˆØ×Ñ× Ò Ü×%Ò% e×,Õ,ØŸ™¨&ÀD˜ÓI�÷ -çØ�ßØ—9‘9 F�9Ó8Ð8Ü˜tÐ%:¸DÓAˆØÒØˆLØ'3Ô$Ü˜“Z¤ V£¬c°%«jÐ9ˆ	Ø$×(Ñ(¨Ó3ˆØÒ Ø"ŸI™I¨V˜IÓAˆMØ&3˜Ñ#ØÐ÷ -×,ús   ÁC,Ã,C<	c               ó    € V ^8„  d   QhRRRR/# rç  rW   )rv   s   "rd   rw   rÜ   ³  s   € ÷ ñ ¨ð °ñ rc   c                ó(  € \        V \        P                  4      '       Ed   V P                  4       '       dN   \        P                  ! R4      ;_uu_ 4        V P                  4       P                  \        RR7      pRRR4       MWV P                  P                  \        P                  8w  d   V P                  4       P                  4       MV P                  4       p\        P                  P                  4       '       d'   XP                  4       '       g   VP                  4       pX# \        V \        4      '       d9   V P!                  4        UUu/ uF  w  r#V\"        P%                  V4      bK  	  upp# \        V \&        4      '       d&   V  Uu. uF  p\"        P%                  V4      NK  	  up# \        V \(        4      '       d-   \(        ;QJ d    . R V  4       F  NK  	  5# ! R V  4       4      # V #   + '       g   i     EL*; iu uppi u upi )z=Recursively normalize replay metadata onto CPU-owned tensors.FTrx  Nc              3  óL   "  € T F  p\         P                  V4      x € K  	  R # 5irz   )rÒ   Ú_move_group_value_to_cpur}   s   & rd   r€   Ú>ParoQuantProcessor._move_group_value_to_cpu.<locals>.<genexpr>Ã  s"   é € Ð_ÑY^ÐPUÔ+×DÑDÀU×KÐKÓY^ùrì  )r„   r…   r†   r‡   r§   r8  r¨   r@   r›   r¥   r4  r¢   Úis_availabler¦   Ú
pin_memoryr]   rº   rÒ   r  rŠ   r‹   )rt   rÁ   r»   r   s   &   rd   r  Ú+ParoQuantProcessor._move_group_value_to_cpu²  s  € ô �eœUŸ\™\×*Ó*Ø×!Ñ!×#Ò#Ü×)Ò)¨%×0Õ0Ø"Ÿ\™\›^×.Ñ.´cÀÐ.ÓE�F÷ 1Ð0ð 27·±×1BÑ1BÄcÇhÁhÔ1N˜Ÿ™›×+Ñ+Ô-ÐTY×T`ÑT`ÓTb�Ü�z‰z×&Ñ&×(Ò(°×1AÑ1A×1CÒ1CØ×*Ñ*Ó,�ØˆMÜ�eœT×"Ò"Ø^c×^iÑ^iÔ^kÔlÑ^kÑPZÐPS�CÔ+×DÑDÀUÓKÒKÑ^kÒlÐlÜ�eœT×"Ò"ÙTYÓZÑTYÈ5Ô&×?Ñ?ÀÖFÑTYÑZÐZÜ�eœU×#Ò#ß”5Ñ_ÑY^Ó_—5Ð_�5Ñ_ÑY^Ó_Ó_Ð_Øˆ÷ 1×0Ð0üó mùâZs   Á&G5Å!H	ÆHÇ5H	c               ó    € V ^8„  d   QhRRRR/# rç  rW   )rv   s   "rd   rw   rÜ   Ç  s   € ÷ ñ °ð ¸ñ rc   c                ó0  € \        V \        P                  4      '       d    \        P	                  W P
                  4      # \        V \        4      '       d9   V P                  4        UUu/ uF  w  rV\        P                  V4      bK  	  upp# \        V \        4      '       d&   V  Uu. uF  p\        P                  V4      NK  	  up# \        V \        4      '       d-   \        ;QJ d    . R V  4       F  NK  	  5# ! R V  4       4      # V # u uppi u upi )zZRebuild inference-mode replay metadata once kwargs are fully assembled on the live device.c              3  óL   "  € T F  p\         P                  V4      x € K  	  R # 5irz   )rÒ   rÓ  r}   s   & rd   r€   ÚGParoQuantProcessor._normalize_group_runtime_metadata.<locals>.<genexpr>Ó  s"   é € ÐhÑbgÐY^Ô+×MÑMÈe×TÐTÓbgùrì  )r„   r…   r†   rŽ   r©   r›   r]   rº   rÒ   rÓ  rŠ   r‹   rí  s   &  rd   rÓ  Ú4ParoQuantProcessor._normalize_group_runtime_metadataÆ  sæ   € ô �eœUŸ\™\×*Ò*Ü$×6Ñ6°u¿l¹lÓKÐKÜ�eœT×"Ò"ð #(§+¡+¤-ôá"/‘J�Cð Ô'×IÑIÈ%ÓPÒPÙ"/òð ô �eœT×"Ò"Ù]bÓcÑ]bÐTYÔ&×HÑHÈÖOÑ]bÑcÐcÜ�eœU×#Ò#ß”5ÑhÑbgÓh—5Ðh�5ÑhÑbgÓhÓhÐhØˆùóùò
 ds   Á)!DÂ(Dc               ó    € V ^8„  d   QhRRRR/# )rs   r  rC   ru   z?tuple[list[_ParoQuantReplayBatch], list[_ParoQuantReplayBatch]]rW   )rv   s   "rd   rw   rÜ   Ö  s#   € ÷ Bñ Bà#ðBð 
IñBrc   c                ó  € \        VRR4      pVe   V# VP                  ;'       g    . pVP                  ;'       g    . pVP                  ;'       g    . pV'       d	   V'       g   \	        R4      h\        V4      \        V4      8w  d   \	        R4      hV'       g%   \        \        V4      4       Uu. uF  p/ NK  	  ppM$\        V4      \        V4      8w  d   \	        R4      h\        V P                  P                  ;'       g    . 4      p\        V P                  P                  ;'       g    . 4      p\        V4      \        V4      8  d3   VP                  R.\        V4      \        V4      ,
          ,          4       \        V4      \        V4      8  d3   VP                  R.\        V4      \        V4      ,
          ,          4       . p	\        VVVVV4       FÑ  w  r«rÍpV
 Uu. uF  pV P                  V4      NK  	  ppT	P                  \        TVP!                  4        UUu/ uF  w  ppVV P                  V4      bK  	  uppV P                  V P#                  V4      4      Vf   RMV P                  V4      Vf   RMV P                  V4      V P%                  V4      R7      4       KÓ  	  T P'                  V	 Uu. uF  pVP(                  NK  	  up\+        V P,                  P.                  4      4      pT P1                  V	 Uu. uF  pVP(                  NK  	  up\+        V P,                  P2                  4      4      p\5        ^ \        V	4      V,
          4      pV	RV V	VR 3pVVn        V# u upi u upi u uppi u upi u upi )zPBuild CPU-owned train/validation replay batches for in-place layer optimization.rO   NzHParoQuant layer optimization requires captured layer inputs and outputs.zHParoQuant layer optimization requires aligned input/output batch counts.zAParoQuant layer optimization requires aligned layer-input kwargs.rÀ   )rè   rJ   rK   rL   rU  rÉ   rÈ   rŠ   Úinputs_cachern   Úattention_masksÚextendrQ  r  r<  rf   rº   r  ró  rü  rh   rp   rÔ   rÉ  r  rÊ  rœ   rO   )r�   r  Úcached_batchesr÷  Úinput_kwargs_batchesÚoutput_batchesrZ  rn   r  rO   rð  rj   Úoutput_batchÚpos_idsÚ	attn_maskrÁ   Ú
cpu_inputsr»   rt   r¾   Útrain_batch_countÚval_batch_countÚ	val_startÚreplay_splits   &&                      rd   Ú_replay_batches_from_stateÚ-ParoQuantProcessor._replay_batches_from_stateÖ  sH  € ô
 ! Ð(8¸$Ó?ˆØÒ%Ø!Ð!à×*Ñ*×0Ð0¨bˆØ$×7Ñ7×=Ð=¸2ÐØ×,Ñ,×2Ð2°ˆß§NÜÐiÓjÐjÜˆ}Ó¤ ^Ó!4Ô4ÜÐiÓjÐjç#Ü05´c¸-Ó6HÔ0IÓ#JÑ0I¨1£BÑ0IÐ Ð#JÐ ÜÐ%Ó&¬#¨mÓ*<Ô<ÜÐbÓcÐcä˜D×-Ñ-×:Ñ:×@Ð@¸bÓAˆÜ˜t×0Ñ0×@Ñ@×FÐFÀBÓGˆÜˆ|Óœs =Ó1Ô1Ø×Ñ  ¬#¨mÓ*<¼sÀ<Ó?PÕ*PÕ QÔRÜˆÓ¤# mÓ"4Ô4Ø×"Ñ" D 6¬S°Ó-?Ä#ÀoÓBVÕ-VÕ#WÔXà68ˆÜKNØØ ØØØöL
ÑGˆK |¸iñ *óá)�Fð ×-Ñ-¨fÖ5Ù)ð ð ð ×!Ñ!Ü%Ø%ð +7×*<Ñ*<Ô*>ô"á*>™J˜C ð ˜T×:Ñ:¸5ÓAÒAÙ*>ò"ð  ×8Ñ8¸×9MÑ9MÈlÓ9[Ó\Ø)0ª¡¸d×>[Ñ>[Ð\cÓ>dØ+4Ò+<¡4À$×B_ÑB_Ð`iÓBjØ"×9Ñ9¸*ÓEô
öñL
ð2 !×=Ñ=Ù'5Ó6¡~˜eˆU�\Œ\¡~Ñ6Ü�—	‘	×+Ñ+Ó,ó
Ðð ×;Ñ;Ù'5Ó6¡~˜eˆU�\Œ\¡~Ñ6Ü�—	‘	×0Ñ0Ó1ó
ˆô ˜œ3˜~Ó.°Õ@ÓAˆ	Ø&Ð'9Ð(9Ð:¸NÈ9È:Ð<VÐWˆØ+ˆÔØÐùòa $Kùò&ùó"ùò 7ùò 7s   Â3M4Ç)M9È)M>ËNÌN	c               ó    € V ^8„  d   QhRRRR/# )rs   rt   rk   ru   rp   rW   )rv   s   "rd   rw   rÜ     s   € ÷ 9ñ 9˜\ð 9¨cñ 9rc   c                ó^   € \        V P                  4       V P                  4       ,          4      # )z'Measure one tensor's storage footprint.)rp   r‹  Úelement_sizerŒ   s   &rd   Ú_tensor_bytesÚ ParoQuantProcessor._tensor_bytes  s#   € ô �5—;‘;“= 5×#5Ñ#5Ó#7Õ7Ó8Ð8rc   c               ó    € V ^8„  d   QhRRRR/# )rs   rt   r   ru   rp   rW   )rv   s   "rd   rw   rÜ     s   € ÷ ñ ¨#ð °#ñ rc   c                ó>  a € \        V\        P                  4      '       d   S P                  V4      # \        V\        4      '       d$   \        V 3R lVP                  4        4       4      # \        V\        \        34      '       d   \        V 3R lV 4       4      # ^ # )z)Measure nested replay metadata footprint.c              3  óF   <"  € T F  pSP                  V4      x € K  	  R # 5irz   ©Ú_nested_tensor_bytes©r~   r   r�   s   & €rd   r€   Ú:ParoQuantProcessor._nested_tensor_bytes.<locals>.<genexpr>$  s   øé € ÐTÁ^¸E�t×0Ñ0°×7Ð7Ã^ùó   ƒ!c              3  óF   <"  € T F  pSP                  V4      x € K  	  R # 5irz   r6  r8  s   & €rd   r€   r9  &  s   øé € ÐKÁU¸E�t×0Ñ0°×7Ð7ÃUùr:  )	r„   r…   r†   r1  r]   Úsumr‰   rŠ   r‹   )r�   rt   s   f&rd   r7  Ú'ParoQuantProcessor._nested_tensor_bytes  sn   ø€ ä�eœUŸ\™\×*Ò*Ø×%Ñ% eÓ,Ð,Ü�eœT×"Ò"ÜÔTÀUÇ\Á\Ä^ÓTÓTÐTÜ�eœd¤E˜]×+Ò+ÜÔKÁUÓKÓKÐKÙrc   c               ó    € V ^8„  d   QhRRRR/# )rs   r¾   rf   ru   rp   rW   )rv   s   "rd   rw   rÜ   )  s   € ÷ 
ñ 
Ð)>ð 
À3ñ 
rc   c                ó‚  a € \        V 3R lVP                   4       4      S P                  VP                  4      ,           VP                  f   ^ MS P                  VP                  4      ,           VP
                  f   ^ MS P                  VP
                  4      ,           S P                  VP                  4      ,           # )z-Estimate one replay batch's device footprint.c              3  óF   <"  € T F  pSP                  V4      x € K  	  R # 5irz   )r1  )r~   rÁ   r�   s   & €rd   r€   Ú9ParoQuantProcessor._replay_batch_bytes.<locals>.<genexpr>,  s   øé € ÐF¹¨v�×"Ñ" 6×*Ð*»ùr:  )r<  rh   r1  rl   rn   ro   r7  rj   )r�   r¾   s   f&rd   Ú_replay_batch_bytesÚ&ParoQuantProcessor._replay_batch_bytes)  sž   ø€ ô ÔF¸¿ºÓFÓFØ× Ñ  §¡Ó.õ/à×&Ñ&Ò.‰q°D×4FÑ4FÀu×GYÑGYÓ4Zõ\ð ×(Ñ(Ò0‰q°d×6HÑ6HÈ×I]ÑI]Ó6^õ`ð ×'Ñ'¨×(:Ñ(:Ó;õ	<ð	
rc   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rs   rÿ  rA  Úparam_groupszSequence[dict[str, object]]rO   r’   ru   rp   rW   )rv   s   "rd   rw   rÜ   3  s8   € ÷ (Mñ (Màð(Mð 2ð	(Mð
 4ð(Mð 
ñ(Mrc   c          	     óð  a € V'       g   ^#  \        VP                  4       4      P                  pVP                  R8w  d   \        V4      #  \        P                  P                  V4      w  rV\        T 3R lTP                  4        4       4      pT\        T 3R lTP                  4        4       4      ,          pT UU	u/ uFJ  pTP                  R. 4       F1  p	\        T	\        P                   4      '       g   K%  \#        T	4      T	bK3  	  KL  	  p
pp	\        R T
P%                  4        4       4      p\'        ^\        T4      4      p\)        ^\        T 3R lTRT  4       4      T,          4      pRp\)        T\+        T4      T,
          T,
          T,
          4      p\)        ^\'        \        T4      Yý,          4      4      #   \        \        3 d    \
        p EL¬i ; i  \         d     ^# i ; iu up	pi )	zDChoose a conservative train-shard size from current free GPU memory.r¢   c              3  ób   <"  € T F$  pSP                  VP                  4       4      x € K&  	  R # 5irz   ©r1  r8  )r~   rv  r�   s   & €rd   r€   Ú@ParoQuantProcessor._layer_train_shard_batches.<locals>.<genexpr>K  s'   øé € Ð]ÑJ\À˜$×,Ñ,¨U¯\©\«^×<Ð<ÓJ\ùó   ƒ,/c              3  ób   <"  € T F$  pSP                  VP                  4       4      x € K&  	  R # 5irz   rH  )r~   rx  r�   s   & €rd   r€   rI  L  s%   øé € Ð]É_À6˜4×-Ñ-¨f¯m©m«o×>Ð>Ë_ùrJ  Úparamsc              3  ób   "  € T F%  p\        VP                  4       4      ^,          x € K'  	  R# 5i)é   N)rp   r‹  )r~   rv  s   & rd   r€   rI  S  s%   é € ÐVÑ?U°eœ#˜eŸk™k›mÓ,¨r×1Ò1Ó?Uùs   ‚-/c              3  óF   <"  € T F  pSP                  V4      x € K  	  R # 5irz   )rB  )r~   r¾   r�   s   & €rd   r€   rI  W  s!   øé € Ð[Ñ=Z°E�×(Ñ(¨×/Ð/Ó=Zùr:  Ni   )rN  rO  r›   rQ  rU  r@   r¥   rÉ   r…   r¢   Úmem_get_infor<  Úbuffersr¯   r„   r   rr  r­   r‰   rÊ   rœ   rp   )r�   rÿ  rE  rO   r“   Ú
free_bytesÚ_total_bytesÚlayer_bytesÚgrouprv  Úactive_paramsÚoptim_bytesÚsample_countÚavg_batch_bytesÚactivation_marginÚheadrooms   f&$$            rd   Ú_layer_train_shard_batchesÚ-ParoQuantProcessor._layer_train_shard_batches3  s³  ø€ ÷ Ùð	 Ü  ×!1Ñ!1Ó!3Ó4×;Ñ;ˆMð ×Ñ Ô'Ü�~Ó&Ð&ð	Ü',§z¡z×'>Ñ'>¸}Ó'MÑ$ˆJô Ô]È%×JZÑJZÔJ\Ó]Ó]ˆØ”sÔ]ÈUÏ]É]Ì_Ó]Ó]Õ]ˆñ &ô
á%�ØŸ™ 8¨RÖ0�Ü˜%¤§¡×.ô ŒBˆu‹I�uÒá0ñ Ù%ð 	ñ 
ô ÑV¸}×?SÑ?SÔ?UÓVÓVˆÜ˜1œc .Ó1Ó2ˆÜØÜÔ[¸^ÈMÈ\Ñ=ZÓ[Ó[Ð_kÕkó
ˆð .ÐÜ�¬¨J«¸+Õ(EÈÕ(SÐVgÕ(gÓhˆÜ�1”cœ#˜nÓ-¨xÕ/JÓKÓLÐLøô7 œ|Ð,ô 	 Ü‹Mð	 ûô ô 	Úð	üó

s/   �#G Á!G  Ã 7G2Ã<G2ÇGÇGÇ G/Ç.G/c               ó    € V ^8„  d   QhRRRR/# )rs   r  rC   ru   úótuple[list[List[torch.Tensor]], list[Dict[str, Any]], list[Any], list[Optional[torch.Tensor]], list[Optional[torch.Tensor]], list[List[torch.Tensor]], list[Dict[str, Any]], list[Any], list[Optional[torch.Tensor]], list[Optional[torch.Tensor]]]rW   )rv   s   "rd   rw   rÜ   ]  s   € ÷ 9ñ 9à#ð9ð
ñ9rc   c                óò  € \        VRR4      pVe   V# VP                  ;'       g    . pVP                  ;'       g    . pVP                  ;'       g    . pV'       d	   V'       g   \	        R4      h\        V4      \        V4      8w  d   \	        R4      hV'       g%   \        \        V4      4       Uu. uF  p/ NK  	  ppM$\        V4      \        V4      8w  d   \	        R4      h\        V P                  P                  ;'       g    . 4      p\        V P                  P                  ;'       g    . 4      p\        V4      \        V4      8  d3   VP                  R.\        V4      \        V4      ,
          ,          4       \        V4      \        V4      8  d3   VP                  R.\        V4      \        V4      ,
          ,          4       V P                  V\        V P                  P                  4      4      p	V P!                  V\        V P                  P"                  4      4      p
\%        ^ \        V4      V
,
          4      pVRV	 VRV	 VRV	 VRV	 VRV	 W;R WKR W[R W{R W‹R 3
pWÁn        V# u upi )zXSlice the preserved layer IO into train/validation batch lists for grouped optimization.rM   NzJParoQuant grouped optimization requires captured layer inputs and outputs.zJParoQuant grouped optimization requires aligned input/output batch counts.zCParoQuant grouped optimization requires aligned layer-input kwargs.)rè   rJ   rK   rL   rU  rÉ   rÈ   rŠ   r  rn   r  r   rü  rp   rÔ   rÉ  r  rÊ  rœ   rM   )r�   r  Úcached_datasetr÷  r"  r#  rZ  rn   r  r(  r)  r*  rM   s   &&           rd   Ú_group_dataset_from_stateÚ,ParoQuantProcessor._group_dataset_from_state]  sS  € ô  ! Ð(9¸4Ó@ˆØÒ%Ø!Ð!à×*Ñ*×0Ð0¨bˆØ$×7Ñ7×=Ð=¸2ÐØ×,Ñ,×2Ð2°ˆß§NÜÐkÓlÐlÜˆ}Ó¤ ^Ó!4Ô4ÜÐkÓlÐlç#Ü05´c¸-Ó6HÔ0IÓ#JÑ0I¨1£BÑ0IÐ Ð#JÐ ÜÐ%Ó&¬#¨mÓ*<Ô<ÜÐdÓeÐeä˜D×-Ñ-×:Ñ:×@Ð@¸bÓAˆÜ˜t×0Ñ0×@Ñ@×FÐFÀBÓGˆÜˆ|Óœs =Ó1Ô1Ø×Ñ  ¬#¨mÓ*<¼sÀ<Ó?PÕ*PÕ QÔRÜˆÓ¤# mÓ"4Ô4Ø×"Ñ" D 6¬S°Ó-?Ä#ÀoÓBVÕ-VÕ#WÔXà ×=Ñ=¸mÌSÐQU×QZÑQZ×QlÑQlÓMmÓnÐØ×;Ñ;¸MÌ3ÈtÏyÉy×OoÑOoÓKpÓqˆÜ˜œ3˜}Ó-°Õ?Ó@ˆ	ð Ð,Ð,Ð-Ø Ð!3Ð"3Ð4ØÐ-Ð-Ð.ØÐ+Ð+Ð,ØÐ.Ð.Ð/Ø˜*Ð%Ø  Ð,Ø˜:Ð&Ø˜Ð$Ø˜JÐ'ð
ˆð !0ÔØÐùò9 $Ks   Â3I4c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r  rC   r“   r–  ru   r_  rW   )rv   s   "rd   rw   rÜ   ˜  s(   € ÷ 6ñ 6à#ð6ð .ð6ð
ñ	6rc   c                óL  a€ V P                  V4      pVe   \        P                  ! V4      M\        o\	        S4      p\        VRR4      pVf	   / pWQn        VP                  V4      pVe   V# R V3R llpR V3R llpR V3R llp	R	 V3R
 llp
V! V^ ,          4      V! V^,          4      V	! V^,          4      V
! V^,          4      V
! V^,          4      V! V^,          4      V! V^,          4      V	! V^,          4      V
! V^,          4      V
! V^	,          4      3
pWµV&   V# )zWMaterialize the cached grouped dataset on the target device once per layer/device pair.NrN   c               ó    € V ^8„  d   QhRRRR/# )rs   r‘   úlist[List[torch.Tensor]]ru   rW   )rv   s   "rd   rw   ÚBParoQuantProcessor._group_dataset_for_device.<locals>.__annotate__µ  s    € ÷ 	_ñ 	_Ð*Bð 	_ÐG_ñ 	_rc   c                óv   <€ V  UUu. uF  q Uu. uF  p\        VSR 7      NK  	  upNK!  	  upp# u upi u uppi ©rž  ©r9   )r‘   r¾   rÁ   r›   s   &  €rd   Ú_move_tensor_batchesÚJParoQuantProcessor._group_dataset_for_device.<locals>._move_tensor_batchesµ  s6   ø€ ÙV]Ô^ÑV]ÈUÀ%ÓHÁ%¸”W˜V¨F×3Á%ÔHÑV]Ò^Ð^ùÒHùÓ^s   ‡5�0¥5°5c               ó    € V ^8„  d   QhRRRR/# )rs   Úkwargs_batchesúlist[Dict[str, Any]]ru   rW   )rv   s   "rd   rw   rh  ¸  s    € ÷ 	Xñ 	XÐ1Eð 	XÐJ^ñ 	Xrc   c                óH   <€ V  Uu. uF  p\        VSR 7      NK  	  up# u upi rj  ©r:   )ro  Úkwargsr›   s   & €rd   Ú_move_kwargs_batchesÚJParoQuantProcessor._group_dataset_for_device.<locals>._move_kwargs_batches¸  s"   ø€ ÙHVÓWÉ¸f”N 6°&×9ÉÑWÐWùÒWó   †c               ó    € V ^8„  d   QhRRRR/# )rs   Útarget_batchesú	list[Any]ru   rW   )rv   s   "rd   rw   rh  »  s   € ÷ 	Vñ 	V°ð 	V¸yñ 	Vrc   c                óH   <€ V  Uu. uF  p\        VSR 7      NK  	  up# u upi rj  rr  )rx  r¾   r›   s   & €rd   Ú_move_target_batchesÚJParoQuantProcessor._group_dataset_for_device.<locals>._move_target_batches»  s"   ø€ ÙFTÓUÁn¸U”N 5°×8ÁnÑUÐUùÒUrv  c               ó    € V ^8„  d   QhRRRR/# )rs   Úoptional_tensorsúlist[Optional[torch.Tensor]]ru   rW   )rv   s   "rd   rw   rh  ¾  s    € ÷ 	oñ 	oÐ5Qð 	oÐVrñ 	orc   c                óR   <€ V  Uu. uF  qf   R M\        VSR7      NK  	  up# u upi )Nrž  rk  )r~  rÁ   r›   s   & €rd   Ú_move_optional_tensorsÚLParoQuantProcessor._group_dataset_for_device.<locals>._move_optional_tensors¾  s,   ø€ Ù]mÓnÑ]mÐSYšN‘D´¸ÀvÔ0NÒNÑ]mÑnÐnùÒns   †$)rb  r…   r›   r@   r®   rè   rN   r¯   )r�   r  r“   rM   r±   rN   ra  rl  rt  r{  r�  Údevice_datasetr›   s   &&&         @rd   Ú_group_dataset_for_deviceÚ,ParoQuantProcessor._group_dataset_for_device˜  s=  ø€ ð" ×8Ñ8¸Ó?ˆØ0=Ò0I”—’˜mÔ,ÌsˆÜ˜“Kˆ	Ü$+¨EÐ3NÐPTÓ$UÐ!Ø$Ò,Ø(*Ð%Ø.GÔ+à2×6Ñ6°yÓAˆØÒ%Ø!Ð!÷	_ð 	_÷	Xð 	X÷	Vð 	V÷	oð 	oñ ! °Õ!3Ó4Ù  °Õ!3Ó4Ù  °Õ!3Ó4Ù" ?°1Õ#5Ó6Ù" ?°1Õ#5Ó6Ù  °Õ!3Ó4Ù  °Õ!3Ó4Ù  °Õ!3Ó4Ù" ?°1Õ#5Ó6Ù" ?°1Õ#5Ó6ð
ˆð 0> )Ñ,ØÐrc   c               ó4   € V ^8„  d   QhRRRRRRRRR	R
RR
RR/# )rs   rÿ  rA  r>  rp   rð  rg   rj   ri   ro   rm   rn   ru   rk   rW   )rv   s   "rd   rw   rÜ   Ð  sP   € ÷ &ñ &àð&ð ð	&ð
 (ð&ð %ð&ð /ð&ð -ð&ð 
ñ&rc   c               ó,  € ? \        VP                  4       4      P                  pV Uu. uF  p\        P                  W‡4      NK  	  p	pV	'       g   \	        R4      hV P                  VV	^ ,          VVVR7      p
V! V	/ V
B pVf   \	        R4      h\        V\        4      '       d
   V^ ,          p\        V\        P                  4      '       g#   \        R\        V4      P                   R24      hV#   \        \        3 d
    \
        p Lçi ; iu upi )zQRun one cached batch through a grouped layer clone and return its primary output.zJParoQuant grouped optimization forward requires at least one input tensor.)r¹  rj   ro   rn   ú:ParoQuant grouped optimization forward returned no output.úCParoQuant grouped optimization expected tensor layer outputs, got `r&  )rN  rO  r›   rQ  rU  r@   rŽ   r©   rä  r„   r‹   r…   r†   rÒ  r¥   rX   )r�   rÿ  r>  rð  rj   ro   rn   Úlayer_deviceÚinprh   rÎ  Úmodule_outputs   &&$$$$$     rd   Ú_forward_group_batchÚ'ParoQuantProcessor._forward_group_batchÐ  s!  € ð ð	Ü × 0Ñ 0Ó 2Ó3×:Ñ:ˆLñ U`Ó`ÑT_ÈSÔ#×5Ñ5°cÖHÑT_ˆÐ`ßÜÐkÓlÐlà ×>Ñ>ØØ�Q�iØ%Ø)Ø%ð ?ó 
Ðñ ˜vÐ;Ð):Ñ;ˆØÒ ÜÐ[Ó\Ð\Ü�m¤U×+Ò+Ø)¨!Õ,ˆMÜ˜-¬¯©×6Ò6ÜðÜ˜]Ó+×4Ñ4Ð5°Rð9óð ð Ðøô1 œ|Ð,ô 	ÜŠLð	üò as   ƒ#C4 ªDÃ4DÄDc               ó(   € V ^8„  d   QhRRRRRRRR/# ©	rs   rÿ  rA  Úreplay_batchrf   Úcache_kwargsrT   ru   rk   rW   )rv   s   "rd   rw   rÜ   ø  s2   € ÷ +ñ +àð+ð ,ð	+ð
 ð+ð 
ñ+rc   c               óæ  € VP                   '       d   VP                   ^ ,          P                  M\        pVP                    Uu. uF'  pV P                  \        P                  WT4      4      NK)  	  ppV'       g   \        R4      hV P                  VV^ ,          VP                  VP                  VP                  VR7      pV P                  V4      p\        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       g   \        V4      '       dR   \        RV Uu. uF  qUP                  4       NK  	  up R\        V4       R\        VP                  4       4       24      hV! V/ VB pVf   \        R	4      h\!        V\"        4      '       d
   V^ ,          p\!        V\$        P&                  4      '       g#   \)        R
\+        V4      P,                   R24      hV# u upi u upi )z=Run one streamed replay batch through the live grouped layer.z:ParoQuant layer replay requires at least one input tensor.)r¹  rj   ro   rn   r°   c              3  ó@   "  € T F  qP                  4       x € K  	  R # 5irz   )r‡   )r~   r‹  s   & rd   r€   Ú;ParoQuantProcessor._forward_replay_batch.<locals>.<genexpr>  s   é € Ð4©V c×Ñ×!Ð!«Vùs   ‚TFzNParoQuant layer replay assembled inference-mode live inputs. inputs_inference=z kwargs_inference=z kwargs_keys=rˆ  r‰  r&  )rh   r›   r@   rê  rŽ   r©   rU  rä  rj   ro   rn   rˆ   r|   r‡   rP  Úkeysr„   r‹   r…   r†   rÒ  r¥   rX   )	r�   rÿ  r‘  r’  rŠ  r‹  rh   rÎ  rŒ  s	   &&$$     rd   Ú_forward_replay_batchÚ(ParoQuantProcessor._forward_replay_batchø  sä  € ð 9E×8K×8KÐ8K�|×*Ñ*¨1Õ-×4Ò4ÔQTˆð $×*Ò*ó
á*�ð ×.Ñ.Ô/@×/RÑ/RÐSVÓ/eÖfÙ*ð 	ð 
÷ ÜÐ[Ó\Ð\à ×>Ñ>ØØ�Q�iØ%×2Ñ2Ø'×6Ñ6Ø%×2Ñ2Øð ?ó 
Ðð !×>Ñ>Ð?PÓQÐß‹3Ñ4©VÓ4�3�3Š3Ñ4©VÓ4×4Ò4Ô8SÐTe×8fÒ8fÜð$ÙCIÓ$JÁ6¸C×%5Ñ%5Ö%7Á6Ñ$JÐ#Kð L$Ü$?Ð@QÓ$RÐ#Sð TÜ%Ð&7×&<Ñ&<Ó&>Ó?Ð@ðBóð ñ ˜vÐ;Ð):Ñ;ˆØÒ ÜÐ[Ó\Ð\Ü�m¤U×+Ò+Ø)¨!Õ,ˆMÜ˜-¬¯©×6Ò6ÜðÜ˜]Ó+×4Ñ4Ð5°Rð9óð ð ÐùòC
ùò& %Ks   Á-G)Ä(G.
c               ó    € V ^8„  d   QhRRRR/# )rs   r‰  ú dict[str, _ParoQuantOptimLinear]ru   r—   rW   )rv   s   "rd   rw   rÜ   &  s   € ÷ /ñ /Ð+Kð /ÐPTñ /rc   c                óT   € V P                  4        F  pVP                  4        K  	  R# )zLClamp masked dummy rotations back to zero after each grouped optimizer step.N)r‰   Úreset_masked_angles)r‰  rŠ  s   & rd   Ú_reset_group_anglesÚ&ParoQuantProcessor._reset_group_angles%  s#   € ð *×0Ñ0Ö2ˆLØ×,Ñ,Ö.ó 3rc   c               ó8   € V ^8„  d   QhRRRRRRRRR	R
RR
RRRR/# )rs   rÿ  rA  r÷  rg  r"  rp  rx  ry  rn   r  r  Úuse_amprT   ru   r…  rW   )rv   s   "rd   rw   rÜ   +  sZ   € ÷  7ñ  7àð 7ð 0ð	 7ð
 3ð 7ð "ð 7ð 3ð 7ð 6ð 7ð ð 7ð 
ñ 7rc   c               ó¤  € V'       g   R# Rp\         P                  ! 4       ;_uu_ 4        \        \        W#WEV4      4       FÂ  w  p	w  r«rÍpV'       d    \         P                  P                  R4      M	\        4       pV;_uu_ 4        V P                  VV	V
VVVR7      pV P                  VVP                  VP                  R7      pV\        \        P                  ! VV4      P                  4       4      ,          pRRR4       KÄ  	  RRR4       V\        ^\!        V4      4      ,          #   + '       g   i     Kû  ; i  + '       g   i     L@; i)zKMeasure full-layer reconstruction error for one grouped optimization stage.r´  r¢   ©r>  rð  rj   ro   rn   ro  N)r…   r§   rú  rQ  ÚampÚautocastr   r�  r  r›   rp  r…  ÚFÚsmooth_l1_lossrK  rœ   rÉ   )r�   rÿ  r÷  r"  rx  rn   r  r   Ú
total_lossr>  rð  rj   r  r%  r&  Úautocast_ctxÚpredsrl   s   &&$$$$$$          rd   Ú_evaluate_group_layerÚ(ParoQuantProcessor._evaluate_group_layer+  s  € ÷ Ùàˆ
Ü×!Ò!×#Õ#Ü^gÜ�M¸ÐWfÓgö_ÑZ�ÑZ˜k¸ÐPY÷ >EœuŸy™y×1Ñ1°&Ô9Ì+Ë-�ß!’\Ø ×5Ñ5ØØ$/Ø$/Ø%1Ø'0Ø%,ð 6ó �Eð "×7Ñ7¸ÈUÏ\É\Ðaf×alÑalÐ7Óm�FØ¤%¬×(8Ò(8¸ÀÓ(G×(LÑ(LÓ(NÓ"OÕO�J÷ "‘\ñ	_÷ $ð  œC ¤3 }Ó#5Ó6Õ6Ð6÷ "—\�\ú÷ $×#ús%   ªAD?ÂA5D+Ã;D?Ä+D<Ä6	D?Ä?E	c               ó4   € V ^8„  d   QhRRRRRRRRR	R
RR
RR/# )rs   rÿ  rA  r>  rp   rð  zlist[torch.Tensor]rj   zdict[str, Any]ro   rm   rn   ru   rk   rW   )rv   s   "rd   rw   rÜ   M  sY   € ÷ 'Wñ 'Wàð'Wð ð	'Wð
 (ð'Wð %ð'Wð /ð'Wð -ð'Wð 
ñ'Wrc   c          	     ó®  a aaa
aa€ S P                  4       '       d	   V'       g   S P                  SSVVVVR7      # V Uu. uF1  pS P                  \        P	                  WwP
                  4      4      NK3  	  ppS P                  V4      oS P                  V4      o
S P                  V4      oR VVV
VVV 3R llp	\        V	.\        V4      O5RR/ # u upi )zLOptionally checkpoint grouped train forwards to reduce activation residency.r¢  c               ó    € V ^8„  d   QhRRRR/# ©rs   Úruntime_inputsrk   ru   rW   )rv   s   "rd   rw   ÚCParoQuantProcessor._forward_group_batch_train.<locals>.__annotate__j  s   € ÷ 	ñ 	 lð 	°|ñ 	rc   c            	     óD   <€ SP                  SS\        V 4      SSSR 7      # )r¢  )r�  rŠ   )r°  r>  rÿ  Úruntime_attention_maskÚruntime_kwargsÚruntime_position_idsr�   s   *€€€€€€rd   Ú_forwardÚ?ParoQuantProcessor._forward_group_batch_train.<locals>._forwardj  s2   ø€ Ø×,Ñ,ØØ'Ü  Ó0Ø+Ø5Ø1ð -ó ð rc   Úuse_reentrantF)r  r�  rê  rŽ   r©   r›   Útorch_checkpointr‹   )r�   rÿ  r>  rð  rj   ro   rn   r‹  r°  r¶  r³  r´  rµ  s   ffd$$$$   @@@rd   Ú_forward_group_batch_trainÚ-ParoQuantProcessor._forward_group_batch_trainM  sà   ý€ ð ×3Ñ3×5Ò5¿[Ø×,Ñ,ØØ'Ø'Ø)Ø-Ø)ð -ó ð ñ #ó
á"�ð ×.Ñ.Ô/@×/RÑ/RÐSV×XbÑXbÓ/cÖdÙ"ð 	ð 
ð ×;Ñ;¸LÓIˆØ!%×!CÑ!CÀNÓ!SÐØ#×AÑAÀ,ÓOÐ÷	õ 	ô   ÐV¬5°Ó+@ÒVÐPUÑVÐVùò%
s   Á 7Cc               ó(   € V ^8„  d   QhRRRRRRRR/# r�  rW   )rv   s   "rd   rw   rÜ   v  s8   € ÷ \ñ \àð\ð ,ð	\ð
 ð\ð 
ñ\rc   c               óÞ   a aaa€ S P                  4       '       d   SP                  '       g   S P                  SSSR7      # R VVVV 3R llp\        V.\	        SP                  4      O5RR/ # )zUOptionally checkpoint streamed grouped train forwards to reduce activation residency.©r‘  r’  c               ó    € V ^8„  d   QhRRRR/# r¯  rW   )rv   s   "rd   rw   ÚDParoQuantProcessor._forward_replay_batch_train.<locals>.__annotate__…  s   € ÷ 	ñ 	 lð 	°|ñ 	rc   c                 óÀ   <€ SP                  S\        \        V 4      SP                  SP                  SP
                  SP                  SP                  R 7      SR7      # )rÀ   r¾  )r—  rf   rŠ   rj   rl   rn   ro   rq   )r°  r’  rÿ  r‘  r�   s   *€€€€rd   r¶  Ú@ParoQuantProcessor._forward_replay_batch_train.<locals>._forward…  s`   ø€ Ø×-Ñ-ØÜ2Ü Ó/Ø!-×!:Ñ!:Ø'×.Ñ.Ø!-×!:Ñ!:Ø#/×#>Ñ#>Ø*×4Ñ4ôð *ð .ó ð rc   r¸  F)r  rh   r—  r¹  r‹   )r�   rÿ  r‘  r’  r¶  s   ffdd rd   Ú_forward_replay_batch_trainÚ.ParoQuantProcessor._forward_replay_batch_trainv  sm   û€ ð ×3Ñ3×5Ò5¸\×=P×=PÐ=PØ×-Ñ-ØØ)Ø)ð .ó ð ÷	ó 	ô   Ð[¬5°×1DÑ1DÓ+EÒ[ÐUZÑ[Ð[rc   c               ó    € V ^8„  d   QhRRRR/# )rs   rE  úList[dict[str, object]]ru   rW   )rv   s   "rd   rw   rÜ   –  s   € ÷ 8!ñ 8!Ø-ð8!à	 ñ8!rc   c                óŽ  € . p/ p\        4       pV  EF/  pVP                  R. 4      p\        V\        P                  4      '       d   V.p. pV FZ  p\        V\        P                  4      '       g   K%  \        V4      pWƒ9   d   K8  VP                  V4       VP                  V4       K\  	  V'       g   K¥  \        VR,          4      p	\        VP                  RR4      4      p
\        ;QJ d$    . R VP                  RR4       4       F  NK  	  5M! R VP                  RR4       4       4      pV^ ,          V^,          3p\        VP                  RR4      4      p\        VP                  R	R
4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR
4      4      pWšWÍWïVV3pVP                  V4      pVf>   \        V4      VV&   VP                  R\        V4      RV	RV
RVRVR	VRVRVRV/	4       EK  VV,          R,          P                  V4       EK2  	  V# )zWMerge equivalent optimizer groups so grouped optimization pays less optimizer overhead.rL  Úlrr  rª  c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   )r…  )r~   Úbetas   & rd   r€   ÚMParoQuantProcessor._normalize_group_optimizer_param_groups.<locals>.<genexpr>²  s   é € Ð\Ñ6[¨dœe DŸk˜kÓ6[ùr‚   r  r  r¯  r  Fr  r´  r   r!  rÄ  )r_   r¯   r„   r   rr  r­   r_  r<  r…  r‹   rT   rÉ   rŠ   r   )rE  Únormalized_groupsÚgroup_index_by_keyÚseen_param_idsÚparam_groupÚ
raw_paramsrL  rv  Úparam_idrÈ  r  Ú	betas_objr  r  r  r  r   r!  r»   Úbucket_indexs   &                   rd   Ú'_normalize_group_optimizer_param_groupsÚ:ParoQuantProcessor._normalize_group_optimizer_param_groups•  s  € ð
 68ÐØsuÐÜ#&£5ˆä'ˆKØ$Ÿ™¨°2Ó6ˆJÜ˜*¤b§l¡l×3Ò3Ø(˜\�
à)+ˆFÛ#�Ü! %¬¯©×6Ò6ÙÜ˜e›9�ØÔ-ÙØ×"Ñ" 8Ô,Ø—‘˜eÖ$ñ $÷ Ùä�{ 4Õ(Ó)ˆBÜ  §¡°ÀÓ!FÓGˆLßœÑ\°k·o±oÀgÈ{Ô6[Ó\Ÿ™Ñ\°k·o±oÀgÈ{Ô6[Ó\Ó\ˆIØ˜q•\ 9¨Q¥<Ð0ˆEÜ˜Ÿ™¨¨uÓ5Ó6ˆCÜ˜;Ÿ?™?¨9°eÓ<Ó=ˆGÜ˜[Ÿ_™_¨Z¸Ó=Ó>ˆHÜ˜kŸo™o¨k¸3Ó?Ó@ˆIÜ˜KŸO™O¨J¸Ó>Ó?ˆHØ U°ÀIÈxÐXˆCà-×1Ñ1°#Ó6ˆLØÒ#Ü*-Ð.?Ó*@Ð" 3Ñ'Ø!×(Ñ(à ¤$ v£,Ø˜bØ&¨Ø Ø˜sØ! 7Ø" HØ# YØ" Hð
÷ð " ,Õ/°Õ9×@Ñ@À×Hñ] (ð` !Ð rc   r¥  r§  c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rs   rÌ  rÆ  r›   r”   r¥  r®   ru   útorch.optim.OptimizerrW   )rv   s   "rd   rw   rÜ   Ñ  s2   € ÷ 
ñ 
Ø2ð
ð ð
ð ð	
ð
 
ñ
rc   c               ó    € \        V VVRR7      # )zMConstruct the grouped stage optimizer after redundant groups are merged away.F)r›   r¥  Úgraph_capture)r0   )rÌ  r›   r¥  s   &$$rd   Ú_build_group_optimizerÚ)ParoQuantProcessor._build_group_optimizerÐ  s   € ô )ØØØ)Øô	
ð 	
rc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   rÌ  rÆ  r›   r”   ru   r×  rW   )rv   s   "rd   rw   rÜ   à  s(   € ÷ 

ñ 

Ø2ð

ð ð

ð 
ñ	

rc   c               ó2   € \         P                  V VRR7      # )zRBackward-compatible wrapper for tests that still exercise the AdamW path directly.r§  ©r›   r¥  )rÒ   rÚ  )rÌ  r›   s   &$rd   Ú_build_group_adamwÚ%ParoQuantProcessor._build_group_adamwß  s%   € ô "×8Ñ8ØØØ"ð 9ó 
ð 	
rc   c               ó$   € V ^8„  d   QhRRRRRR/# )rs   Ú	state_keyr®   Úactive_prefixesútuple[str, ...]ru   rT   rW   )rv   s   "rd   rw   rÜ   í  s'   € ÷ lñ l°Sð lÈ?ð lÐ_cñ lrc   c                óx   a € \         ;QJ d    V 3R lV 4       F  '       g   K   R# 	  R# ! V 3R lV 4       4      # )z@Match state keys that belong to the active grouped modules only.c              3  óf   <"  € T F&  pSV8H  ;'       g    SP                  V R 24      x € K(  	  R# 5i)rÞ  N)r§  )r~   Úprefixrâ  s   & €rd   r€   ÚGParoQuantProcessor._group_state_key_matches_prefixes.<locals>.<genexpr>ï  s5   øé € ÐkÑ[jÐQW�9 Ñ&×LÐL¨)×*>Ñ*>À&ÀÈ¸|Ó*LÔLÓ[jùs   ƒ1”1TF)rˆ   )râ  rã  s   f&rd   Ú!_group_state_key_matches_prefixesÚ4ParoQuantProcessor._group_state_key_matches_prefixesì  s0   ø€ ÷ ‹sÔkÑ[jÓk�sŒsÐkŠsÐkˆsÔkÑ[jÓkÓkÐkrc   r“   Útarget_dtypec          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )rs   rÿ  rA  rã  rä  r“   r–  rë  zOptional[torch.dtype]ru   údict[str, torch.Tensor]rW   )rv   s   "rd   rw   rÜ   ñ  s<   € ÷ 
ñ 
àð
ð )ð	
ð
 .ð
ð ,ð
ð 
!ñ
rc   c          
     ó  € VP                  4       P                  4        UUu/ uFÑ  w  rVV P                  WR4      '       g   K  TVfJ   Ve(   VP                  4       '       d   VP                  V8X  d   VP                  4       P                  4       MdVP                  4       P                  Ve   TMVP                  Ve   VP                  4       '       d   TMVP                  R7      P                  4       bKÓ  	  upp# u uppi )zOCapture only the mutable grouped module state instead of the whole layer clone.ro  )	Ú
state_dictrº   ré  Úis_floating_pointrp  r8  rÅ  r¨   r›   )r�   rÿ  rã  r“   rë  r»   rÁ   s   &&$$$  rd   Ú_snapshot_group_best_stateÚ-ParoQuantProcessor._snapshot_group_best_stateñ  sé   € ð$  %×/Ñ/Ó1×7Ñ7Ô9ô
ñ  :‘�Ø×5Ñ5°c×KôˆCà Ò(¨lÒ.BÈ&×JbÑJb×JdÒJdÐhn×htÑhtð  yEô  iEð —‘“×%Ñ%Ô'à—]‘]“_×'Ñ'Ø,9Ò,E™=È6Ï=É=Ø*6Ò*BÀv×G_ÑG_×GaÒGa™,Ðgm×gsÑgsð (ó ÷ ‘%“'òñ  :ò
ð 	
ùó 
s   ¢C=ÁC=Á!A2C=Ã%C=c               ó$   € V ^8„  d   QhRRRRRR/# )rs   rÿ  rA  Ú
best_staterí  ru   r—   rW   )rv   s   "rd   rw   rÜ     s(   € ÷ .ñ .Øð.ð ,ð.ð 
ñ	.rc   c               ó  € V'       g   R# V P                  RR7      p\        P                  ! 4       ;_uu_ 4        VP                  4        F  w  r4W#,          P	                  V4       K  	  RRR4       R#   + '       g   i     R# ; i)z3Restore the captured grouped module state in-place.NT)Ú	keep_vars)rï  r…   Úno_gradrº   Úcopy_)rÿ  rô  Ú
live_stater»   rÁ   s   &$   rd   Ú_restore_group_best_stateÚ,ParoQuantProcessor._restore_group_best_state  sY   € ÷ ÙØ×%Ñ%°Ð%Ó5ˆ
Ü�]Š]�_�_Ø)×/Ñ/Ö1‘�Ø•×%Ñ% fÖ-ñ  2÷ �_�_Š_ús   º1A5Á5B	c               óT   € V ^8„  d   QhRRRRRRRRR	R
RRRRRRRRRR
RRRRRRRRRR/# )rs   rÿ  rA  r‰  rš  Úinput_batches_trainrg  Úinput_kwargs_trainrp  Útarget_batches_trainry  Úposition_ids_trainr  Úattention_masks_trainÚinput_batches_valÚinput_kwargs_valÚtarget_batches_valÚposition_ids_valÚattention_masks_valrE  rÆ  Úepochsrp   ru   r™  rW   )rv   s   "rd   rw   rÜ     s±   € ÷ @2ñ @2àð@2ð 8ð	@2ð
 6ð@2ð 1ð@2ð (ð@2ð 9ð@2ð  <ð@2ð 4ð@2ð /ð@2ð &ð@2ð 7ð@2ð :ð@2ð .ð@2ð  ð!@2ð" 
ñ#@2rc   c               óž	  € \        V P                  P                  RR7       \        \	        V P                  RR4      4      pV P                  V4      p\        VP                  4       4      P                  pVP                  R8H  p\        VV4      ;_uu_ 4        V^ 8:  g	   V'       g?   V P                  VVVVVVVR7      pV P                  VVV	V
VVVR7      pVV3uuRRR4       # V P                  VVVR7      p\        ^V\        ^\        V4      4      ,          4      pVP                   Uu. uF  p\!        VR	,          4      NK  	  pp\"        P$                  P'                  VR
7      p\)        VP+                  4       4      p\,        ;QJ d*    R VP/                  4        4       F  '       g   K   RM	  RM! R VP/                  4        4       4      p\1        \	        V P                  RR4      VR7      pRp\!        R4      pRp^ p \3        V4       EFT  p!Rp"^ p#VP5                  RR7       \7        \9        VVVVV4      4       EFÎ  w  p$w  p%p&p'p(p)V'       d    \"        P$                  P;                  R4      M	\=        4       p*V*;_uu_ 4        V P?                  VV$V%V&V)V(R7      p+V PA                  V'V+P                  V+PB                  R7      p,\D        PF                  ! V+V,4      p-RRR4       VPI                  X-4      PK                  4        VPM                  V4       VPO                  4        VP5                  RR7       V ^,          p RR\P        PR                  ! \P        PT                  \W        V V4      ,          V,          4      ,           ,          p.\9        VP                  V4       F.  w  pp/V/R,          V/V/R,          ,
          V.,          ,           VR	&   K0  	  V PY                  V4       V"\!        V-P[                  4       4      ,          p"V#^,          p#EKÑ  	  V"\        ^V#4      ,          pV P                  VVV	V
VVVR7      pVe   VV8  g   EK>  TpV P]                  VVVR7      pEKW  	  Ve   V P_                  VVR7       V PY                  V4       VV3uuRRR4       # u upi   + '       g   i     ELÌ; i  + '       g   i     R# ; i)zHRun one grouped optimization stage against preserved full-layer outputs.r¿  r/  r¦  r§  r¢   )r÷  r"  rx  rn   r  r   NrÞ  rÈ  ©Úenabledc              3  óL   "  € T F  qP                   P                  x € K  	  R # 5irz   ©ru  rp  ©r~   rŠ  s   & rd   r€   Ú6ParoQuantProcessor._run_group_stage.<locals>.<genexpr>M  ó   é € Ð#pÑYoÈ×$6Ñ$6×$DÖ$DÓYoùrì  TFr½  r¾  ©r¼  r›   Úinfr´  ©Úset_to_noner¢  ro  ç      à?ç      ð?ç      4@)rã  rë  ©rô  )0r*   rÔ   rÒ  r+   rè   rÔ  rN  rO  r›   r¥   r(   rª  rÚ  rœ   rÉ   rE  r…  r…   r£  Ú
GradScalerr‹   r–  rˆ   r‰   r.   rÈ   Ú	zero_gradrú  rQ  r¤  r   rº  r  rp  r¥  r¦  ÚscaleÚbackwardÚstepr|  ÚmathÚcosÚpirÊ   r�  rK  rñ  rú  )0r�   rÿ  r‰  rý  rþ  rÿ  r   r  r  r  r  r  r  rE  r  r¥  rÌ  Ú
opt_devicer   r×  r„  Ú	optimizerÚtotal_stepsrU  Úbase_lrsÚscalerrã  Úneeds_angle_resetr¼  rô  Úbest_val_lossÚlast_train_lossÚglobal_stepÚ_epochÚ
epoch_lossÚbatch_countr>  rð  rj   r  r%  r&  r¨  r©  rl   ÚlossÚcosine_ratioÚbase_lrs0   &&$$$$$$$$$$$$$                                 rd   Ú_run_group_stageÚ#ParoQuantProcessor._run_group_stage  s  € ô& 	˜DŸI™I×4Ñ4¸LÕIÜ1´'¸$¿)¹)À_ÐV]Ó2^Ó_ˆØ ×HÑHÈÓVÐä˜%×*Ñ*Ó,Ó-×4Ñ4ˆ
Ø—/‘/ VÑ+ˆÜ# EÐ+<×=Õ=Ø˜Œ{×"3Ø!×7Ñ7ØØ"5Ø);Ø#7Ø!3Ø$9Ø#ð 8ó �
ð  ×5Ñ5ØØ"3Ø)9Ø#5Ø!1Ø$7Ø#ð 6ó �ð " 8Ð+÷) >Ò=ð, ×3Ñ3Ø!Ø!Ø-ð 4ó ˆIô
 ˜a ¬#¨a´Ð5HÓ1IÓ*JÕ!JÓKˆKØ8A×8NÒ8NÓOÑ8N¨uœ˜e D�kÖ*Ñ8NˆHÐOÜ—Y‘Y×)Ñ)°'Ð)Ó:ˆFÜ# M×$6Ñ$6Ó$8Ó9ˆOß #£Ñ#pÐYf×YmÑYmÔYoÓ#p§§¢Ñ#pÐYf×YmÑYmÔYoÓ#pÓ pÐÜAÜ!(¨¯©Ð4JÈFÓ!SØ!ô Ðð =AˆJÜ! %›LˆMØ!ˆOØˆKä Ÿ-�Ø �
Ø�Ø×#Ñ#°Ð#Ô5äbkÜØ+Ø*Ø,Ø*Ø-ó÷cÑ^�KÑ!^ +¨|¸\È7ÐT]÷ BI¤5§9¡9×#5Ñ#5°fÔ#=ÌkËm�Lß%šØ $× ?Ñ ?Ø!Ø(3Ø(3Ø)5Ø+4Ø)0ð !@ó !˜ð "&×!;Ñ!;¸LÐQV×Q]ÑQ]Ðej×epÑepÐ!;Ó!q˜Ü ×/Ò/°°vÓ>˜÷ &ð —L‘L Ó&×/Ñ/Ô1Ø—K‘K 	Ô*Ø—M‘M”OØ×'Ñ'°DÐ'Ô9à 1Õ$�KØ#&¨#´·²¼¿¹Ä3À{ÐT_ÓC`Õ9`ÐcnÕ9nÓ0oÕ*oÕ#p�LÜ*-¨i×.DÑ.DÀhÖ*O™˜˜wØ'.°¥~¸7ÀgÐPTÅnÕ;UÐYeÕ:eÕ&f˜˜d›ñ +Pð ×,Ñ,¨]Ô;Ø¤%¨¯	©	«Ó"4Õ4�JØ 1Õ$“KñEcðH #-¬s°1°kÓ/BÕ"B�Ø×5Ñ5ØØ"3Ø)9Ø#5Ø!1Ø$7Ø#ð 6ó �ð Ò%¨°M×)AØ$,�MØ!%×!@Ñ!@ØØ(7Ø%5ð "Aó "“Jñk (ðv Ò%Ø×.Ñ.¨uÀÐ.ÔLØ×$Ñ$ ]Ô3Ø" MÐ1÷O >Ò=ùò8 P÷8 &Ÿ˜ú÷q >×=Ð=úsT   ÂR;Â)3R;Ã'AR;Ä/R"Å
A%R;Æ2CR;ÊAR'Ë$E'R;ÑAR;Ò"R;Ò'R8Ò2	R;Ò;S	r�   c               ó0   € V ^8„  d   QhRRRRRRRRR	R
RR/# )rs   rÿ  rA  rO   r’   r   rT   r“   r”   r�   r–   ru   r…  rW   )rv   s   "rd   rw   rÜ   —  sG   € ÷ $8ñ $8àð$8ð 4ð	$8ð
 ð$8ð $ð$8ð Fð$8ð 
ñ$8rc   c               óÆ  € V'       g   R# Rp\        VV^VR7      pV'       d    \        P                  P                  R4      M	\	        4       p\        P
                  ! 4       ;_uu_ 4        T;_uu_ 4        VP                  4        F‡  p	V	^ ,          p
V P                  VV
RR7      pV P                  V
P                  VP                  VP                  RR7      pV\        \        P                  ! W¼4      P                  4       4      ,          pK‰  	  RRR4       RRR4       V\!        ^\#        V4      4      ,          #   + '       g   i     L4; i  + '       g   i     L?; i)zWMeasure full-layer reconstruction error while streaming one validation batch at a time.r´  ©r“   r•   r�   r¢   Fr¾  ©r›   rp  r°   N)rŽ   r…   r£  r¤  r   r§   rÎ   r—  r  rl   r›   rp  r…  r¥  r¦  rK  rœ   rÉ   )r�   rÿ  rO   r   r“   r�   r§  Úloaderr¨  rÍ   r‘  r©  rl   s   &&$$$$       rd   Ú_evaluate_group_layer_streamedÚ1ParoQuantProcessor._evaluate_group_layer_streamed—  s  € ÷ Ùàˆ
Ü"ØØ'ØØ)ô	
ˆ÷ 6=”u—y‘y×)Ñ)¨&Ô1Ä+Ã-ˆÜ×!Ò!×#Õ#§\¢\Ø×+Ñ+Ö-�Ø$ Q�x�Ø×2Ñ2ØØ!-Ø!&ð 3ó �ð
 ×3Ñ3Ø ×'Ñ'Ø Ÿ<™<ØŸ+™+Øð	 4ó �ð œe¤A×$4Ò$4°UÓ$C×$HÑ$HÓ$JÓKÕK’
ñ .÷ &2×#ð œC ¤3 ~Ó#6Ó7Õ7Ð7÷ &2§\ú×#×#ús%   Á*EÁ5BD=	ÄEÄ=EÅEÅE 	c               ó8   € V ^8„  d   QhRRRRRRRRRR	R
RRRRR/# )rs   rÿ  rA  r‰  rš  Úreplay_batches_trainr’   Úreplay_batches_valrE  rÆ  r  rp   r�   r–   ru   r™  rW   )rv   s   "rd   rw   rÜ   ½  se   € ÷ |2ñ |2àð|2ð 8ð	|2ð
 :ð|2ð 8ð|2ð .ð|2ð ð|2ð Fð|2ð 
ñ|2rc   c               ó
  € \        V P                  P                  RR7       \        \	        V P                  RR4      4      pV P                  V4      p	\        VP                  4       4      P                  p
V
P                  R8H  p\        W4      ;_uu_ 4        V^ 8:  g	   V	'       g:   V P                  VVVV
VR7      pV P                  VVVV
VR7      pWÍ3uuRRR4       # V P                  V	V
VR7      p\        ^V\        ^\        V4      4      ,          4      pVP                   Uu. uF  p\!        VR	,          4      NK  	  pp\"        P$                  P'                  VR
7      p\)        VP+                  4       4      p\,        ;QJ d*    R VP/                  4        4       F  '       g   K   RM	  RM! R VP/                  4        4       4      p\1        \	        V P                  RR4      \2        R7      pRp\!        R4      pRp^ pV P5                  VV	VR7      p\7        V4       EFp  pRp^ pVP9                  RR7       \;        VV
VVR7      pV'       d    \"        P$                  P=                  R4      M	\?        4       pT;_uu_ 4        VPA                  4        EFœ  p V  EF‘  p!V PC                  VV!RR7      p"V PE                  V!PF                  V"P                  V"PH                  RR7      p#\J        PL                  ! V"V#4      p$VPO                  V$4      PQ                  4        VPS                  V4       VPU                  4        VP9                  RR7       V^,          pRR\V        PX                  ! \V        PZ                  \]        VV4      ,          V,          4      ,           ,          p%\_        VP                  V4       F.  w  pp&V&R,          V&V&R,          ,
          V%,          ,           VR	&   K0  	  V'       d   V Pa                  V4       V\!        V$Pc                  4       4      ,          pV^,          pEK”  	  EKŸ  	  RRR4       V\        ^V4      ,          pV P                  VVVV
VR7      pVe   VV8  g   EKU  TpV Pe                  VV\2        VR7      pEKs  	  Ve   V Pg                  VVR7       V'       d   V Pa                  V4       VV3uuRRR4       # u upi   + '       g   i     Lª; i  + '       g   i     R# ; i)zYRun one grouped layer stage while streaming train shards and validation batches from CPU.r¿  r/  r¦  r§  r¢   )rO   r   r“   r�   NrÞ  rÈ  r	  c              3  óL   "  € T F  qP                   P                  x € K  	  R # 5irz   r  r  s   & rd   r€   Ú?ParoQuantProcessor._run_group_stage_streamed.<locals>.<genexpr>ê  r  rì  TFr½  r¾  r  r  r´  )rE  rO   r  r3  r¾  r4  r  r  r  )rã  r“   rë  r  )4r*   rÔ   rÒ  r+   rè   rÔ  rN  rO  r›   r¥   r(   r6  rÚ  rœ   rÉ   rE  r…  r…   r£  r  r‹   r–  rˆ   r‰   r.   r@   r\  rÈ   r  rŽ   r¤  r   rÎ   rÃ  r  rl   rp  r¥  r¦  r  r  r  r|  r  r  r  rÊ   rQ  r�  rK  rñ  rú  )'r�   rÿ  r‰  r9  r:  rE  r  r�   r¥  rÌ  r   r   r×  r„  r!  r"  rU  r#  r$  rã  r%  r¼  rô  r&  r'  r(  r•   r)  r*  r+  Útrain_loaderr¨  rÍ   r‘  r©  rl   r,  r-  r.  s'   &&$$$$$$                               rd   Ú_run_group_stage_streamedÚ,ParoQuantProcessor._run_group_stage_streamed½  s‰  € ô 	˜DŸI™I×4Ñ4¸LÕIÜ1´'¸$¿)¹)À_ÐV]Ó2^Ó_ˆØ ×HÑHÈÓVÐä˜%×*Ñ*Ó,Ó-×4Ñ4ˆ
Ø—/‘/ VÑ+ˆÜ# E×=Õ=Ø˜Œ{×"3Ø!×@Ñ@ØØ#7Ø#Ø",Ø#1ð Aó �
ð  ×>Ñ>ØØ#5Ø#Ø",Ø#1ð ?ó �ð "Ð+÷! >Ò=ð$ ×3Ñ3Ø!Ø!Ø-ð 4ó ˆIô
 ˜a ¬#¨a´Ð5IÓ1JÓ*KÕ!KÓLˆKØ8A×8NÒ8NÓOÑ8N¨uœ˜e D�kÖ*Ñ8NˆHÐOÜ—Y‘Y×)Ñ)°'Ð)Ó:ˆFÜ# M×$6Ñ$6Ó$8Ó9ˆOß #£Ñ#pÐYf×YmÑYmÔYoÓ#p§§¢Ñ#pÐYf×YmÑYmÔYoÓ#pÓ pÐÜAÜ!(¨¯©Ð4JÈFÓ!SÜô Ðð =AˆJÜ! %›LˆMØ!ˆOØˆKØ ×;Ñ;ØØ.Ø3ð <ó ˆMô   Ÿ-�Ø �
Ø�Ø×#Ñ#°Ð#Ô5Ü0Ø(Ø",Ø"/Ø#1ô	 �÷ >EœuŸy™y×1Ñ1°&Ô9Ì+Ë-�ß!’\Ø!-×!9Ñ!9×!;˜Ü,1˜LØ$(×$DÑ$DØ %Ø-9Ø-2ð %Eó %˜Eð
 &*×%?Ñ%?Ø ,× 3Ñ 3Ø',§|¡|Ø&+§k¡kØ&+ð	 &@ó &˜Fô $%×#3Ò#3°E¸6Ó#B˜Dà"ŸL™L¨Ó.×7Ñ7Ô9Ø"ŸK™K¨	Ô2Ø"ŸM™MœOØ%×/Ñ/¸DÐ/ÔAà'¨1Õ,˜KØ+.°#¼¿ºÄÇÁÌ3È{Ð\gÓKhÕAhÐkvÕAvÓ8wÕ2wÕ+x˜LÜ25°i×6LÑ6LÈhÖ2W¡  wØ/6¸­~À7ÈgÐX\ÍnÕC]ÐamÕBmÕ.n  d£ñ 3X÷  1Ø $× 8Ñ 8¸Ô GØ&¬%°·	±	³Ó*<Õ<˜JØ'¨1Õ,›Kô7 -2ñ "<÷ "ð> #-¬s°1°kÓ/BÕ"B�Ø×>Ñ>ØØ#5Ø#Ø",Ø#1ð ?ó �ð Ò%¨°M×)AØ$,�MØ!%×!@Ñ!@ØØ(7Ü&)Ø%5ð	 "Aó "“Jñk (ðx Ò%Ø×.Ñ.¨uÀÐ.ÔLß Ø×(Ñ(¨Ô7Ø" MÐ1÷U >Ò=ùò0 P÷< "—\ú÷m >×=Ð=ús_   ÂS0Â(.S0Ã!AS0Ä)SÅA%S0Æ,CS0Ê	E-SÏ7ASÐ;9S0Ñ:>S0Ò9S0ÓS0ÓS-Ó(S0Ó0T	c               ó$   € V ^8„  d   QhRRRRRR/# ©rs   r  rC   r|  rL  ru   ztuple[dict[str, object], float]rW   )rv   s   "rd   rw   rÜ   ;  s2   € ÷ \Dñ \Dà#ð\Dð )ð\Dð 
)ñ	\Drc   c                ó$	  € V'       g   \        R4      hVP                  f   \        R4      hVP                  pV^ ,          P                  P                  pV^ ,          P                  P
                  pVP                  V\        P                  R7      pV P                  V4       V P                  W24       V P                  V4      pV P                  V4      w  rx/ p	/ p
/ pV P                  4       p VP                  4        F  pVP                  R4       K  	  V FY  p\!        W>P"                  4      W¾P"                  &   V P%                  V4      p\'        W>P"                  V4       WúVP"                  &   K[  	  V P)                  V4       T P+                  TT
TTV
P-                  4        Uu. uF*  pRVP.                  .RV P0                  P2                  /VCNK,  	  upV
P-                  4        Uu. uF*  pRVP4                  .RV P0                  P2                  /VCNK,  	  up,           \7        V P0                  P8                  4      V	R7       V
P-                  4        F  pVP;                  4        K  	  T P+                  TT
TTV
P-                  4        Uu. uF*  pRVP                  .RV P0                  P<                  /VCNK,  	  upV
P-                  4        Uu. uFG  pVP>                  f   K  RVP>                  PA                  4       RV P0                  PB                  /VCNKI  	  up,           \7        V P0                  PD                  4      V	R7      w  ppV	PG                  4        / pV F1  pW®P"                  ,          p\I        VVVRR7      VVP"                  &   K3  	  VV3\K        V4       F:  pVPM                  VP"                  4      pVf   K$  \'        W>P"                  V4       K<  	  VP                  WER7       V PO                  V4       # u upi u upi u upi u upi   \K        T4       F:  pTPM                  TP"                  4      pTf   K$  \'        Y>P"                  T4       K<  	  TP                  YER7       T PO                  T4       i ; i)	z@Optimize the live layer in place for full-layer ParoQuant scope.r~  zKParoQuantProcessor layer-scope optimization requires the live layer module.ro  FrL  rÈ  )r‰  r9  r:  rE  r  r�   ©r×  r„  Úused_identity)(r  rF   rU  rf  r›   rp  r¨   r…   r~  rE  rU  rg  r,  r"  rO  Úrequires_grad_r<   rW  r6  r=   ry  r?  r‰   Úchannel_scales_optrÔ   rÎ  ru  rp   rÌ  Úinit_quantizerrÏ  Ú	quantizerÚoptim_paramsrÐ  rÍ  rŸ  r/   rk  r¯   rl  )r�   r  r|  rÿ  r“   Úoriginal_layer_dtypeÚattn_impl_overridesr9  r:  r�   r‰  Úoriginal_modulesÚoptimizer_group_kwargsrv  rS  rŠ  r×  r„  ÚresultsÚoriginal_modules   &&&                 rd   Ú_optimize_live_layerÚ'ParoQuantProcessor._optimize_live_layer;  sf  € ÷ ÜÐdÓeÐeØ×ÑÒ%ÜÐlÓmÐmà×"Ñ"ˆØ% aÕ(×/Ñ/×6Ñ6ˆØ,¨QÕ/×6Ñ6×<Ñ<ÐØ—‘ ´U·]±]�ÓCˆØ×*Ñ*¨5Ô1Ø×.Ñ.¨uÔDØ"×?Ñ?ÀÓFÐØ37×3RÑ3RÐSXÓ3YÑ0ÐØ>@ˆØ:<ˆØ79ÐØ!%×!CÑ!CÓ!EÐðE	DØ×)Ñ)Ö+�Ø×$Ñ$ UÖ+ñ ,ó !.�Ü6EÀe×M^ÑM^Ó6_Ð ×!2Ñ!2Ñ3Ø#×=Ñ=¸lÓK�Ü ×'8Ñ'8¸,ÔGØ3?˜l×/Ñ/Ó0ñ	 !.ð ×9Ñ9¸%Ô@à×*Ñ*ØØ+Ø%9Ø#5ð )6×(<Ñ(<Ô(>óá(>˜ð  × ?Ñ ?Ð@À$ÈÏ	É	×HaÑHaÐ|Ðe{Ó|Ù(>ñð
 )6×(<Ñ(<Ô(>óá(>˜ð  × 2Ñ 2Ð3°T¸4¿9¹9×;TÑ;TÐoÐXnÓoÙ(>ñõô ˜4Ÿ9™9×8Ñ8Ó9Ø-ð +ô ð  !.× 4Ñ 4Ö 6�Ø×+Ñ+Ö-ñ !7ð $(×#AÑ#AØØ+Ø%9Ø#5ð )6×(<Ñ(<Ô(>óá(>˜ð  × 3Ñ 3Ð4°d¸D¿I¹I×<SÑ<SÐnÐWmÓnÙ(>ñð
 )6×(<Ñ(<Ô(>óá(>˜Ø#×-Ñ-ô B�X˜|×5Ñ5×BÑBÓDÀdÈDÏIÉI×LfÑLfð  Bð  kAó  BÙ(>ñõô ˜4Ÿ9™9×8Ñ8Ó9Ø-ð $Bó $Ñ ˆJ˜ð" × Ñ Ô"à)+ˆGÛ -�Ø,×->Ñ->Õ?�Ü-?Ø Ø)Ø%Ø"'ô	.�˜×)Ñ)Ó*ñ !.ð ˜HÐ$ä (¨Ö 7�Ø"2×"6Ñ"6°|×7HÑ7HÓ"I�Ø"Ô.Ü# E×+<Ñ+<¸oÖNñ !8ð �H‰H˜MˆHÔFØ×.Ñ.Ð/BÕCùòiùòùò ùòøô, !)¨Ö 7�Ø"2×"6Ñ"6°|×7HÑ7HÓ"I�Ø"Ô.Ü# E×+<Ñ+<¸oÖNñ !8ð �H‰H˜MˆHÔFØ×.Ñ.Ð/BÕCúsQ   Ã1B9P! Æ*0PÇP! Ç.0PÈA5P! Ê0PËP! ËPË,8PÌ$A<P! ÐP! Ð!+RÑ>Rc               ó    € V ^8„  d   QhRRRR/# )rs   r|  rL  ru   rT   rW   )rv   s   "rd   rw   rÜ   š  s   € ÷ >ñ >Ð2Cð >Èñ >rc   c                ó”  a€ V '       g   R# RpRp^ p^ pV  Fž  p\        VRR4      oSP                  R^4      R	,          p\        ;QJ d    V3R lV 4       F  '       g   K   RM	  RM! V3R lV 4       4      '       g   SP                  V4      '       d   V^,          pK‹  V^,          pVR
9  g   K�   R# 	  V^ 8„  ;'       d    V^ 8H  # )a4  Restrict in-place layer optimization to dense decoder blocks.

The official apples-to-apples path is one full dense decoder layer at a
time. Expert-heavy layers can contain hundreds or thousands of modules,
so keep those on the cloned grouped path until a dedicated MoE/shared
rotation implementation lands.
FrW  rˆ  rÞ  c              3  ó,   <"  € T F	  qS9   x € K  	  R # 5irz   rW   )r~   Úmarkerr$  s   & €rd   r€   Ú@ParoQuantProcessor._supports_live_layer_scope.<locals>.<genexpr>°  s   øé € ÐF±~¨V˜[Ö(³~ùs   ƒT)ÚexpertÚexpertsÚshared_expertÚgate_up_proj)zexperts.zmlp.experts.zmoe.rŠ  >   Úk_projÚo_projÚq_projÚv_projÚup_projÚ	down_projÚ	gate_proj)rè   rà  rˆ   r§  )r|  Úexpert_markersÚexpert_prefixesÚexpert_like_modulesÚdense_modulesrß   Úleafr$  s   &      @rd   Ú_supports_live_layer_scopeÚ-ParoQuantProcessor._supports_live_layer_scope™  sÄ   ø€ ÷ Ùð
ˆð ?ˆØÐØˆÛ#ˆFÜ! &¨&°"Ó5ˆKØ×%Ñ% c¨1Ó-¨bÕ1ˆDß‹sÔF±~ÓF�s�sŠsÔF±~ÓF×FÒFÈ+×J`ÑJ`Ðap×JqÒJqØ# qÕ(Ð#ÙØ˜QÕˆMØÐhÖhÚñ $ð ˜qÑ ×=Ð=Ð%8¸AÑ%=Ð=rc   c               ó$   € V ^8„  d   QhRRRRRR/# rB  rW   )rv   s   "rd   rw   rÜ   ¸  s-   € ÷ X%ñ X%à#ðX%ð )ðX%ð 
)ñ	X%rc   c                óî  € V P                  4       R8X  dz   V P                  V4      '       dc   \        P                  ! R4      ;_uu_ 4        \        P                  ! 4       ;_uu_ 4        V P                  W4      uuRRR4       uuRRR4       # \        P                  ! R4      ;_uu_ 4        \        P                  ! 4       ;_uu_ 4        V P                  W4      w  r4VP                  4        F  pVP                  R4       K  	  V P                  V\        VP                  4       4      P                  4      w
  pppp	p
pppppV P                  4       pT P                  TTTTTT	T
TTTTTVP                  4        Uu. uF*  pRVP                  .RV P                   P"                  /VCNK,  	  upVP                  4        Uu. uF*  pRVP$                  .RV P                   P"                  /VCNK,  	  up,           \'        V P                   P(                  4      R7       VP                  4        F  pVP+                  4        K  	  T P                  TTTTTT	T
TTTTTVP                  4        Uu. uF*  pRVP,                  .RV P                   P.                  /VCNK,  	  upVP                  4        Uu. uFG  pVP0                  f   K  RVP0                  P3                  4       RV P                   P4                  /VCNKI  	  up,           \'        V P                   P6                  4      R7      w  pp/ pV F2  pVVP8                  ,          p\;        VVVRR7      VVP8                  &   K4  	  VV3uuRRR4       uuRRR4       #   + '       g   i     M; iRRR4       EK    + '       g   i     EL ; iu upi u upi u upi u upi   + '       g   i     M; iRRR4       R#   + '       g   i     R# ; i)zXOptimize one compute_block or whole-layer group against the preserved full-layer target.rÿ  FNrL  rÈ  )r‰  rý  rþ  rÿ  r   r  r  r  r  r  r  rE  r  rD  )rå  rh  r…   r§   rÇ  rQ  r‹  rO  rF  r„  rN  r›   r"  r/  r‰   rG  rÔ   rÎ  ru  rp   rÌ  rH  rf  rÏ  rI  rJ  rÐ  rÍ  rW  r/   )r�   r  r|  rˆ  r‰  rv  rý  rþ  rÿ  r   r  r  r  r  r  r  rN  rŠ  r×  r„  rO  rS  s   &&&                   rd   Ú_optimize_groupÚ"ParoQuantProcessor._optimize_group¸  s²  € ð ×ÑÓ! WÔ,°×1PÑ1PÐQ^×1_Ò1_Ü×%Ò% e×,Õ,¬e×.?Ò.?×.AÕ.AØ×0Ñ0°ÓF÷ /BÑ.A×,Ò,ô ×!Ò! %×(Õ(¬%×*;Ò*;×*=Õ*=Ø)-×)FÑ)FÀuÓ)\Ñ&ˆKØ$×/Ñ/Ö1�Ø×$Ñ$ UÖ+ñ 2ð ×.Ñ.¨u´d¸;×;QÑ;QÓ;SÓ6T×6[Ñ6[Ó\ñØ#Ø"Ø$Ø"Ø%Ø!Ø Ø"Ø Ø#à%)×%GÑ%GÓ%IÐ"à×!Ñ!ØØ+Ø$7Ø#5Ø%9Ø#5Ø&;Ø"3Ø!1Ø#5Ø!1Ø$7ð )6×(<Ñ(<Ô(>óá(>˜ð  × ?Ñ ?Ð@À$ÈÏ	É	×HaÑHaÐ|Ðe{Ó|Ù(>ñð
 )6×(<Ñ(<Ô(>óá(>˜ð  × 2Ñ 2Ð3°T¸4¿9¹9×;TÑ;TÐoÐXnÓoÙ(>ñõô ˜4Ÿ9™9×8Ñ8Ó9ð) "ô ð. !.× 4Ñ 4Ö 6�Ø×+Ñ+Ö-ñ !7ð $(×#8Ñ#8ØØ+Ø$7Ø#5Ø%9Ø#5Ø&;Ø"3Ø!1Ø#5Ø!1Ø$7ð )6×(<Ñ(<Ô(>óá(>˜ð  × 3Ñ 3Ð4°d¸D¿I¹I×<SÑ<SÐnÐWmÓnÙ(>ñð
 )6×(<Ñ(<Ô(>óá(>˜Ø#×-Ñ-ô B�X˜|×5Ñ5×BÑBÓDÀdÈDÏIÉI×LfÑLfð  Bð  kAó  BÙ(>ñõô ˜4Ÿ9™9×8Ñ8Ó9ð+ $9ó $Ñ ˆJ˜ð0 *,ˆGÛ -�Ø,¨\×->Ñ->Õ?�Ü-?Ø Ø)Ø%Ø"'ô	.�˜×)Ñ)Ó*ñ !.ð ˜HÐ$÷] +>Ñ*=×(Ò(÷ /B×.Aú×,×,×,Ó,üòFùòùò.ùò÷{ +>×*=ú×(×(×(Ò(ús‡   ÁNÁ)N 	Á:
NÂ-O#ÃB3O	Å>0N2(Æ.O	Ç0N7*Ç2A<O	É.0N<(ÊO	Ê2O*Ë8O*Ë?A,O	Í+
O#Î NÎNÎN/	Î2O	ÏOÏO#Ï#O4	c               ó    € V ^8„  d   QhRRRR/# )rs   r$  r®   ru   rW   )rv   s   "rd   rw   rÜ   	  s   € ÷ 'ñ '°ð '¸ñ 'rc   c                ó†   € V P                  R^4      R	,          pVR
9   d   R# VR8X  d   R# VR9   d   R# VR8X  d   R# RV  2# )zGMap common projection archetypes to compute_block optimization buckets.rÞ  Úattn_qkvr]  Úattn_oÚmlp_gate_upra  Úmlp_downzsingle:rŠ  >   r\  r^  r_  >   r`  rb  rß  )r$  rg  s   & rd   Ú_module_compute_block_labelÚ.ParoQuantProcessor._module_compute_block_label	  sV   € ð ×!Ñ! # qÓ)¨"Õ-ˆØÐ1Ô1ÙØ�8ÔÙØÐ+Ô+Ù Ø�;ÔÙØ˜˜Ð&Ð&rc   c               ó    € V ^8„  d   QhRRRR/# )rs   r$  r®   ru   ztuple[int, str]rW   )rv   s   "rd   rw   rÜ   !	  s   € ÷ 3ñ 3°ð 3¸ñ 3rc   c                ó|   € V P                  R^4      R	,          pR^ R^R^R^R^R^R^/pVP                  V^d4      V 3# )
z<Keep compute_block members in canonical architectural order.rÞ  r^  r\  r_  r]  rb  r`  ra  rŠ  )rà  r¯   )r$  rg  Úorders   &  rd   Ú_module_compute_block_orderÚ.ParoQuantProcessor._module_compute_block_order 	  s]   € ð ×!Ñ! # qÓ)¨"Õ-ˆà�aØ�aØ�aØ�aØ˜Ø�qØ˜ð
ˆð —	‘	˜$ Ó$ kÐ2Ð2rc   c               ó    € V ^8„  d   QhRRRR/# )rs   r  rC   ru   z#list[tuple[str, list[NamedModule]]]rW   )rv   s   "rd   rw   rÜ   /	  s"   € ÷ eñ eà#ðeð 
-ñerc   c                óx  a € S P                  4       p\        VP                  4       Uu. uF  q1P                  V,          NK  	  ppVR8X  d   V Uu. uF  qUP                  V.3NK  	  up# VR8X  d�   / pV F=  pVP	                  S P                  VP                  4      . 4      P                  V4       K?  	  V F  pWg,          P                  V 3R lR7       K!  	  \        V4       Uu. uF  qwWg,          3NK  	  up# VR8X  d   RV3.# \        RS P                  P                   R24      hu upi u upi u upi )zâResolve the optimization scope for the current layer.

`module` keeps today's per-linear behavior. `compute_block` and `layer`
are scaffolded here so the lifecycle can switch scopes explicitly once
their execution paths land.
rß   r  c                ó:   <€ SP                  V P                  4      # rz   )ry  rW  )rß   r�   s   &€rd   rL  ÚCParoQuantProcessor._optimization_groups_for_layer.<locals>.<lambda>B	  s   ø€ °t×7WÑ7WÐX^×XcÑXcÔ7drc   rN  rÿ  z0ParoQuantProcessor: unsupported optimize scope `r&  )rå  rP  rE   rW  r;  rt  r<  Úsortr  rÔ   rÞ   )r�   r  ÚmoderW  rK  rß   ÚgroupedÚlabels   f&      rd   Ú_optimization_groups_for_layerÚ1ParoQuantProcessor._optimization_groups_for_layer/	  s&  ø€ ð ×#Ñ#Ó%ˆÜ9?ÀÇÁÔ9NÓOÑ9N°Ÿ™ t×,Ð,Ñ9NˆÐOØ�8ÔÙ:GÓH¹-°—[‘[ 6 (Ó+¹-ÑHÐHØ�?Ô"Ø46ˆGÛ'�Ø×"Ñ" 4×#CÑ#CÀFÇKÁKÓ#PÐRTÓU×\Ñ\Ð]cÖdñ (ã �Ø•×#Ñ#Ô(dÐ#Öeñ !ä9?À¼ÓI¹°˜G�NÓ+¹ÑIÐIØ�7Œ?Ø˜mÐ,Ð-Ð-ÜÐKÈDÏIÉI×L_ÑL_ÐK`Ð`bÐcÓdÐdùò PùâHùò Js   ©D-ÁD2Ã)D7c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r-  rp   r  rC   ru   r—   rW   )rv   s   "rd   rw   rÜ   H	  s-   € ÷ RAñ RA¨3ð RAÐ7Kð RAÐPTñ RArc   c                óŽ
  € VP                   '       d   R# V P                  V4      pV P                  4       pV P                  V4      pVP	                  4        F=  w  rgVP                  4       ^ 8X  g   K  V P                  '       d   K0  \        RV R24      h	  VR8X  d°   \        VP                  P	                  4       4       F†  w  rhVP                  V4      p	V	f   \        P                  ! ^ 4      p	\        P                  ! 4       p
V P                  W‰4      w  r¼\        P                  ! 4       V
,
          pV P!                  W‰WÍ4       Kˆ  	  EMÛVP"                  e   VP$                  f&   \        RT RV UUu. uF  w  rïVNK	  	  upp 24      hV EF“  w  pp\        P                  ! 4       p
V P'                  VV4      w  pp\        P                  ! 4       V
,
          pV\)        ^\+        V4      4      ,          pV FÀ  pV P-                  V4      P/                  4       P1                  4       pVVP2                  ,          pV P5                  VVV4       VR8X  d   \7        VP8                  \:        R7       VP                  VP2                  4      p	V	f   \        P                  ! ^ 4      p	V P!                  W‰VV4       KÂ  	  VR	8X  g   EK9  \=        V P>                  R
R4      '       g   EKY  V'       d#   V P-                  V^ ,          4      P@                  MRp\C        VRRR7       EK–  	  RVn         V PD                  ;_uu_ 4        \        VP                  4       FS  pV PF                  P                  V4      pVf   K$  VP                  R4      V8X  g   K<  . VR&   VPI                  RR4       KU  	  RRR4       VP                  PK                  4        VPL                  PK                  4        VPN                  PK                  4        RVn        RVn(        RVn        RVn)        RVn*        RVn+        RVn,        RVn-        RVn.        RVn/        \a        V R4      '       d   V Pb                  PK                  4        \a        V R4      '       d   V Pd                  PK                  4        \a        V R4      '       d   V Pf                  PK                  4        \a        V R4      '       d   V Ph                  PK                  4        \a        V R4      '       d   V Pj                  PK                  4        R# R# u uppi   + '       g   i     EL‹; i)zEQuantize every captured module in a layer once all subsets are ready.Nz;ParoQuantProcessor error: missing activation features for `z` with fallback disabled.rß   zoParoQuantProcessor grouped optimization requires preserved layer inputs and outputs. Resolved groups for layer z: rÿ  rž  r  Úoffload_to_diskFT)r›   ÚgcÚsyncr-  rh   r5  rÈ  rÅ  r
  r°  r»  )6rU   rƒ  rå  r[  rº   r‹  rù   rU  rŠ   rE   r¯   r…   rR  ÚtimeÚperf_counterrÙ  r–  rJ   rL   rl  rœ   rÉ   rj  r8  rÅ  rW  r€  r9   rß   r@   rè   rÔ   r›   rA   rV   r9  rT  rŸ  rS   rR   rK   rG   rH   rI   rM   rN   rO   rQ   r`  rÈ  rÅ  r
  r°  r»  )r�   r-  r  Úoptimization_groupsr€  Ú
input_featr$  rÁ   rS  rƒ  rË   Ú_train_lossr„  r†  r‚  Ú_modulesÚ_group_labelr|  Úgroup_resultsÚgroup_val_lossÚduration_per_modulerm  r  Úflush_devicer=  s   &&&                      rd   Ú_quantize_layerÚ"ParoQuantProcessor._quantize_layerH	  sY  € à�?�?ˆ?Ùà"×AÑAÀ%ÓHÐØ×#Ñ#Ó%ˆà×/Ñ/°Ó6ˆ
Ø#-×#3Ñ#3Ö#5ÑˆKØ�|‰|‹~ Ö"¨4¯=¯=ª=Ü"ØQÐR]ÐQ^Ð^wÐxóð ñ $6ð �8ÔÜ-1°%·-±-×2EÑ2EÓ2GÖ-HÑ)�Ø!—~‘~ kÓ2�Ø’<Ü Ÿ;š; q›>�Dä×)Ò)Ó+�Ø(,×(AÑ(AÀ,Ó(UÑ%�Ü×,Ò,Ó.°Õ6�Ø×&Ñ& |¸8ÖNó .Ið ×!Ñ!Ò)¨U×-@Ñ-@Ò-HÜ"ð1Ø1<°¸RÑ^qÔ@rÑ^qÉ?È5ÃÑ^qÒ@rÐ?sðuóð ô
 0CÑ+�˜mÜ×)Ò)Ó+�Ø04×0DÑ0DÀUÈMÓ0ZÑ-�˜~Ü×,Ò,Ó.°Õ6�Ø&.´°Q¼¸MÓ8JÓ1KÕ&KÐ#ã$1�LØ&*×&@Ñ&@ÀÓ&N×&UÑ&UÓ&W×&]Ñ&]Ó&_�OØ*¨<×+<Ñ+<Õ=�FØ×3Ñ3°LÀ&È/ÔZØ˜w”Ü × 3Ñ 3¼CÕ@Ø%Ÿ>™>¨,×*;Ñ*;Ó<�DØ’|Ü$Ÿ{š{¨1›~˜Ø×*Ñ*¨<¸~ÐObÖcñ %2ð ˜?×*¬w°t·y±yÐBSÐUZ×/[Õ/[ßZg 4×#=Ñ#=¸mÈAÕ>NÓ#O×#VÒ#VÐmq�LÜ%¨\¸eÈ$×OÐOñ' 0Cð* ˆŒØ�Y�Y‹YÜ# E§M¡MÖ2�ØŸ
™
Ÿ™ {Ó3�ØÔ$¨¯©°=Ó)AÀ[Ö)PØ&(�E˜(‘OØ—I‘IÐ3°TÖ:ñ	  3÷ ð 	�‰×ÑÔØ×Ñ×#Ñ#Ô%Ø×Ñ×%Ñ%Ô'Ø!ˆÔØ#'ˆÔ Ø"ˆÔØ&*ˆÔ#Ø-1ˆÔ*Ø7;ˆÔ4Ø $ˆÔØ*.ˆÔ'Ø#ˆÔØ!ˆÔÜ�4Ð6×7Ò7Ø×,Ñ,×2Ñ2Ô4Ü�4Ð8×9Ò9Ø×.Ñ.×4Ñ4Ô6Ü�4Ð.×/Ò/Ø×$Ñ$×*Ñ*Ô,Ü�4Ð4×5Ò5Ø×*Ñ*×0Ñ0Ô2Ü�4ÐB×CÒCØ×8Ñ8×>Ñ>Ö@ñ Dùói As÷2 �YˆYús   Å8T-Í5T3Í>T3ÎT3Ô3U	c               ó   € V ^8„  d   QhRR/# )rs   rß   r   rW   )rv   s   "rd   rw   rÜ   œ	  s   € ÷ Bñ B ñ Brc   c                óX  € V P                   P                  VP                  R7      RJ d   R# \        W P                   P                  4      V n        V P                  VP                  4      pVP                  ;_uu_ 4        WP                  VP                  &   VP                  P                  RVP                  4      Vn        VP                  P                  VP                  4       RRR4       V P                  VP                  VP                  4       R#   + '       g   i     L8; i)zIRegister a module for later activation capture and deferred quantization.)Ú
layer_nameFNrF   )rÔ   r'  ri  r4   rù   r/  r-  rV   rE   rW  r  r¯   rF   rS   r_  rC  )r�   rß   rù   rs  Úlayer_states   &&&, rd   Ú
preprocessÚParoQuantProcessor.preprocessœ	  sÐ   € à�9‰9× Ñ ¨F×,<Ñ,<Ð Ó=ÀÓFÙä*¨8·Y±Y×5GÑ5GÓHˆŒØ×+Ñ+¨F×,>Ñ,>Ó?ˆØ××ÓØ/5×Ñ §¡Ñ,Ø'-§|¡|×'7Ñ'7¸È×H`ÑH`Ó'aˆKÔ$Ø×'Ñ'×+Ñ+¨F¯K©KÔ8÷ ð
 	× Ñ  §¡¨f×.@Ñ.@ÖA÷ ×ús   Â A)DÄD)	c               ó    € V ^8„  d   QhRRRR/# )rs   rß   r   ru   rT   rW   )rv   s   "rd   rw   rÜ   ª	  s   € ÷ ;ñ ; ð ;°ñ ;rc   c                óR   € V P                   P                  VP                  R4      RJ # )zDReport whether a module has been excluded from ParoQuant processing.F)r9  r¯   rW  )r�   rß   s   &&rd   Ú
is_skippedÚParoQuantProcessor.is_skippedª	  s    € à�z‰z�~‰~˜fŸk™k¨5Ó1°UÐ:Ð:rc   c               ó    € V ^8„  d   QhRRRR/# )rs   rW  r®   ru   z@Callable[[Module, Tuple[torch.Tensor, ...], torch.Tensor], None]rW   )rv   s   "rd   rw   rÜ   ®	  s   € ÷ 	ñ 	¨ð 	Ð1qñ 	rc   c                ó   a a€ R VV 3R llpV# )z>Capture input activations during the calibration forward pass.c               ó    € V ^8„  d   QhRRRR/# )rs   r‹  zTuple[torch.Tensor, ...]Úoutrk   rW   )rv   s   "rd   rw   Ú=ParoQuantProcessor.pre_process_fwd_hook.<locals>.__annotate__°	  s   € ÷ 	6ñ 	6Ð6ð 	6¸\ñ 	6rc   c                ó’   <€ ? ?V'       g   R # \        V\        \        34      '       d
   V^ ,          MTpSP                  SV4       R # rz   )r„   r‹   rŠ   r?  )rß   r‹  r¤  r2  rW  r�   s   &&& €€rd   ÚhookÚ5ParoQuantProcessor.pre_process_fwd_hook.<locals>.hook°	  s9   ø€ Ø˜ßÙÜ *¨3´¼°× >Ò >�c˜!–fÀCˆGØ×&Ñ& t¨WÖ5rc   rW   )r�   rW  r§  s   ff rd   Úpre_process_fwd_hookÚ'ParoQuantProcessor.pre_process_fwd_hook®	  s   ù€ ÷	6ñ 	6ð ˆrc   c               ó0   € V ^8„  d   QhRRRRRRRRRR	R
R	/# )rs   rß   r   r›   r”   Úsubsetz Optional[Dict[str, NamedModule]]Úprevious_subsetÚsubset_indexrP   rQ   rW   )rv   s   "rd   rw   rÜ   ¹	  sF   € ÷ 9ñ 9àð9ð ð9ð 1ð	9ð
 :ð9ð $ð9ð $ñ9rc   c                ó  € ??VP                   pV P                  V4      pVP                  ;_uu_ 4        Ve   VP                  P	                  V4       Ve   Whn        Ve   VP                  P                  V4       VP                  P                  VP                  4       VP                  '       * ;'       d–    \        VP                  4      ;'       dy    VP                  P                  VP                  P                  4       4      '       * ;'       d9    VP
                  RJ ;'       g#    \        VP                  4      VP
                  8¬  p	V	'       d   V P!                  Wx4       RRR4       R#   + '       g   i     R# ; i)zKMark subset progress and quantize once the whole layer is capture-complete.N)r-  r/  rV   rE   r|  rQ   rR   r_  rS   ÚdiscardrW  rU   rT   Úintersectionr–  rÉ   r•  )
r�   rß   r›   r¬  r­  r®  rQ   r-  r  Úshould_quantizes
   &&&&&&&   rd   ÚprocessÚParoQuantProcessor.process¹	  s6  € ð �OØ×(Ñ(ˆØ×%Ñ% kÓ2ˆà�Z�Z‹ZØÒ!Ø—‘×$Ñ$ VÔ,ØÒ'Ø%1Ô"ØÒ'Ø×'Ñ'×+Ñ+¨LÔ9à×!Ñ!×)Ñ)¨&¯+©+Ô6ð —O‘OÔ#÷ gð gÜ˜Ÿ™Ó'÷gð gà×-Ñ-×:Ñ:¸5¿=¹=×;MÑ;MÓ;OÓPÔP÷gð gð ×'Ñ'¨4Ð/×eÐe´3°u×7NÑ7NÓ3OÐSX×SeÑSeÑ3eð	 ÷ Ø×$Ñ$ [Ô8÷# �Z�ZŠZús*   ´BE3Â:E3Ã?E3ÄE3Ä-*E3ÅE3Å3F	r®  rQ   c               ó4   € V ^8„  d   QhRRRRRRRRRR	R
R	RR/# )rs   r-  rp   rJ   r%  rK   zList[Dict[str, torch.Tensor]]rL   r®  rP   rQ   ru   r—   rW   )rv   s   "rd   rw   rÜ   Ú	  sR   € ÷ 2ñ 2ð ð2ð /ð	2ð
 :ð2ð 0ð2ð $ð2ð $ð2ð 
ñ2rc   c               ó4  € ?V P                  V4      pVP                  ;_uu_ 4        VP                  f   W'n        VP                  f   W7n        VP                  f   WGn        Ve   VP
                  f   Wgn        RRR4       R#   + '       g   i     R# ; i)z=Preserve noisy grouped inputs plus clean float layer targets.N)r/  rV   rJ   rK   rL   rQ   )r�   r-  rJ   rK   rL   r®  rQ   r  s   &$$$$$$ rd   Úreceive_layer_forward_contextÚ0ParoQuantProcessor.receive_layer_forward_contextÚ	  sy   € ð Ø×%Ñ% kÓ2ˆØ�Z�Z‹ZØ×!Ñ!Ò)Ø%1Ô"Ø×'Ñ'Ò/Ø+=Ô(Ø×"Ñ"Ò*Ø&3Ô#ØÒ'¨E×,>Ñ,>Ò,FØ%1Ô"÷ �Z�ZŠZús   §ABÂB	c               ó$   € V ^8„  d   QhRRRRRR/# )rs   r-  rp   rF   rA  ru   r—   rW   )rv   s   "rd   rw   rÜ   ñ	  s/   € ÷ Yñ Yð ðYð &ð	Yð
 
ñYrc   c               ó2  € V P                  4       R8X  d   R# V P                  V4      pVP                  ;_uu_ 4        VP                  f0   \        P
                  ! V4      P                  \        R7      Vn        RRR4       R#   + '       g   i     R# ; i)zKPreserve an untouched float layer snapshot for grouped optimization clones.rÿ  Nrž  )rå  r/  rV   rG   r¤   r€  r¨   r@   )r�   r-  rF   r  s   &$$ rd   Úreceive_pristine_layer_moduleÚ0ParoQuantProcessor.receive_pristine_layer_moduleñ	  si   € ð ×ÑÓ! WÔ,ÙØ×%Ñ% kÓ2ˆØ�Z�Z‹ZØ×*Ñ*Ò2Ü.2¯mªm¸LÓ.I×.LÑ.LÔTWÐ.LÓ.X�Ô+÷ �Z�ZŠZús   ½>BÂB	c               ó    € V ^8„  d   QhRRRR/# ©rs   rß   r   rî   r   rW   )rv   s   "rd   rw   rÜ   ÿ	  s   € ÷ .ñ .¨ð .¸Zñ .rc   c                ó*   € V P                  WR7       R# )z@Pack one optimized float module into its ParoQuant runtime form.)rî   N)r;   )r�   rß   rî   rs  s   &&&,rd   Úsubmodule_finalizeÚ%ParoQuantProcessor.submodule_finalizeÿ	  s   € à×Ñ˜ÐÖ-rc   c               ó    € V ^8„  d   QhRRRR/# r¾  rW   )rv   s   "rd   rw   rÜ   
  s   € ÷ Mñ M +ð M°jñ Mrc   c                ó2  € VP                  4        V P                  ;_uu_ 4        VP                  P                  RR4       VP                  P                  RR4       VP                  P                  R4      P	                  4       pVP                  P                  R4      P	                  4       pVP                  P                  R4      P	                  4       pVP                  P                  R4      P	                  4       pVP                  P                  R4      P	                  4       pVP                  P                  R	4      P	                  4       pRRR4       \        X\        R
7      VP                  n        V P                  VP                  4      p	VP                  VP                  P                  VP                  4      /p
\        VR\        VRR4      4      p\        R\        VR7      ;_uu_ 4        \!        VP                  4      ;_uu_ 4        \#        VP                  V	V P$                  P&                  V P$                  P(                  V P$                  P*                  V P$                  P,                  VP                  VV P$                  P.                  V P$                  P0                  VP2                  V P$                  P4                  V P6                  RV P$                  P9                  4       R7       RRR4       RRR4       VP                  P                  VP                  4      p\;        WÉ4      '       d   VP                  V/M/ p\        R\        VR7      ;_uu_ 4        \!        VP                  4      ;_uu_ 4        \=        VP                  VXXRV
V	V P                  V P$                  R7	       RRR4       RRR4       WÑP                  ,          p\;        V\>        4      '       g0   \A        RVP                   R\C        V4      PD                   R24      hVPF                  PI                  XPK                  VPF                  P0                  VPF                  PL                  R7      4       VPN                  PI                  XPK                  VPN                  P0                  VPN                  PL                  R7      4       VPP                  PI                  XPK                  VPP                  P0                  VPP                  PL                  R7      4       VPS                  4        R#   + '       g   i     ELê; i  + '       g   i     ELS; i  + '       g   i     EL_; i  + '       g   i     ELº; i  + '       g   i     ELÆ; i)z@Replace a float module with a packed ParoQuant quantized module.rw  Nry  rq  rs  rr  rt  ru  rv  rž  ri  rW  rˆ  r7   )Úloggerr$  F)rW  Ú
linear_clsr_  Údesc_actÚdynamicr  rß   Ú	submoduler`  r›   Úlm_head_nameÚ
pack_dtyperv   Úregister_buffersÚinit_kwargsÚpack)	rW  ÚqModulesrr  rs  Úq_g_idxÚlayersÚquant_linear_clsrV   Úquantize_configz
Expected `z#` to be packed as ParoLinear, got `r&  ro  )*Ústream_syncrV   r  rT  rÅ  r9   r@   rf  rg  r*  ri  rî   Úget_submodulerè   r5   r  r>   r7   rÔ   ra  rÆ  rÇ  r  r`  r›   Úlm_headrÊ  rv   Úquant_linear_init_kwargsr„   r;   r"   rÒ  r¥   rX   rt  rø  r¨   rp  ru  rv  Ú	post_init)r�   rß   rî   rq  rs  rr  rt  ru  rv  rÑ  rÐ  Úmodule_labelÚqmoduleÚqmoduless   &&&           rd   r;   ÚParoQuantProcessor.pack_module
  s  € à×ÑÔØ�Y�Y‹YØ�L‰L×Ñ˜[¨$Ô/Ø�L‰L×Ñ˜T 4Ô(Ø Ÿ,™,×*Ñ*¨=Ó9×?Ñ?ÓAˆKØ—l‘l×&Ñ& yÓ1×7Ñ7Ó9ˆGØ—|‘|×'Ñ'¨
Ó3×9Ñ9Ó;ˆHØ—L‘L×$Ñ$ WÓ-×3Ñ3Ó5ˆEØ—L‘L×$Ñ$ WÓ-×3Ñ3Ó5ˆEØ#Ÿ\™\×-Ñ-Ð.>Ó?×EÑEÓGˆN÷ ô % [¼Ô=ˆ�‰ÔØ×7Ñ7¸×8HÑ8HÓIÐØ×"Ñ" E§K¡K×$=Ñ$=¸f×>NÑ>NÓ$OÐPˆÜ˜v {´G¸FÀFÈBÓ4OÓPˆäØ!ÜØ$÷
ö 
ô
 $ F×$4Ñ$4×5Õ5Ü#Ø×)Ñ)Ø/ØŸ™×/Ñ/Ø!ŸY™Y×/Ñ/Ø ŸI™I×-Ñ-Ø#Ÿy™y×3Ñ3Ø Ÿ;™;Ø$ØŸ	™	Ÿ™ØŸ9™9×+Ñ+Ø!&§¡Ø#Ÿy™y×3Ñ3ØŸ;™;Ø%*Ø $§	¡	× BÑ BÓ Dõ÷ 6÷
ð0 —+‘+×+Ñ+¨F×,<Ñ,<Ó=ˆô ˜'×4Ò4ð ×Ñ˜wÑ'àð 	ô
 ØÜØ$÷
ö 
ô
 $ F×$4Ñ$4×5Õ5ÜØ×)Ñ)Ø%Ø%Ø#Ø Ø!Ø%5ØŸ™Ø$(§I¡Iõ
÷ 6÷
ð$ ×+Ñ+Õ,ˆÜ˜'¤:×.Ò.ÜØ˜V×-Ñ-Ð.Ð.QÔRVÐW^ÓR_×RhÑRhÐQiÐikÐlóð ð 	�‰×Ñ˜EŸH™H¨G¯M©M×,@Ñ,@ÈÏÉ×H[ÑH[˜HÓ\Ô]Ø�‰×Ñ˜EŸH™H¨G¯M©M×,@Ñ,@ÈÏÉ×H[ÑH[˜HÓ\Ô]Ø×Ñ×$Ñ$Ø×Ñ W×%;Ñ%;×%BÑ%BÈ'×J`ÑJ`×JfÑJfÐÓgô	
ð 	×ÑÖ÷U �YˆYú÷( 6×5Ð5ú÷
÷ 
ð 
ú÷F 6×5Ð5ú÷
÷ 
ð 
úsU   ¥D/T5Ç:UÈC'U		Ë?UÍ/VÎ3U1	Ï VÔ5U	Õ	UÕ	UÕU.	Õ1VÕ<	VÖV	c               ó   € V ^8„  d   QhRR/# )rs   rî   r   rW   )rv   s   "rd   rw   rÜ   R
  s   € ÷ 0ñ 0˜jñ 0rc   c                óx   <€ RVn         \        P                  VP                  n        \
        SV `  ! RRV/VB  R# )zFMark the model as ParoQuant-quantized before shared finalization work.Trî   NrW   )rU   r$   ÚPAROrÒ  rï   rë   Úfinalize)r�   rî   rs  rû   s   &&,€rd   rß  ÚParoQuantProcessor.finalizeR
  s1   ø€ àˆŒÜ'-§{¡{ˆ×ÑÔ$Ü‰ÒÑ/˜uÐ/¨Ô/rc   c               ó    € V ^8„  d   QhRRRR/# )rs   Úprocessor_indexrp   ru   rT   rW   )rv   s   "rd   rw   rÜ   X
  s   € ÷ ñ ¸#ð À$ñ rc   c                ó:   € ?V P                   f   \        R4      hR# )zJRequire calibration data because ParoQuant always needs activation replay.z:ParoQuantProcessor's calibration_dataset must be provided.T)r  r  )r�   râ  s   &&rd   Úverify_calibration_datasetÚ-ParoQuantProcessor.verify_calibration_datasetX
  s    € àØ×#Ñ#Ò+ÜÐYÓZÐZÙrc   c               ó   € V ^8„  d   QhRR/# rø  rW   )rv   s   "rd   rw   rÜ   `
  s   € ÷ ñ �Sñ rc   c                ó   € R# )z#Return the processor registry name.Ú	paroquantrW   )Úclss   &rd   rW  ÚParoQuantProcessor.name_
  s   € ñ rc   c               ó    € V ^8„  d   QhRRRR/# )rs   rW  r®   ru   rT   rW   )rv   s   "rd   rw   rÜ   d
  s   € ÷ `ñ `¨3ð `°4ñ `rc   c                ó  € V P                   P                  V4      ;'       g    / pVP                  R. 4      pVRJ;'       dK    \        V4      ^ 8„  ;'       d5    \        ;QJ d    R V 4       F  '       d   K   R# 	  R# ! R V 4       4      # )zGReport whether non-empty activation batches were captured for a module.rh   Nc              3  óF   "  € T F  qP                  4       ^ 8„  x € K  	  R# 5irÆ   )r‹  )r~   Úts   & rd   r€   Ú<ParoQuantProcessor.has_captured_input_ids.<locals>.<genexpr>h
  s   é € Ð?_ÑW^ÐRSÇÁÃ	ÈAÆÓW^ùs   ‚!FT)r9  r¯   rÉ   rO  )r�   rW  r=  rX  s   &&  rd   Úhas_captured_input_idsÚ)ParoQuantProcessor.has_captured_input_idsd
  su   € à—
‘
—‘˜tÓ$×*Ð*¨ˆØ&+§i¡i°¸"Ó&=ˆØ˜dÐ"×_Ð_¤s¨7£|°aÑ'7×_Ð_¿C»CÑ?_ÑW^Ó?_¿C¼CÐ_ºCÐ_¸CÑ?_ÑW^Ó?_Ó<_Ð_rc   )r÷   rÈ  rÅ  rÉ  r°  r»  r
  rò   ró   rõ   rô   rö   rø   rì   rÚ   rù   rv   rí   rî   rñ   )TFNrz   )NNNNN)grX   rY   rZ   r[   r\   rž   rÿ   r  r	  r  rð   r!  r*  r/  r?  rC  r[  rb  rÐ   rj  r€  r–  rÙ  rá  ræ  rÆ  rå  r  r  r
  r  r  rÑ  r"  rþ   r+  r6  r>  rE  rU  rg  rl  ry  r‹  r“  rš  r¢  rª  r¶  rÁ  rä  rê  ró  rü  r  r  r  r  rÓ  r,  r1  r7  rB  r\  rb  r„  r�  r—  r�  rª  rº  rÃ  rÔ  rÚ  rß  ré  rñ  rú  r/  r6  r?  rQ  rh  rl  rt  ry  rƒ  r•  r›  rŸ  r©  r³  r·  r»  rÀ  r;   rß  rä  ÚclassmethodrW  rð  rb   Ú__classcell__)rû   s   @rd   rÒ   rÒ   Í   sý  ø† ÙO÷0&ò 0&õdõ
Kòaõuõ
õC÷&õõ;õ&
+õõ(%ð ôó ðõ +õDõ452ðn ô,ó ð,õ1õEõNõ õ2õ1õõlõ

Fõ
õ
Põ	:õ-8ð^ ôó ðõ	ð ôó ðð( ôó ðð2 ô2ó ð2ð
 ô1ó ð1õ@:*õxKð öó ðõ#ðJ ô~ó ð~õ#õ**ð<Sð ÷Sðj ôó ðð& ô	Aó ð	Aõ"õ""ð& ôó ððð ÷ð8 ôó ðð& ôó ðõBðH ô9ó ð9õõ
õ(MõT9õv6õp&õP+ðZ ô/ó ð/õ
 7õD'WõR\ð> ô8!ó ð8!ðt ð
ð &ö	
ó ð
ð ô

ó ð

ð ôló ðlð
ð
 15ð
ð /3÷
ð, ô.ó ð.õ@2ðD$8ð IM÷$8ðL|2ð IM÷|2õ|\Dð| ô>ó ð>õ<X%ðt ô'ó ð'ð ô3ó ð3õeõ2RA÷hBõ;õ	÷9ðB2ð '+ð2ð '+÷2õ.Yõ.õM÷^0ð 0õð ôó ð÷`ó `rc   rÒ   )fr\   Ú
__future__r   r¤   rñ  rÏ  r  r`   rŠ  Ú
contextlibr   Údataclassesr   r   Útypingr   r   r   r	   r
   r   r   r   r   r…   Útorch.nn.functionalr   Ú
functionalr¥  Útorch.nnr   Útorch.utils.checkpointr   r¹  Úlooper.loop_processorr   r   r   r   Úlooper.named_moduler   Úmodelsr   Úmodels.writerr   r   r   r   r   r   r   r   r    Únn_modules.hooked_linearr!   Únn_modules.qlinear.paroquantr"   Úquantization.configr#   r$   r%   r&   Ú#quantization.paroquant.optimizationr'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   Úutils.fallbackr4   Úutils.loggerr5   r6   Úutils.modelr7   r8   r9   r:   r;   r<   r=   Úutils.module_locksr>   Úutils.paroquantr?   Úutils.torchr@   rA   r  rC   rf   r|   rŽ   rÒ   rW   rc   rd   Ú<module>r
     sÿ   ðñõ #ã Û Û Û Û Û Ý "ß (ß V× VÕ Vã ß Ð Ý Ý Ý Aç lÓ lÝ -Ý ÷
÷ 
õ 
õ 4Ý 5ß VÓ V÷÷ ÷ õ õ 0ß 7÷÷ ñ õ 4Ý Bß 0áƒn€ð ÷Að Aó ðAð* ÷ð ó ðõ÷Hñ HôT[&`˜ö [&`rc   