+
    "Ý³j¹) ã                  óÖ  a € 0 t $ R t^ RIHt ^ RIt^ RIt^ RIt^ RIHtH	t	H
t
HtHtHt ^ RIt^ RIHt ^RIHt ^RIHt ^RIHt ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIH t  ^RI!H"t" ^RI#H$t$ ^RI%H&t& ^RI'H(t(H)t)H*t*H+t+ ^RI,H-t- ^RI.H/t/ ^RI0H1t1 ^RI2H3t3 ^RI4H5t5 ^RI6H7t7H8t8H9t9H:t:H;t; ^RI<H=t=H>t>H?t?H@t@HAtAHBtBHCtCHDtD ^RIEHFtF ^RIGHHtHHItI ^RIJHKtKHLtLHMtM ^RINHOtO ^RIPHQtQHRtRHStSHTtT ^RIUHVtV ^RIWHXtX ^RIYHZtZ ^RI[H\t\ ^R I]H^t^ ]5! 4       t_Rs`R!]aR"&   ]PÄ                  ! 4       tc]'       d   ^ R#IdHete  ! R$ R%]4      tfR& R' ltgR( R) lth ! R* R+]i4      tjR, R- ltk ! R. R/]V4      tlR# )0aæ  Utilities for orchestrating the quantisation loop across multiple devices.

ModuleLooper is the high-level coordinator that fans calibration batches across
the available accelerators, runs each processing stage, and keeps the shell and
turtle model state coherent. The implementation mixes synchronous orchestration
with asynchronous workers, so the helpers below focus on keeping device context
consistent and ensuring data dependencies survive the roundtrips through the
thread pool.
)ÚannotationsN)ÚTYPE_CHECKINGÚAnyÚDictÚListÚ
NamedTupleÚOptional)ÚDEVICE_THREAD_POOL)ÚDequantizeProcessor)ÚEoraProcessor)ÚGPTQProcessor)Ú
InputCache)ÚLoopProcessor)ÚNamedModule)Ú
BaseQModel)ÚSUPPORTS_MODULE_TYPES)ÚCAPTURE_ONLY_FLAG)ÚHookedLinear)ÚMETHODÚQuantizeEmbedÚQuantizeEmbedConfigÚVramStrategy)Úapply_keep_mask_bt)Ú
get_device)Úemit_device_telemetry)Úestimate_disk_io_speed)Úsetup_logger)Úclone_module_for_devicesÚforward_batch_workerÚnormalize_device_likeÚrehome_module_to_deviceÚselect_forward_devices)ÚMoETopKStateÚget_layers_with_prefixesÚ
get_moduleÚget_module_by_name_prefixÚmove_toÚrestore_moe_topkÚset_moe_topkÚuntie_word_embeddings)Úoffload_to_disk)Úhas_gil_controlÚhas_gil_disabled)ÚCPUÚMETAÚtf32_high_precision_guard)ÚAWQProcessor)ÚLoopContextÚLoopExtensionsÚLoopPlanÚLoopStep)ÚDeviceAssignmentState)ÚForwardExecutor)ÚParoQuantProcessor)ÚStageInputsCapture)Úrun_layer_stageúOptional[float]Ú_IO_WRITE_SPEED_MB)ÚProgressBarc                  ó:   € ] tR t^Ut$ RtR]R&   R]R&   R]R&   RtR	# )
ÚFinalizeProgressInfoz6Progress payload for processor finalization reporting.úOptional[str]Úmodule_labelÚstrÚprocess_nameúOptional[int]Ú	layer_idx© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__rE   ó    Ú_/home/luca/screen-checking/.venv/lib/python3.14/site-packages/gptqmodel/looper/module_looper.pyr>   r>   U   s   ‡ Ù@àÓØÓØ×rM   r>   c               ó$   € V ^8„  d   QhRRRRRR/# )é   Úmethodr   Úquant_devicesúList[torch.device]ÚreturnrE   )Úformats   "rN   Ú__annotate__rV   ]   s#   € ÷ ñ ¨sð ÐCUð ÐZlñ rM   c                ó   €  V e   \        V 4      MRpV\         P                  8w  g	   V'       g   V# V Uu. uF  p\	        VRR4      R8w  g   K  VNK  	  ppV'       d   V^ ,          .# VR,          #   \        \        3 d    Rp Lti ; iu upi )zBApply method-specific device constraints for quantization workers.NÚtypeÚcpu:Né   N)r   Ú	TypeErrorÚ
ValueErrorÚPAROÚgetattr)rQ   rR   Únormalized_methodÚdeviceÚnon_cpu_devicess   &&   rN   Ú"_restrict_quant_devices_for_methodrb   ]   s“   € ð!Ø.4Ò.@œF 6œNÀdÐð œFŸK™KÔ'¯}ØÐá,9Ód©M &¼WÀVÈVÐUYÓ=ZÐ^cÑ=c—v�v©M€OÐdßØ Õ"Ð#Ð#à˜ÕÐøô ”zÐ"ô !Ø Òð!üò es   ‚A2 ¶BÁBÁ2BÂBc               ó(   € V ^8„  d   QhRRRRRRRR/# )rP   Úconfigured_devicesúOptional[List[str]]Úavailable_devicesrS   ÚlabelrA   rT   rE   )rU   s   "rN   rV   rV   o   s0   € ÷ !ñ !Ø+ð!à)ð!ð ð	!ð
 ñ!rM   c               óü  € V '       g   \        V4      # V Uu/ uF0  p\        V4      f   K  \        \        V4      4      \        V4      bK2  	  pp. pV  Fˆ  p\        V4      pVf   \        RV RV: R24      hVP	                  \        V4      4      pVf-   \        RV RV  R\        VP                  4       4       R24      hW…9  g   Kw  VP                  V4       KŠ  	  V'       g   \        RV R24      hV# u upi )zLResolve one strategy device pool as a validated subset of available devices.zModuleLooper: z3 device pool contains an unsupported device value: Ú.z device pool z1 must be a subset of the visible compute devices z* device pool is empty after normalization.)Úlistr   rA   r\   ÚgetÚkeysÚappend)	rd   rf   rg   r`   Úavailable_by_nameÚresolvedÚdevice_nameÚ
normalizedÚmatcheds	   &&$      rN   Ú_resolve_strategy_device_poolrs   o   s!  € ÷ ÜÐ%Ó&Ð&ñ (óá'ˆFÜ  Ó(ô 	JŒÔ! &Ó)Ó*Ô,AÀ&Ó,IÒIÙ'ð ð ð
 $&€HÛ)ˆÜ*¨;Ó7ˆ
ØÒÜ˜~¨e¨WÐ4gÐhsÑgvÐvwÐxÓyÐyØ#×'Ñ'¬¨J«Ó8ˆØŠ?ÜØ    }Ð5GÐ4HÐHyÜÐ)×.Ñ.Ó0Ó1Ð2°!ð5óð ð Ö"Ø�O‰O˜GÖ$ñ *÷ Ü˜>¨%¨Ð0ZÐ[Ó\Ð\à€Oùò-s
   ˜C9¬"C9c                  ó   € ] tR t^“tRtRtR# )ÚStopMainLoopz5Signal that the module loop should abort immediately.rE   N)rF   rG   rH   rI   rJ   rL   rE   rM   rN   ru   ru   “   s   † Ý?rM   ru   c               ó   € V ^8„  d   QhRR/# )rP   rT   r:   rE   )rU   s   "rN   rV   rV   —   s   € ÷ ñ ˜oñ rM   c                 óÞ   € \         e   \         # \        ;_uu_ 4        \         e   \         uuRRR4       # \        4       p V R,          s RRR4       \         #   + '       g   i     \         # ; i)z;Estimate and cache sustained disk write throughput in MB/s.Ni   )r;   Ú_IO_WRITE_SPEED_LOCKr   )Ú
disk_speeds    rN   Úio_write_performancerz   —   sZ   € ô Ò%Ü!Ð!ß	Ö	ÜÒ)Ü%÷ 
Ò	ô ,Ó-ˆ
Ø'¨;Õ7Ð÷	 
ô
 Ð÷ 
Ö	ô
 Ðús   �A¶AÁA,	c                  ó  € ] tR t^¥t$ RtRtR]R&   RtR]R&   RkRRl/R R llltR	 R
 lt	 ! R R4      t
 ! R R4      tR R ltR R ltR R ltR R ltR tR tR R ltR R ltR R ltR R  ltR! R" ltR# R$ ltR% R& ltR' tR( R) ltR* R+ ltR, R- ltR. R/ ltR0 R1 ltR2 R3 ltR4 R5 lt R6 R7 lt!R8 R9 lt"R: R; lt#R< R= lt$RkR> R? llt%R@ RA lt&RB RC lt'RDR/RE RF llt(RDR/RG RH llt)RI RJ lt*RK RL lt+RM RN lt,RORRPRRQRRRRRSRRTRRURRVRWRXRWRYRZ/
R[ R\ llt-RQRRRRRSRRTRRURRXRWRYRZ/R] R^ llt.RQRRRRRSRRTRRURRYRZ/R_ R` llt/Ra Rb lt0Rc Rd lt1RkRe lt2RkRf lt3]4Pj                  ! 4       RkRg l4       t6RkRh Ri llt7Rjt8R# )mÚModuleLooperzùDrive the per-layer quantisation workflow over one or more devices.

The looper executes work on the shared global :class:`DeviceThreadPool`
instance so tasks such as module reloading, forward passes, and finalisation
reuse the same worker threads.
Nr?   Úinput_embeddings_nameÚoutput_embeddings_nameÚ
extensionsc               ó$   € V ^8„  d   QhRRRRRR/# )rP   Úmodelr   Ú
processorszList[LoopProcessor]Úembed_quant_configzOptional[QuantizeEmbedConfig]rE   )rU   s   "rN   rV   ÚModuleLooper.__annotate__¯   s-   € ÷ Jñ JàðJð (ðJð :ñ	JrM   c          
     óÎ  € W n         \        V4      V n        Wn        V'       d   VP                  MRV n        V'       d   VP
                  MRV n        VP                  V n        \        P                  ! 4       V n	        \        V \        R7      V n        \        VRR4      V n        \        P                  ! 4       V n        RV n        RV n        . V n        \        P                  ! 4       V n        \+        4       pVe_   V^d8  d   \        P-                  RVR R24       M<V^È8  d   \        P/                  RVR R24       M\        P1                  RVR R	24       \        V P                  P2                  R
R4      p\5        V4      pV'       d   \7        V4      M\8        .pV'       g   \8        .pV^ ,          V n        \        V P                  P2                  RR4      p	V	e1   V	! V4      p
\=        V
4      ^8¼  d   T
pM\        P/                  R4       \?        \        V P                  P2                  RR4      V4      pW¸8w  dI   \        P/                  RV^ ,          V Uu. uF  qÌV^ ,          8w  g   K  \A        V4      NK  	  up4       TpW€n!        ^ V n"        / V n#        \        P                  ! 4       V n$        \        V P                  P2                  R\J        PL                  4      p\O        V\@        4      '       d    \K        VPQ                  4       4      p\        V P                  R\J        PL                  \J        PT                  .4      p\O        V\J        4      '       d   V.pWÞ9  dZ   \        PW                  R\        V P                  R\Y        V P                  4      4      PZ                  V4       \J        PL                  p\        V P                  P2                  R\J        PL                  4      p\O        V\@        4      '       d    \K        VPQ                  4       4      p\        V P                  R\J        PL                  \J        PT                  .4      p\O        V\J        4      '       d   V.pVV9  dZ   \        PW                  R\        V P                  R\Y        V P                  4      4      PZ                  V4       \J        PL                  pWÐn.        Wðn/        \        V P                  P2                  RR4      p\        V P                  P2                  RR4      p\a        VVRR7      V n1        \a        VVRR7      V n2        \g        V4      ;'       g    V P\                  \J        PL                  8g  V n4        \g        V4      ;'       g    V P^                  \J        PL                  8g  V n5        ^V n6        \        V P                  RR4      V n7        RV n8        \        V P                  RR4      '       de   V P                  Ps                  V P                  Pt                  Pv                  4      pV P                  P2                  Py                  V4      V n<        MRV n<        V P                  P2                  P{                  4       V n=        V P}                  4        V P                    F  pV P                  V4       K  	  V P                  e/   \�        V P                  Pt                  4      V P                  n:        RV nA        RV nB        RV nC        RV nD        V P                  e÷   \        V P                  RR4      p\        V P                  RR4      p\‹        V4      '       d   V! 4       MRV nA        \‹        V4      '       d   V! 4       MRV nB        \        V P                  RR4      p\        V P                  RR4      pV P‚                  e   \‹        V4      '       d   V! 4       MRV nC        V P„                  e   \‹        V4      '       d   V! 4       MRV nD        R# R# u upi   \R         d    \J        PL                  p EL(i ; i  \R         d    \J        PL                  p EL>i ; i)z:Initialize loop state, device policy, and callback wiring.N©ÚloggerÚlayer_callbackFz,Disk subsystem write throughput detected at z.1fz1 MB/s; quantization may be severely slowed by IO.z( MB/s; quantization may be slowed by IO.z MB/s.r`   Úcompute_device_filterzUcompute_device_filter returned empty device list. Using all devices for quantization.rQ   z„ModuleLooper: METHOD.PARO forcing single-device quantization on `%s`; ignoring additional devices %s to avoid multi-GPU sync issues.Údense_vram_strategyÚsupported_dense_vram_strategieszZModuleLooper: Model %s does not support dense VRAM strategy %s; falling back to exclusive.Ú	__class__Úmoe_vram_strategyÚsupported_moe_vram_strategieszXModuleLooper: Model %s does not support MoE VRAM strategy %s; falling back to exclusive.Údense_vram_strategy_devicesÚmoe_vram_strategy_devices)rg   Úsubset_callbackÚdynamic_expert_indexÚget_input_embeddingsÚget_output_embeddingsÚget_input_embeddings_nameÚget_output_embeddings_name)Fr‚   r2   r   Ú
gptq_modelÚembed_quant_modeÚ
embed_onlyÚsupport_batch_quantizeÚ	threadingÚLockÚlockr6   ÚlogÚ_forward_executorr^   Ú_layer_callbackÚEventÚ_loop_stop_eventÚ_loop_stop_excÚ_loop_stop_waitedÚ_dangling_threadsÚ_dangling_threads_lockrz   ÚerrorÚwarnÚinfoÚquantize_configr   r!   r-   Ú_primary_quant_deviceÚlenrb   rA   Ú_quant_devicesÚ_quant_device_rrÚ_module_device_mapÚ_quant_device_lockr   Ú	EXCLUSIVEÚ
isinstanceÚlowerr\   ÚBALANCEDÚdebugrX   rF   Ú_dense_vram_strategyÚ_moe_vram_strategyrs   Ú_dense_quant_devicesÚ_moe_quant_devicesÚboolÚ_dense_vram_strategy_explicitÚ_moe_vram_strategy_explicitÚ_moe_subset_thresholdÚ_subset_callbackÚ_current_subsetÚget_num_expertsr�   ÚconfigÚmoe_routing_overrideÚmoe_routing_bypassÚ _emit_moe_parallel_quant_runtimeÚ_processor_mask_tlsr)   Úinput_embeddings_moduleÚoutput_embeddings_moduler}   r~   Úcallable)Úselfr�   r‚   rƒ   r   Úio_write_speedÚquant_device_hintÚnormalized_quant_devicerR   r‰   Úquant_devices_filteredÚrestricted_quant_devicesr`   rŠ   Úsupported_dense_strategiesr�   Úsupported_moe_strategiesÚdense_strategy_devicesÚmoe_strategy_devicesÚnum_expertsÚ	processorr“   r”   Úget_input_nameÚget_output_names   &&&&$                    rN   Ú__init__ÚModuleLooper.__init__¯   sF  € ð %ŒÜ(¨Ó4ˆŒØŒßGYÐ 2× CÒ CÐ_cˆÔß;MÐ,×7Ò7ÐSWˆŒà&+×&BÑ&BˆÔ#Ü—N’NÓ$ˆŒ	Ü!0°¼cÔ!BˆÔÜ& uÐ.>ÀÓEˆÔÜ )§¢Ó 1ˆÔØ7;ˆÔØ!&ˆÔØ9;ˆÔÜ&/§n¢nÓ&6ˆÔ#ä-Ó/ˆØÒ%Ø Ô#Ü—	‘	ØBØ% cÐ*Ð*[ð]õð   #Ô%Ü—‘ØBØ% cÐ*Ð*RðTõô
 —‘ØBØ% cÐ*¨&ð2ôô
 $ D§O¡O×$CÑ$CÀXÈtÓTÐÜ"7Ð8IÓ"JÐßKbÔ.Ð/FÔGÔilÐhmˆßÜ ˜EˆMØ%2°1Õ%5ˆÔ"ô !(¨¯©×(GÑ(GÐI`ÐbfÓ gÐØ Ò,Ù%:¸=Ó%IÐ"ÜÐ)Ó*¨aÔ/Ø 6‘ä—‘ð:ôô
 $FÜ�D—O‘O×3Ñ3°X¸tÓDØó$
Ð ð $Ô4Ü�H‰HðQà(¨Õ+Ù+8Ób©= ÐF^Ð_`ÕFaÑ<a””�V–©=Ñbô	ð 5ˆMà+ÔØ !ˆÔØ;=ˆÔÜ"+§.¢.Ó"2ˆÔô & d§o¡o×&EÑ&EÐG\Ô^j×^tÑ^tÓuÐÜÐ)¬3×/Ò/ð=Ü&2Ð3F×3LÑ3LÓ3NÓ&OÐ#ô &-Ø�O‰OØ-ä×&Ñ&Ü×%Ñ%ðó&
Ð"ô Ð0´,×?Ò?Ø*DÐ)EÐ&ØÔ@Ü�I‰IØlÜ˜Ÿ™¨´d¸4¿?¹?Ó6KÓL×UÑUØ#ôô
 #/×"8Ñ"8Ðä#Ø�O‰O×+Ñ+ØÜ×"Ñ"ó
Ðô
 Ð'¬×-Ò-ð;Ü$0Ð1B×1HÑ1HÓ1JÓ$KÐ!ô $+Ø�O‰OØ+ä×&Ñ&Ü×%Ñ%ðó$
Ð ô Ð.´×=Ò=Ø(@Ð'AÐ$ØÐ$<Ô<Ü�I‰IØjÜ˜Ÿ™¨´d¸4¿?¹?Ó6KÓL×UÑUØ!ôô
 !-× 6Ñ 6Ðà$7Ô!Ø"3ÔÜ!(¨¯©×)HÑ)HÐJgÐimÓ!nÐÜ& t§¡×'FÑ'FÐHcÐeiÓjÐÜ$AØ"ØØ/ô%
ˆÔ!ô
 #@Ø ØØ-ô#
ˆÔô .2Ð2HÓ-I÷  .Að  .AÈT×MfÑMfÔjv÷  kAñ  kAñ  NAˆÔ*Ü+/Ð0DÓ+E×+zÐ+zÈ×I`ÑI`Ôdp×dzÑdzÑIzˆÔ(à%'ˆÔ"Ü '¨¯©Ð9JÈDÓ QˆÔð :>ˆÔô �4—?‘?Ð$:¸D×AÒAØŸ/™/×9Ñ9¸$¿/¹/×:OÑ:O×:VÑ:VÓWˆKØ(,¯©×(GÑ(G×(\Ñ(\Ð]hÓ(iˆDÕ%à(,ˆDÔ%Ø"&§/¡/×"AÑ"A×"TÑ"TÓ"VˆÔØ×-Ñ-Ô/àŸœˆIØ×$Ñ$ YÖ/ñ )ð × Ñ Ò,Ü$9¸$¿/¹/×:OÑ:OÓ$PˆD�O‰OÔ!ð
 (,ˆÔ$Ø(,ˆÔ%Ø%)ˆÔ"Ø&*ˆÔ#Ø× Ñ Ò,Ü#*¨4¯?©?Ð<RÐTXÓ#YÐ Ü$+¨D¯O©OÐ=TÐVZÓ$[Ð!ä*2Ð3G×*HÒ*HÑ$Ô&Èdð Ô(ô ,4Ð4I×+JÒ+JÑ%Ô'ÐPTð Ô)ô % T§_¡_Ð6QÐSWÓXˆNÜ% d§o¡oÐ7SÐUYÓZˆOð ×/Ñ/Ò;ÄÈ×@XÒ@Xñ Ô àð Ô&ð ×0Ñ0Ò<ÄÈ/×AZÒAZñ  Ô!àð Ö'ñ! -ùò] cøô ô =Ü&2×&<Ñ&<Ó#ð=ûô8 ô ;Ü$0×$:Ñ$:Ó!ð;ús0   É `É4`Ì `# Ða à#aáaáa$á#a$c               ó   € V ^8„  d   QhRR/# ©rP   rT   ÚNonerE   )rU   s   "rN   rV   r„   {  s   € ÷ 3
ñ 3
°$ñ 3
rM   c                ó  € \        V P                  RR4      '       g   R# V P                   Uu. uF  p\        V4      NK  	  ppV P                   Uu. uF  p\        V4      NK  	  pp\
        P                  P                  R4      p\        4       p\        4       pV P                  e   RMV P                  '       d   RMRp\        T;'       d    \        V P                  4      ^ 8„  4      p\        P                  RVVVV P                  VVV4       V'       d'   V'       d   V'       g   \        P!                  RV4       \#        R	VVV P$                  V P&                  VV P                  V P                  VVVVR
7       R# u upi u upi )zLLog the runtime knobs that decide whether MoE quant can fan out efficiently.r’   NÚ
PYTHON_GILÚoverrideÚbypassÚnativez¤ModuleLooper: MoE quant runtime dense_pool=%s moe_pool=%s routing_mode=%s routing_override=%s PYTHON_GIL=%s gil_disabled=%s free_threaded_parallel_quant_eligible=%sz¬ModuleLooper: MoE quant is configured for device fan-out across %s but Python GIL is still enabled; rerun with PYTHON_GIL=0 to unlock the free-threaded parallel quant path.Úmoe_parallel_quant_runtime)Údense_devicesÚmoe_devicesÚdense_strategyÚmoe_strategyÚrouting_modeÚrouting_overrideÚrouting_bypassÚpython_gil_envÚpython_gil_controllableÚpython_gil_disabledÚ%free_threaded_parallel_quant_eligible)r^   r—   r¸   rA   r¹   ÚosÚenvironrk   r,   r+   rÂ   rÃ   rº   r¬   rž   r©   r¨   r   r¶   r·   )	rÉ   r`   râ   rã   Úgil_envÚgil_disabledÚgil_controllableræ   rì   s	   &        rN   rÄ   Ú-ModuleLooper._emit_moe_parallel_quant_runtime{  sn  € ô �t—‘Ð(>À×EÒEÙà37×3LÒ3LÓMÑ3L¨œ˜VžÑ3LˆÐMØ15×1HÒ1HÓIÑ1H v”s˜6–{Ñ1HˆÐIÜ—*‘*—.‘. Ó.ˆÜ'Ó)ˆÜ*Ó,Ðð ×(Ñ(Ò4ñ ð ×&×&Ð&ñ àð 	ô 15°\×5fÐ5fÄcÈ$×JaÑJaÓFbÐefÑFfÓ0gÐ-ä�‰ðUàØØØ×%Ñ%ØØØ1ô
	
÷ ×+·LÜ�H‰Hð[àôô 	Ø(Ø'Ø#Ø×4Ñ4Ø×0Ñ0Ø%Ø!×6Ñ6Ø×2Ñ2Ø"Ø$4Ø ,Ø2W÷	
ùòA NùÚIs   ®FÁFc                  ó2   € ] tR tRtRtR R ltR tR tRtR# )	Ú&ModuleLooper.MoERoutingOverrideContexti°  zÁ
Context manager that temporarily overrides MoE routing top-k.

On entry, applies the specified top-k override to all MoE routing modules.
On exit, restores the original routing configuration.
c               ó   € V ^8„  d   QhRR/# )rP   rÂ   ÚintrE   )rU   s   "rN   rV   Ú3ModuleLooper.MoERoutingOverrideContext.__annotate__¸  s   € ÷ 	4ñ 	4¸ñ 	4rM   c                ó,   € Wn         W n        RV n        R# )z8Capture the model and temporary top-k override to apply.N)r�   rÂ   Ú_state)rÉ   r�   rÂ   s   &&&rN   r×   Ú/ModuleLooper.MoERoutingOverrideContext.__init__¸  s   € ð ŒJà(<Ô%à/3ˆDŽKrM   c                ót   € V P                   '       d&   \        V P                  V P                   4      V n        V # )z=Apply the temporary routing override before the forward pass.)rÂ   r(   r�   rù   ©rÉ   s   &rN   Ú	__enter__Ú0ModuleLooper.MoERoutingOverrideContext.__enter__Â  s-   € ð ×(×(Ð(Ü*¨4¯:©:°t×7PÑ7PÓQ�”ØˆKrM   c                óT   € V P                   '       d   \        V P                  4       R# )z<Restore the original routing state when leaving the context.F)rÂ   r'   rù   )rÉ   Úexc_typeÚexcÚtbs   &&&&rN   Ú__exit__Ú/ModuleLooper.MoERoutingOverrideContext.__exit__Ê  s   € ð ×(×(Ð(Ü  §¡Ô-ÙrM   )rù   r�   rÂ   N©	rF   rG   rH   rI   rJ   r×   rý   r  rL   rE   rM   rN   ÚMoERoutingOverrideContextrô   °  s   † ñ	õ	4ò	ö	rM   r  c                  ó,   € ] tR tRtRtR tR tR tRtR# )Ú ModuleLooper.MoELifecycleContextiÒ  z4Context manager for MoE lifecycle hooks integration.c                ól   € Wn         W n        W0n        W@n        WPn        RV n        RV n        RV n        R# )z8Capture the replica state needed to patch the MoE block.FN)Úmodule_looperÚmodulerÔ   Úcurrent_subsetÚordered_module_namesÚmoe_hooks_activeÚ	moe_blockÚmoe_forward_original)rÉ   r
  r  rÔ   r  r  s   &&&&&&rN   r×   Ú)ModuleLooper.MoELifecycleContext.__init__Õ  s7   € ð "/ÔØ ŒKØ&ŒNØ"0ÔØ(<Ô%Ø$)ˆDÔ!Ø!ˆDŒNØ(,ˆDÖ%rM   c                óD  a aa€ S P                   P                  S P                  S P                  S P                  R7      '       dÛ   S P                   P
                  P                  oSP                  S P                  S P                   P
                  P                  S P                  R7      S n	        S P                  eb   S P                  P                  S n        SP                  S P                  S P                  4      oVVV 3R lpVS P                  n
        RS n        S # )z)Set up MoE lifecycle hooks if applicable.©r  c                ó  <€ SP                   ! RRSP                  RV RSP                  RSP                  RSP                  RSP
                  RSP                  P                  P                  RSP                  R	SR
SP                  /
VB # )z7Route the replica forward through the all-experts hook.r  Úhidden_statesrÔ   Úsubsetr  Úoriginal_forwardÚmodel_classr
  Úmoe_block_prefixÚreplica_modulerE   )
Úforward_to_all_expertsr  rÔ   r  r  r  r
  r—   rŒ   r  )r  ÚkwargsÚhooksr  rÉ   s   &,€€€rN   Úmoe_forward_wrapperÚGModuleLooper.MoELifecycleContext.__enter__.<locals>.moe_forward_wrapperö  s¸   ø€ ð  %×;Ò;ñ  Ø&*§n¡nð à*7ð ð '+§n¡nð ð $(×#6Ñ#6ð	 ð
 26×1JÑ1Jð ð .2×-FÑ-Fð ð )-×(:Ñ(:×(EÑ(E×(OÑ(Oð ð +/×*<Ñ*<ð ð .>ð ð ,0¯;©;ð ð %ñ ð rM   T)r
  Ú_should_use_moe_lifecycler  rÔ   r  r—   Úmoe_lifecycle_hooksÚget_moe_block_for_subsetrŒ   r  Úforwardr  Ú_extract_moe_block_prefixr  )rÉ   r  r  r  s   f @@rN   rý   Ú*ModuleLooper.MoELifecycleContext.__enter__á  sï   ú€ à×!Ñ!×;Ñ;Ø—‘Ø—‘Ø#×2Ñ2ð <÷ ò ð
 ×*Ñ*×5Ñ5×IÑI�Ø!&×!?Ñ!?Ø—K‘KØ×&Ñ&×1Ñ1×;Ñ;Ø#'×#6Ñ#6ð "@ó "�”ð —>‘>Ò-à04·±×0FÑ0F�DÔ-ð (-×'FÑ'FÀt×GZÑGZÐ\`×\jÑ\jÓ'kÐ$÷ð$ .A�D—N‘NÔ*Ø,0�DÔ)àˆKrM   c                ó˜   € V P                   '       d8   V P                  e*   V P                  e   V P                  V P                  n        R# )z6Restore original MoE forward method if it was patched.F)r  r  r  r#  )rÉ   r   Úexc_valÚexc_tbs   &&&&rN   r  Ú)ModuleLooper.MoELifecycleContext.__exit__  s:   € à×$×$Ð$¨×)BÑ)BÒ)NÐSW×SaÑSaÒSmØ)-×)BÑ)B�—‘Ô&ÙrM   )r  r  r
  r  r  r  r  rÔ   Nr  rE   rM   rN   ÚMoELifecycleContextr  Ò  s   † ÙBò
	-ò*	öX	rM   r*  c               ó   € V ^8„  d   QhRR/# rÚ   rE   )rU   s   "rN   rV   r„     s   € ÷ (ñ (°4ñ (rM   c                ó   € Wn         R# )z7Register or replace the layer-complete callback target.N)r    ©rÉ   Úcallbacks   &&rN   Úregister_layer_callbackÚ$ModuleLooper.register_layer_callback  s   € à'ÖrM   c               ó   € V ^8„  d   QhRR/# rÚ   rE   )rU   s   "rN   rV   r„     s   € ÷ )ñ )°Dñ )rM   c                ó   € Wn         R# )z5Register or replace the subset event callback target.N)r¾   r-  s   &&rN   Úregister_subset_callbackÚ%ModuleLooper.register_subset_callback  s   € à (ÖrM   c               ó    € V ^8„  d   QhRRRR/# )rP   Úwatcherzthreading.ThreadrT   rÛ   rE   )rU   s   "rN   rV   r„     s   € ÷ 3ñ 3Ð0@ð 3ÀTñ 3rM   c                ó<  € V P                   ;_uu_ 4        V P                  '       d9   V P                   Uu. uF  q"P                  4       '       g   K  VNK  	  upV n        V P                  P                  V4       RRR4       R# u upi   + '       g   i     R# ; i)z9Track a watcher thread that should be joined before exit.N)r¦   r¥   Úis_aliverm   )rÉ   r6  Úthreads   && rN   Úregister_dangling_threadÚ%ModuleLooper.register_dangling_thread  sw   € ð ×(×(Ó(Ø×%×%Ð%à)-×)?Ò)?ó*Ù)?˜vÇ?Á?×CT—F�FÑ)?ñ*�Ô&ð ×"Ñ"×)Ñ)¨'Ô2÷ )Ñ(ùò*÷ )×(Ð(ús"   •!B
¶BÁBÁ#B
ÂB
Â
B	c               ó   € V ^8„  d   QhRR/# rÚ   rE   )rU   s   "rN   rV   r„   %  s   € ÷ ñ  tñ rM   c                óP  € V P                   ;_uu_ 4        \        V P                  4      pV P                  P                  4        RRR4       X Uu. uF  q"P	                  4       '       g   K  VNK  	  ppV F  pVP                  4        K  	  R#   + '       g   i     LS; iu upi )z>Join any still-running watcher threads and clear the registry.N)r¦   rj   r¥   Úclearr8  Újoin)rÉ   Úthreadsr9  Úalive_threadss   &   rN   Úwait_dangling_threadsÚ"ModuleLooper.wait_dangling_threads%  sw   € ð ×(×(Ó(Ü˜4×1Ñ1Ó2ˆGØ×"Ñ"×(Ñ(Ô*÷ )ñ /6ÓK©g F¿¹×9JŸ˜©gˆÐKÛ#ˆFØ�K‰KŽMó $÷	 )×(üò Ls   •0BÁB#Á-B#ÂB 	c           	     óÜ   € \        V RR4      \        V RR4      \        V P                  RR4      \        V P                  RR4      \        V P                  RR4      3 F  pVf   K	  Vu # 	  R# )zBResolve the active layer-complete callback using legacy fallbacks.r    Nrˆ   Ú
callbackupr.  ©r^   r—   ©rÉ   Ú	candidates   & rN   Ú_resolve_layer_callbackÚ$ModuleLooper._resolve_layer_callback/  sn   € ô �DÐ+¨TÓ2Ü�DÐ*¨DÓ1Ü�D—O‘OÐ%5°tÓ<Ü�D—O‘O \°4Ó8Ü�D—O‘O Z°Ó6ó
ˆIð Ô$Ø Ò ñ
ñ rM   c                ó„   € \        V RR4      \        V RR4      \        V P                  RR4      3 F  pVf   K	  Vu # 	  R# )z>Resolve the active subset callback from looper or model state.r¾   Nr‘   rF  rG  s   & rN   Ú_resolve_subset_callbackÚ%ModuleLooper._resolve_subset_callback=  sN   € ô �DÐ,¨dÓ3Ü�DÐ+¨TÓ2Ü�D—O‘OÐ%6¸Ó=ó
ˆIð
 Ô$Ø Ò ñ
ñ rM   c               ó    € V ^8„  d   QhRRRR/# )rP   rD   rö   Úsubmodule_finalizedrº   rE   )rU   s   "rN   rV   r„   I  s   € ÷ ñ  Cð ¸dñ rM   c                ó  € V P                  4       pVf   R# \        VRR4      pVf   \        V4      '       d   TpVf   R#  V! WR7      pT\        J d   \        RT 24      h\	        T\        4      '       d   ThT#   \         d    h i ; i)z<Invoke the layer callback and normalize stop-loop responses.NÚlayer_complete©rD   rO  z'Layer callback requested stop at layer )rI  r^   rÈ   ru   r²   )rÉ   rD   rO  r.  ÚhandlerÚresults   &&&   rN   rE  ÚModuleLooper.callbackupI  s�   € ð ×/Ñ/Ó1ˆØÒÙä˜(Ð$4°dÓ;ˆØŠ?œx¨×1Ò1ØˆGØŠ?Ùð	Ù yÔZˆFð ”\Ó!ÜÐ!HÈÈÐTÓUÐUÜ�fœl×+Ò+ØˆLØˆøô ô 	Øð	ús   Á	A= Á=B
c               ó4   € V ^8„  d   QhRRRRRRRRRRR	RR
R/# )rP   ÚstagerA   rD   rö   Úsubset_indexÚsubset_totalÚmodule_namesú	List[str]rÔ   rT   rÛ   rE   )rU   s   "rN   rV   r„   `  sR   € ÷ 
ñ 
ð ð
ð ð	
ð
 ð
ð ð
ð  ð
ð ð
ð 
ñ
rM   c          
     ó6   € V P                  VVVVVVRR7       R# )z>Emit a subset event immediately and surface callback failures.T)rW  rD   rX  rY  rZ  rÔ   Úraise_in_placeN)Ú_emit_subset_event)rÉ   rW  rD   rX  rY  rZ  rÔ   s   &$$$$$$rN   Ú_subset_event_dispatchÚ#ModuleLooper._subset_event_dispatch`  s,   € ð 	×ÑØØØ%Ø%Ø%ØØð 	 ö 	
rM   c               ó    € V ^8„  d   QhRRRR/# )rP   r  zOptional[BaseException]rT   rÛ   rE   )rU   s   "rN   rV   r„   v  s   € ÷ $ñ $Ð&=ð $À$ñ $rM   c                óÊ   € V P                   ;_uu_ 4        V P                  f   Ve   Wn        RRR4       V P                  P                  4        R#   + '       g   i     L,; i)z6Record the first stop reason and signal loop shutdown.N)r�   r£   r¢   Úset)rÉ   r  s   &&rN   Ú_request_loop_stopÚModuleLooper._request_loop_stopv  sD   € ð �Y�Y‹YØ×"Ñ"Ò*¨sªØ&)Ô#÷ ð 	×Ñ×!Ñ!Ö#÷ �Yús   •AÁA"	c               ó   € V ^8„  d   QhRR/# )rP   rT   rº   rE   )rU   s   "rN   rV   r„   ~  s   € ÷ 
ñ 
 $ñ 
rM   c                óÚ   € V P                   P                  4       '       g   R# V P                  '       g   \        P                  ! 4        RV n        V P
                  e   V P
                  hR# )z=Drain outstanding work and re-raise any recorded stop signal.FT)r¢   Úis_setr¤   r	   Úwaitr£   rü   s   &rN   Ú_check_loop_stopÚModuleLooper._check_loop_stop~  sW   € ð ×$Ñ$×+Ñ+×-Ò-ÙØ×%×%Ð%Ü×#Ò#Ô%Ø%)ˆDÔ"Ø×ÑÒ*Ø×%Ñ%Ð%ÙrM   c               ó8   € V ^8„  d   QhRRRRRRRRRRR	RR
RRR/# )rP   rW  rA   rD   rö   rX  rY  rZ  r[  rÔ   r]  rº   rT   rÛ   rE   )rU   s   "rN   rV   r„   Š  s\   € ÷ 6ñ 6ð ð6ð ð	6ð
 ð6ð ð6ð  ð6ð ð6ð ð6ð 
ñ6rM   c          	     ó<  € V P                  4       pVf   R# \        VRR4      p	V	f   \        V4      '       d   Tp	V	f   R#  V	! VVVVVVR7      p
T
\        J d/   \        RT RT 24      pT P	                  T4       T'       d   ThR# \        T
\        4      '       d   T P	                  T
4       T'       d   T
hR# R#   \         d&   pT P	                  T4       T'       d   h  Rp?R# Rp?i\
         d&   pT P	                  T4       T'       d   h  Rp?R# Rp?ii ; i)z<Forward a subset lifecycle event to the configured callback.NÚsubset_event)rW  rD   rX  rY  rZ  rÔ   z(Subset callback requested stop at layer z subset )rL  r^   rÈ   ru   rd  ÚBaseExceptionr²   )rÉ   rW  rD   rX  rY  rZ  rÔ   r]  r.  rS  rT  r  s   &$$$$$$$    rN   r^  ÚModuleLooper._emit_subset_eventŠ  s  € ð ×0Ñ0Ó2ˆØÒÙä˜( N°DÓ9ˆØŠ?œx¨×1Ò1ØˆGØŠ?Ùð	ÙØØ#Ø)Ø)Ø)Ø#ôˆFð& ”\Ó!ÜÐ!IÈ)ÈÐT\Ð]iÐ\jÐkÓlˆCØ×#Ñ# CÔ(ßØ�	Ùä�fœl×+Ò+Ø×#Ñ# FÔ+ßØ�Ùñ	 ,øô% ô 	Ø×#Ñ# CÔ(ßØÝûÜô 	Ø×#Ñ# CÔ(ßØÝûð		ús*   ÁB= Â=DÃC(Ã(DÃ5DÃ6DÄDc               ó(   € V ^8„  d   QhRRRRRRRR/# )rP   rD   rö   rO  rº   r]  rT   rÛ   rE   )rU   s   "rN   rV   r„   Â  s2   € ÷ )ñ )àð)ð "ð)ð
 ð)ð 
ñ)rM   c               óü   €  V P                  WR7       R#   \         d    T P                  R4        R# \         d=   pT'       d   h \        P                  RTT4       T P                  T4        Rp?R# Rp?ii ; i)z@Notify listeners that a layer finished and handle stop requests.rR  NzPLayer completion callback raised an exception (layer=%s, submodule_finalized=%s))rE  ru   rd  ro  rž   Ú	exception)rÉ   rD   rO  r]  r  s   &&&$ rN   Ú_emit_layer_completeÚ!ModuleLooper._emit_layer_completeÂ  sl   € ð	)Ø�O‰O iˆOÖYøÜô 	Ø×#Ñ# DÔ)ÚÜô 	)ßØÜ�M‰MØbØØ#ôð
 ×#Ñ# C×(Ò(ûð	)ús   ‚ –A;µA;¾A;¿1A6Á6A;c               ó    € V ^8„  d   QhRRRR/# )rP   rÔ   r   rT   zthreading.localrE   )rU   s   "rN   rV   r„   Ü  s   € ÷ ñ ¨]ð ¸ñ rM   c                ól   € \        VRR4      pVf#   \        P                  ! 4       p\        VRV4       V# )z<Get or create thread-local storage for the active keep mask.Ú	_mask_tlsN)r^   r›   ÚlocalÚsetattr©rÉ   rÔ   Útlss   && rN   rÅ   Ú ModuleLooper._processor_mask_tlsÜ  s4   € ô �i ¨dÓ3ˆØŠ;Ü—/’/Ó#ˆCÜ�I˜{¨CÔ0Øˆ
rM   c                ór   € \        VR4      '       g   \        P                  ! 4       Vn        VP                  # )z9Get or create thread-local storage for hooks_paused flag.Ú_hooks_paused_tls)Úhasattrr›   ry  r  )rÉ   rÔ   s   &&rN   Ú_processor_hooks_paused_tlsÚ(ModuleLooper._processor_hooks_paused_tlså  s+   € ä�yÐ"5×6Ò6Ü*3¯/ª/Ó*;ˆIÔ'Ø×*Ñ*Ð*rM   c               ó    € V ^8„  d   QhRRRR/# )rP   rÔ   r   Úpausedrº   rE   )rU   s   "rN   rV   r„   ë  s   € ÷ ñ °]ð ÈDñ rM   c                ó4   € V P                  V4      pW#n        R# )z*Set hooks paused state for current thread.N)r�  Úvalue)rÉ   rÔ   r„  r|  s   &&& rN   Ú_set_processor_hooks_pausedÚ(ModuleLooper._set_processor_hooks_pausedë  s   € à×.Ñ.¨yÓ9ˆØŽ	rM   c               ó    € V ^8„  d   QhRRRR/# )rP   rÔ   r   rT   rº   rE   )rU   s   "rN   rV   r„   ð  s   € ÷ >ñ >°]ð >Àtñ >rM   c                óJ   € \        VRR4      pV'       d   \        VRR4      # R# )z8Get hooks paused state for current thread (thread-safe).r  Nr†  F©r^   r{  s   && rN   Ú_get_processor_hooks_pausedÚ(ModuleLooper._get_processor_hooks_pausedð  s'   € ä�iÐ!4°dÓ;ˆß/2Œw�s˜G UÓ+Ð=¸Ð=rM   c               ó   € V ^8„  d   QhRR/# ©rP   rÔ   r   rE   )rU   s   "rN   rV   r„   õ  s   € ÷ ñ ¨]ñ rM   c                ó4   € V P                  V4      pW#n        R# )z=Store the active sequence mask for the current worker thread.N)rÅ   r†  )rÉ   rÔ   Úmaskr|  s   &&& rN   Ú_set_processor_maskÚ ModuleLooper._set_processor_maskõ  s   € ð ×&Ñ& yÓ1ˆØŽ	rM   c               ó   € V ^8„  d   QhRR/# r�  rE   )rU   s   "rN   rV   r„   û  s   € ÷ <ñ <¨]ñ <rM   c                óJ   € \        VRR4      pV'       d   \        VRR4      # R# )z<Return the sequence mask bound to the current worker thread.rx  Nr†  r‹  r{  s   && rN   Ú_get_processor_maskÚ ModuleLooper._get_processor_maskû  s(   € ô �i ¨dÓ3ˆß.1Œw�s˜G TÓ*Ð;°tÐ;rM   c               ó   € V ^8„  d   QhRR/# ©rP   rT   rC   rE   )rU   s   "rN   rV   r„     s   € ÷ ñ  ]ñ rM   c                óV   € Vf   R#  \        V4      #   \        \        3 d     R# i ; i)z?Return ``len(sequence)`` when the object exposes a safe length.N)r¬   r[   ÚAttributeError)rÉ   Úsequences   &&rN   Ú	_safe_lenÚModuleLooper._safe_len  s3   € ð ÒÙð	Ü�x“=Ð øÜœ>Ð*ô 	Úð	ús   ˆ
 “(§(c               ó   € V ^8„  d   QhRR/# r™  rE   )rU   s   "rN   rV   r„     s   € ÷ ñ  }ñ rM   c                óô  € Vf   R# \        V\        4      '       d   \        V4      # \        V\        4      '       d   V# \        VRR4      p\	        V4      '       d	    V! 4       # \        P                  ! V4      '       d2   VP                  4       ^8X  d    \        VP                  4       4      # R#  \        V4      #   \
         d     Lfi ; i  \
         d     R# i ; i  \        \        3 d     R# i ; i)z?Best-effort conversion for scalar-like values used in counters.NÚ	__index__)r²   rº   rö   r^   rÈ   Ú	ExceptionÚtorchÚ	is_tensorÚnumelÚitemr[   r\   )rÉ   r†  Úindexers   && rN   Ú_coerce_to_intÚModuleLooper._coerce_to_int  sã   € ð Š=ÙÜ�eœT×"Ò"Ü�u“:ÐÜ�eœS×!Ò!ØˆLä˜% ¨dÓ3ˆÜ�G×ÒðÙ“yÐ ô �?Š?˜5×!Ò!Ø�{‰{‹} Ô!ð Ü˜uŸz™z›|Ó,Ð,ñ ð	Ü�u“:Ðøô ô Ùðûô !ô  Úð ûô œ:Ð&ô 	Úð	ús6   ÁB? ÂC Â4
C" Â?CÃCÃCÃCÃ"C7Ã6C7c               ó   € V ^8„  d   QhRR/# )rP   rT   rö   rE   )rU   s   "rN   rV   r„   )  s   € ÷ ñ ÀCñ rM   c                óò   € V P                  V4      pV P                  V4      pV P                  V4      pVe    V^ 8”  d   Ve   V^ 8¼  d   \        W54      # V# Ve   \        V^ 4      # Ve   \        V^ 4      # ^ # )zCResolve a non-negative batch count from explicit or inferred input.)r¨  r�  ÚminÚmax)rÉ   Ú	raw_countÚfallback_sequenceÚcountÚfallback_lenÚfallbacks   &&&   rN   Ú_resolve_batch_totalÚ!ModuleLooper._resolve_batch_total)  sƒ   € ð ×#Ñ# IÓ.ˆØ—~‘~Ð&7Ó8ˆØ×&Ñ& |Ó4ˆàÒ ¨¤ØÒ#¨°A¬Ü˜5Ó+Ð+ØˆLàÒÜ�x Ó#Ð#àÒÜ�u˜a“=Ð árM   c               ó    € V ^8„  d   QhRRRR/# )rP   Úbatch_inputszOptional[List[torch.Tensor]]rT   rö   rE   )rU   s   "rN   rV   r„   =  s   € ÷ ñ Ð-Ið Ècñ rM   c                ó(  € V'       g   ^ # V^ ,          p\        V\        P                  4      '       d[   VP                  ^ 8”  d'   \	        \        VP                  ^ ,          4      ^ 4      # \	        \        VP                  4       4      ^ 4      # \        V\        \        34      '       di   V'       da   V^ ,          p\        V\        P                  4      '       d8   VP                  ^ 8”  d'   \	        \        VP                  ^ ,          4      ^ 4      # ^ # )z;Infer how many rows a cached batch contributes to progress.)
r²   r£  ÚTensorÚndimr­  rö   Úshaper¥  rj   Útuple)rÉ   r¶  ÚprimaryÚnested_firsts   &&  rN   Ú_batch_row_countÚModuleLooper._batch_row_count=  s½   € ÷ Ùà˜q•/ˆä�gœuŸ|™|×,Ò,Ø�|‰|˜aÔÜœ3˜wŸ}™}¨QÕ/Ó0°!Ó4Ð4Ü”s˜7Ÿ=™=›?Ó+¨QÓ/Ð/ä�g¤¤e˜}×-Ò-·'Ø" 1�:ˆLÜ˜,¬¯©×5Ò5¸,×:KÑ:KÈaÔ:OÜœ3˜|×1Ñ1°!Õ4Ó5°qÓ9Ð9árM   c               ó    € V ^8„  d   QhRRRR/# )rP   Úlayer_inputsz"Optional[List[List[torch.Tensor]]]rT   z	List[int]rE   )rU   s   "rN   rV   r„   Q  s   € ÷ 
ñ 
Ð0Rð 
ÐW`ñ 
rM   c                ó†   € V'       g   . # . pV F.  pV P                  V4      pTP                  V^ 8”  d   TM^ 4       K0  	  V# )z3Collect per-batch row counts for progress tracking.)r¾  rm   )rÉ   rÁ  Úcountsr¶  r°  s   &&   rN   Ú_collect_row_countsÚ ModuleLooper._collect_row_countsQ  sF   € ÷ ØˆIàˆÛ(ˆLØ×)Ñ)¨,Ó7ˆEØ�M‰M 5¨1¤9™%°!Ö4ñ )ð ˆrM   c               ó    € V ^8„  d   QhRRRR/# )rP   Úmodule_namer?   rT   rE   )rU   s   "rN   rV   r„   ]  s   € ÷ ñ °-ð ÀMñ rM   c                ó$  € V'       g   R# RV9   d?   VP                  R^4      w  r#VP                  R^4      ^ ,          pV'       d   V RV 2# R# RV9   d   VP                  R^4      w  r%V R2# RV9   d   VP                  R^4      w  r%V R2# R# )zACollapse expert module names into a stable MoE routing group key.Nz	.experts.ri   z.shared_experts.z.shared_expertsz.shared_expert.z.shared_expert)Úsplit)rÉ   rÇ  ÚprefixÚ	remainderÚ	expert_idÚ_s   &&    rN   Ú_extract_moe_group_keyÚ#ModuleLooper._extract_moe_group_key]  sª   € ÷ Ùà˜+Ô%Ø +× 1Ñ 1°+¸qÓ AÑˆFØ!Ÿ™¨¨QÓ/°Õ2ˆIßØ ˜ ¨9¨+Ð6Ð6Ùà Ô,Ø#×)Ñ)Ð*<¸aÓ@‰IˆFØ�X˜_Ð-Ð-à Ô+Ø#×)Ñ)Ð*;¸QÓ?‰IˆFØ�X˜^Ð,Ð,árM   c               ó    € V ^8„  d   QhRRRR/# )rP   rÇ  rA   rT   rº   rE   )rU   s   "rN   rV   r„   t  s   € ÷ ñ °Sð ¸Tñ rM   c                óÚ   € V'       g   R# VP                  4       pRV9   g   RV9   d   R# RV9   g   RV9   d   R# VP                  R4      '       g   VP                  R4      '       d   R# R# )	zAHeuristically detect attention modules from their qualified name.Fz.attnzattn.Tz
.attentionz
attention.ÚattnÚ	attention)r³   Úendswith)rÉ   rÇ  Úlowereds   && rN   Ú_is_attention_module_nameÚ&ModuleLooper._is_attention_module_namet  sc   € ÷ Ùà×#Ñ#Ó%ˆØ�gÔ ¨GÔ!3ÙØ˜7Ô" l°gÔ&=ÙØ×Ñ˜F×#Ò# w×'7Ñ'7¸×'DÒ'DÙÙrM   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rP   r  z	nn.ModulerÔ   r   r  úOptional[Dict[str, Any]]rT   rº   rE   )rU   s   "rN   rV   r„   ƒ  s2   € ÷ +ñ +àð+ð !ð+ð 1ð	+ð
 
ñ+rM   c                óÀ  € V P                   P                  P                  4       pV'       g   R# \        V P                   RR4      pVf:   \        P                  RV P                   P                  P                   R24       R# VP                  VV P                   P                  VR7      pVf0   \        P                  RVP                  P                   R24       R# R	# )
zÊ
Check if MoE lifecycle hooks should be used for this module.

Returns True if:
- pass_whole_dataset_to_each_expert flag is enabled
- Model has lifecycle hooks configured
- Module contains an MoE block
Fr!  Nz1pass_whole_dataset_to_each_expert is enabled but z¨ model does not have 'moe_lifecycle_hooks' configured. MoE optimization will be disabled. Please ensure your model definition has proper MoE lifecycle hooks configured.r  zNpass_whole_dataset_to_each_expert is enabled but no MoE block found in module z‡. MoE optimization will be disabled for this module. This may indicate an issue with the model's MoE configuration or module structure.T)	r—   rª   rÃ   r^   rž   r¨   rŒ   rF   r"  )rÉ   r  rÔ   r  Úflag_enabledr  r  s   &&&&   rN   r   Ú&ModuleLooper._should_use_moe_lifecycleƒ  sÙ   € ð —‘×6Ñ6×IÑIÓKˆßÙô ˜Ÿ™Ð)>ÀÓEˆØŠ=Ü�H‰HØCÀDÇOÁO×D]ÑD]×DfÑDfÐCgð hað bôñ
 ð ×2Ñ2ØØ�O‰O×%Ñ%Ø)ð 3ó 
ˆ	ð
 ÒÜ�H‰HØ`Ø×#Ñ#×,Ñ,Ð-ð .eðfôñ
 árM   c               ó   € V ^8„  d   QhRR/# )rP   rT   zDict[str, List[str]]rE   )rU   s   "rN   rV   r„   °  s   € ÷ 
ñ 
Ð(<ñ 
rM   c                ó   € \        V P                  P                  RR4      '       d   \        V P                  4      MV P                  .pR\        V P                  4      .RV P                   Uu. uF  p\        V4      NK  	  upRV P                   Uu. uF  p\        V4      NK  	  upRV P                   Uu. uF  p\        V4      NK  	  upRV Uu. uF  p\        V4      NK  	  up/# u upi u upi u upi u upi )z@Ordered placement pools, including forward-only visible devices.Úauto_forward_data_parallelTr¼  ÚquantizationÚdenseÚmoer#  )	r^   r—   rª   r!   r«   rA   r­   r¸   r¹   )rÉ   Úforward_devicesr`   s   &  rN   Úexecution_device_poolsÚ#ModuleLooper.execution_device_pools°  sã   € ô �t—‘×6Ñ6Ð8TÐVZ×[Ò[ô # 4×#=Ñ#=Ô>à×,Ñ,Ð-ð 	ð œ˜D×6Ñ6Ó7Ð8Ø°t×7JÒ7JÓKÑ7J¨VœS ž[Ñ7JÑKØ°×0IÒ0IÓJÑ0I f”c˜&–kÑ0IÑJØ¨d×.EÒ.EÓFÑ.E F”C˜–KÑ.EÑFØ±/ÓB±/¨œ˜Fž±/ÑBð
ð 	
ùâKùÚJùÚFùÚBs   Á/C,
ÂC1Â9C6ÃC;c               ó$   € V ^8„  d   QhRRRRRR/# )rP   Únamed_moduler   Úfallback_deviceútorch.devicerT   rE   )rU   s   "rN   rV   r„   ¿  s(   € ÷ *ñ *à!ð*ð &ð*ð 
ñ	*rM   c           	     ó~  a€ \        VRR4      ;'       g    VP                  pV P                  ;_uu_ 4        V P                  P	                  V4      pVe   \        RVVR7       VuuRRR4       # \        VP                  P	                  R4      4      oSef   \        ;QJ d)    V3R lV P                   4       F  '       g   K   RM	  RM! V3R lV P                   4       4      '       d   Sp\        R	VVR7       M~\        V P                  4      ^8:  d   V P                  ^ ,          pMPV P                  V P                  \        V P                  4      ,          ,          pV ;P                  ^,          un        Vf   TpWPP                  V&   \        R
VVVR7       VuuRRR4       #   + '       g   i     R# ; i)z>Pick and memoize the quantization device for one named module.Ú	full_nameNÚquant_device_cache_hit©r  Útarget_deviceÚpreferred_quant_devicec              3  ó8   <"  € T F  qf   K  VS8H  x € K  	  R # 5i©NrE   )Ú.0ÚdevÚpreferred_devices   & €rN   Ú	<genexpr>Ú?ModuleLooper._assign_quant_device_for_module.<locals>.<genexpr>Ò  s   øé € Ð3~ÑWjÐPSÔ4K°CÐ;KÖ4KÓWjùs   ƒ�TFÚquant_device_preferred_hintÚquant_device_assign)r  rî  rè  )r^   Únamer°   r¯   rk   r   r   ÚstateÚanyr­   r¬   r®   )rÉ   rç  rè  ÚkeyÚcachedr`   rô  s   &&&   @rN   Ú_assign_quant_device_for_moduleÚ,ModuleLooper._assign_quant_device_for_module¿  sm  ø€ ô �l K°Ó6×KÐK¸,×:KÑ:KˆØ×$×$Ó$Ø×,Ñ,×0Ñ0°Ó5ˆFØÒ!Ü%Ø,ØØ"(õð
 ÷ %Ò$ô  5°\×5GÑ5G×5KÑ5KÐLdÓ5eÓfÐØÒ+·³Ô3~ÐW[×WjÒWjÓ3~··²Ô3~ÐW[×WjÒWjÓ3~×0~Ò0~Ø)�Ü%Ø1ØØ"(öô
 �T×(Ñ(Ó)¨QÔ.Ø×,Ñ,¨QÕ/‘à×,Ñ,¨T×-BÑ-BÄSÈ×I\ÑI\ÓE]Õ-]Õ^�Ø×%Ò%¨Õ*Õ%àŠ~Ø(�à+1×#Ñ# CÑ(Ü!Ø%ØØ$Ø /õ	ð ÷E %×$×$Ó$ús$   ·/F+Á12F+Â$F+Ã'F+Ã-B3F+Æ+F<	Úfallback_modulesc               ó(   € V ^8„  d   QhRRRRRRRR/# )rP   r  úDict[str, NamedModule]Ú
device_mapúDict[str, torch.device]r   ú$Optional[Dict[str, torch.nn.Module]]rT   rE   )rU   s   "rN   rV   r„   ë  s2   € ÷ .ñ .à&ð.ð ,ð.ð
 ?ð.ð 
!ñ.rM   c          	     óL  € / pV'       g   V# VP                  4        Fð  w  rVVP                  V4      pRpVf   Ve   VP                  V4      pTpM(Ve%   \        V\        4      '       d   VP                  MTpVf   Kb   \        V4      p	Ve   V	e	   W–8X  d   K~  V	e   W”V&   \        RVe   \        VRV4      MTV	VR7       \        W†R7       \        W†RRR7       \        V\        4      '       d   \        VRV4       \        VRV4       Kò  	  V#   \         d    Rp	 L—i ; i)	zNMove selected modules to temporary forward devices and record prior placement.NÚforward_override_applyrë  )r  Úcurrent_devicerî  ©r`   T©Úmove_parametersÚmove_buffersrî  )Úitemsrk   r²   r   r  r   r¢  r   r^   r&   r    rz  )
rÉ   r  r  r   Úpreviousrù  Útargetrç  Ú
module_refÚcurrents
   &&&$      rN   Ú_apply_forward_device_overridesÚ,ModuleLooper._apply_forward_device_overridesë  s2  € ð -/ˆßØˆOà&×,Ñ,Ö.‰LˆDØ!Ÿ:™: dÓ+ˆLØˆJØÒ#Ð(8Ò(DØ-×1Ñ1°$Ó7�
Ø)‘ØÒ)Ü4>¸|Ì[×4YÒ4Y˜\×0Ò0Ð_k�
ØÒ!ÙðÜ$ ZÓ0�ð Š~ 'Ò"5¸'Ô:KÙàÒ"Ø!(˜‘ä!Ø(ØCOÒC[”w˜|¨[¸$Ô?ÐaeØ&Ø$õ	ô �JÕ.Ü# JÈÐ[_Õ`Ü˜,¬×4Ò4Ü˜ o°vÔ>Ü�J °Ö8ñ? /ðB ˆøô+ ô Ø’ðús   Â DÄD#Ä"D#c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rP   r  r  Úprevious_devicesr  r   r  rT   rÛ   rE   )rU   s   "rN   rV   r„     s8   € ÷ @ñ @à&ð@ð 2ð@ð
 ?ð@ð 
ñ@rM   c          	     óØ  € V'       g   R# VP                  4        FË  w  rEVP                  V4      pRpVf   Ve   VP                  V4      pTpM(Ve%   \        V\        4      '       d   VP                  MTpVf   Kb  \        RVe   \        VRV4      MTVR7       \        WuR7       \        WuRRR7       \        V\        4      '       d   \        VRV4       \        VRV4       KÍ  	  R# )	zGRestore module placements saved by ``_apply_forward_device_overrides``.NÚforward_override_restorerë  rí  r	  Tr
  rî  )
r  rk   r²   r   r  r   r^   r&   r    rz  )rÉ   r  r  r   rù  Úrevert_devicerç  r  s   &&&$    rN   Ú!_restore_forward_device_overridesÚ.ModuleLooper._restore_forward_device_overrides  sÛ   € ÷  Ùà#3×#9Ñ#9Ö#;ÑˆDØ!Ÿ:™: dÓ+ˆLØˆJØÒ#Ð(8Ò(DØ-×1Ñ1°$Ó7�
Ø)‘ØÒ)Ü4>¸|Ì[×4YÒ4Y˜\×0Ò0Ð_k�
ØÒ!ÙÜ!Ø*ØCOÒC[”w˜|¨[¸$Ô?ÐaeØ+õô
 �JÕ5Ü# JÈtÐbfÕgÜ˜,¬×4Ò4Ü˜ o°}ÔEÜ�J °Ö?ó' $<rM   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	rP   rÔ   r   rç  r   rî  ré  rT   rÛ   rE   )rU   s   "rN   rV   r„   <  s2   € ÷ 3%ñ 3%à ð3%ð "ð3%ð $ð	3%ð
 
ñ3%rM   c           
     ód  € \        VRR4      pV'       g   R# VP                  VP                  4      pVf   R# VP                  P                  R4      p\	        V\
        P                  P                  4      '       d   \        VR4      '       d   Wen	        \        VRR4      p\        V4      '       d   V! V4       R# \        VRR4      p\	        V\
        P                  P                  4      '       d(   \        WƒR7       \        WƒRRR7       \        VR	V4       \        VR
R4      p	\	        V	\
        P                  P                  4      '       d(   \        W“R7       \        W“RRR7       \        V	R	V4       \        VRR4      p
V
e%   \        V
R4      '       d    V
P                  V4       RpV FN  p\        W\R4      p\	        V\
        P                   4      '       g   K1  \        W\VP                  VRR7      4       KP  	  \        VR4      '       d	   W5n        R# R#   \         d     Li ; i)zBMove processor-owned task state alongside the module it quantizes.ÚtasksNÚquant_source_moduler  Ú	to_devicer	  Tr
  rî  ÚlayerÚ	quantizerÚto)r`   Únon_blockingró  )ÚHÚmodule_copy)r^   rk   rù  rú  r²   r£  ÚnnÚModuler€  r  rÈ   r&   r    rz  r"  r¢  r¸  ró  )rÉ   rÔ   rç  rî  Útask_mapÚtaskÚquant_sourceÚto_device_fnÚmodule_attrÚ
layer_attrr!  Útensor_attrsÚ	attr_nameÚtensor_values   &&&&          rN   Ú_rehome_processor_taskÚ#ModuleLooper._rehome_processor_task<  sÂ  € ô ˜9 g¨tÓ4ˆßÙà�|‰|˜L×-Ñ-Ó.ˆØŠ<Ùà#×)Ñ)×-Ñ-Ð.CÓDˆÜ�l¤E§H¡H§O¡O×4Ò4¼ÀÀx×9PÒ9PØ&ŒKä˜t [°$Ó7ˆÜ�L×!Ò!Ù˜Ô'Ùä˜d H¨dÓ3ˆÜ�k¤5§8¡8§?¡?×3Ò3Ü�KÕ6Ü# KÐPTÐcgÕhÜ�K °-Ô@ä˜T 7¨DÓ1ˆ
Ü�j¤%§(¡(§/¡/×2Ò2Ü�JÕ5Ü# JÈtÐbfÕgÜ�J °Ô?ä˜D +¨tÓ4ˆ	ØÒ ¤W¨Y¸×%=Ò%=ðØ—‘˜]Ô+ð ,ˆÛ%ˆIÜ" 4°DÓ9ˆLÜ˜,¬¯©×5Ô5Ü˜¨¯©ÀÐ\`¨Ó)aÖbñ &ô
 �4˜×ÒØ$ŽHñ  øô ô Ùðús   ÆH! È!H/È.H/c               ó$   € V ^8„  d   QhRRRRRR/# )rP   rç  r   rè  ré  rT   útorch.nn.ModulerE   )rU   s   "rN   rV   r„   q  s(   € ÷ ñ à!ðð &ðð 
ñ	rM   c                ó  € \        VP                  4      pV\        8X  d   TpV P                  P	                  VP                  VRVR7      pWAP                  Jd   WAn        \        VRV4       \        VP                  RV4       V# )zCPrepare one named module for the forward role before replay starts.r#  ©Útarget_submoduler`   Úrolerç  rî  )r   r  r.   r—   Úshell_module_materializerz  )rÉ   rç  rè  rî  Úprepareds   &&&  rN   Ú!_prepare_named_module_for_forwardÚ.ModuleLooper._prepare_named_module_for_forwardq  sƒ   € ô # <×#6Ñ#6Ó7ˆØœDÔ Ø+ˆMà—?‘?×;Ñ;Ø)×0Ñ0Ø ØØ%ð	 <ó 
ˆð ×.Ñ.Ó.Ø"*Ôä�˜o¨}Ô=Ü�×#Ñ# _°mÔDØˆrM   c               ó(   € V ^8„  d   QhRRRRRRRR/# )rP   rÔ   r   rç  r   rè  ré  rT   rE   )rU   s   "rN   rV   r„   ‰  s2   € ÷ *ñ *à ð*ð "ð*ð &ð	*ð
 
ñ*rM   c                ó˜  €  \        VP                  4      pV P                  VVR7      p\	        VP
                  P                  R4      \        P                  P                  4      '       d@   V P                  P                  VP                  VRVR7      pWbP                  Jd   Wbn        M\        VP                  VR7       \        VP                  VRRR7       \        VR	V4       \        VP                  R	V4       \        R
\!        VRVP"                  4      VVR7       V P%                  WV4       V#   \         d    Rp EL1i ; i)zAPlace a named module and its processor task on the chosen device.N)rè  r  r*  r6  r	  Tr
  rî  Úquant_preparerë  )r  Úprevious_devicerî  )r   r  r¢  rþ  r²   rú  rk   r£  r&  r'  r—   r9  r&   r    rz  r   r^   rù  r1  )rÉ   rÔ   rç  rè  r@  rî  r:  s   &&&&   rN   Ú&_prepare_named_module_for_quantizationÚ3ModuleLooper._prepare_named_module_for_quantization‰  s4  € ð	#Ü(¨×)<Ñ)<Ó=ˆOð ×<Ñ<ØØ+ð =ó 
ˆô
 �l×(Ñ(×,Ñ,Ð-BÓCÄUÇXÁXÇ_Á_×UÒUØ—‘×?Ñ?Ø!-×!4Ñ!4Ø$Ø#Ø)ð	 @ó ˆHð ×2Ñ2Ó2Ø&.Ô#øä�L×'Ñ'°Õ>Ü × 3Ñ 3°]ÐTXÐgkÕlä�˜o¨}Ô=Ü�×#Ñ# _°mÔDÜØÜ˜<¨°l×6GÑ6GÓHØ+Ø'õ		
ð 	×#Ñ# I¸]ÔKàÐøôA ô 	#Ø"‹Oð	#ús   ‚D8 Ä8E	ÅE	r  r  Úprogress_pbÚprogress_titleÚprogress_stageÚprogress_rows_per_batchÚprogress_total_rowsÚforce_serialFÚpreserve_module_devicesÚapply_moe_configTc          .     ót   € V ^8„  d   QhRRRRRRRRR	R
RRRRRRRRRRRRRRRRRRRRRRRRRR R!R"R#RR$RR%RR&R
/# )'rP   r  r4  rÔ   r   r  rÙ  r  re   rÁ  úList[List[torch.Tensor]]Úlayer_input_kwargsúList[Dict[str, torch.Tensor]]Úposition_idsúList[torch.Tensor]Úattention_masksÚcur_layer_deviceré  Úis_lm_head_modulerº   Úshared_kv_cache_dictúDict[int, torch.Tensor]Úlayer_indexrö   Úneed_outputsÚreuse_kvrC  r<   rD  r?   rE  rF  úOptional[List[int]]rG  rC   rH  rI  rJ  rT   rE   )rU   s   "rN   rV   r„   µ  sò   € ÷ 4
ñ 4
ð  ð4
ð !ð	4
ð
 1ð4
ð 2ð4
ð /ð4
ð :ð4
ð )ð4
ð ,ð4
ð 'ð4
ð  ð4
ð 6ð4
ð ð4
ð ð4
ð  ð!4
ð" !ð#4
ð$ &ð%4
ð& &ð'4
ð( "5ð)4
ð* +ð+4
ð, ð-4
ð. "&ð/4
ð0 ð14
ð2 
"ñ34
rM   c               ón   € V P                   P                  VVVVVVVVV	V
VVVVVVVVVVVV\        R7      # )zIRun cached batches through the module using serial or parallel execution.)r  rÔ   r  r  rÁ  rM  rO  rQ  rR  rS  rT  rV  rW  rX  rC  rD  rE  rF  rG  rH  rI  rJ  Úselect_forward_devices_fn)rŸ   Úrunr!   )rÉ   r  rÔ   r  r  rÁ  rM  rO  rQ  rR  rS  rT  rV  rW  rX  rC  rD  rE  rF  rG  rH  rI  rJ  s   &$$$$$$$$$$$$$$$$$$$$$$rN   Ú_run_forward_batchesÚ!ModuleLooper._run_forward_batchesµ  sg   € ð8 ×%Ñ%×)Ñ)ØØØ)Ø!5Ø%Ø1Ø%Ø+Ø-Ø/Ø!5Ø#Ø%ØØ#Ø)Ø)Ø$;Ø 3Ø%Ø$;Ø-Ü&<ð/ *ó 
ð 	
rM   c          (     óh   € V ^8„  d   QhRRRRRRRRR	R
RR
RRRRRRRRRRRRRRRRRRRRRRRRR RR!R/# )"rP   r  r4  rÔ   r   rÁ  rL  rM  rN  rO  rP  rQ  rR  ré  rS  rº   rT  rU  rV  rö   rW  rX  rC  ú'ProgressBar' | NonerD  r?   rE  rF  rY  rG  rC   rI  rJ  rT   rE   )rU   s   "rN   rV   r„   ë  sÔ   € ÷ -
ñ -
ð  ð-
ð !ð	-
ð
 /ð-
ð :ð-
ð )ð-
ð ,ð-
ð 'ð-
ð  ð-
ð 6ð-
ð ð-
ð ð-
ð ð-
ð *ð-
ð  &ð!-
ð" &ð#-
ð$ "5ð%-
ð& +ð'-
ð( "&ð)-
ð* ð+-
ð, 
"ñ--
rM   c               ó^   € V P                   P                  VVVVVVVVV	V
VVVVVVVVVR7      # )zPRun cached batches on a single device and return ordered outputs when requested.)r  rÔ   rÁ  rM  rO  rQ  rR  rS  rT  rV  rW  rX  rC  rD  rE  rF  rG  rI  rJ  )rŸ   Ú
run_single)rÉ   r  rÔ   rÁ  rM  rO  rQ  rR  rS  rT  rV  rW  rX  rC  rD  rE  rF  rG  rI  rJ  s   &$$$$$$$$$$$$$$$$$$$rN   Ú_run_forward_batches_singleÚ(ModuleLooper._run_forward_batches_singleë  s[   € ð2 ×%Ñ%×0Ñ0ØØØ%Ø1Ø%Ø+Ø-Ø/Ø!5Ø#Ø%ØØ#Ø)Ø)Ø$;Ø 3Ø$;Ø-ð' 1ó 
ð 	
rM   c          (     óh   € V ^8„  d   QhRRRRRRRRR	R
RR
RRRRRRRRRRRRRRRRRRRRRRRR R!RR"R/# )#rP   r  r4  rÔ   r   rÁ  rL  rM  rN  rO  rP  rQ  rR  ré  rS  rº   rT  rU  rV  rö   rW  rX  ÚdevicesrS   rC  r`  rD  r?   rE  rF  rY  rG  rC   rJ  rT   rE   )rU   s   "rN   rV   r„     sÔ   € ÷ 0
ñ 0
ð  ð0
ð !ð	0
ð
 /ð0
ð :ð0
ð )ð0
ð ,ð0
ð 'ð0
ð  ð0
ð 6ð0
ð ð0
ð ð0
ð ð0
ð $ð0
ð  *ð!0
ð" &ð#0
ð$ &ð%0
ð& "5ð'0
ð( +ð)0
ð* ð+0
ð, 
"ñ-0
rM   c               ó|   € V P                   P                  VVVVVVVVV	V
VVVVVVVVV\        \        \        R7      # )zTRun cached batches across device replicas and preserve batch ordering in the result.)r  rÔ   rÁ  rM  rO  rQ  rR  rS  rT  rV  rW  rX  rf  rC  rD  rE  rF  rG  rJ  Úclone_module_for_devices_fnÚforward_batch_worker_fnÚdevice_thread_pool)rŸ   Úrun_parallelr   r   r	   )rÉ   r  rÔ   rÁ  rM  rO  rQ  rR  rS  rT  rV  rW  rX  rf  rC  rD  rE  rF  rG  rJ  s   &$$$$$$$$$$$$$$$$$$$rN   Ú_run_forward_batches_parallelÚ*ModuleLooper._run_forward_batches_parallel  sd   € ð2 ×%Ñ%×2Ñ2ØØØ%Ø1Ø%Ø+Ø-Ø/Ø!5Ø#Ø%ØØØ#Ø)Ø)Ø$;Ø 3Ø-Ü(@Ü$8Ü1ð- 3ó 
ð 	
rM   c               ó    € V ^8„  d   QhRRRR/# ©rP   rÔ   r   Úhook_sourcerA   rE   )rU   s   "rN   rV   r„   L  s   € ÷ Cñ C¨mð CÐVYñ CrM   c                ó    a aaa€ VVVV 3R lpV# )zHWrap a forward hook so it sees masked activations for the current batch.c                óö  <€ SP                  S4      '       d   R# SP                  S4      p\        SRR4      p\        SP                  RR4      pV'       d   \        P
                  ! 4       MRpTp \        V\        \        34      '       d´   \        V4      ^ 8”  d¤   \        P                  ! V^ ,          4      '       d�   V^ ,          pV'       gp   Vel   VP                  4       ^8¼  dW   \        Wƒ4      p	\        V\        4      '       d   V	3\        VR,          4      ,           pMV	.\        VR,          4      ,           pTp
 \        V\        \        34      '       d®   \        V4      ^ 8”  dž   V^ ,          pV'       gŒ   \        P                  ! V4      '       dp   Vel   VP                  4       ^8¼  dW   \        W³4      p\        V\        4      '       d   V3\        VR,          4      ,           p
McV.\        VR,          4      ,           p
MHV'       gA   \        P                  ! V4      '       d%   Ve!   VP                  4       ^8¼  d   \        W#4      p
 S! WV
4      Ve5   Ve0   VP                  R\        P
                  ! 4       V,
          SR7       # # #   \         d    Tp ELfi ; i  \         d    Tp
 Lei ; i  Te5   Te0   TP                  R\        P
                  ! 4       T,
          SR7       i i i ; i)zEApply the thread-local keep mask before delegating to ``inner_hook``.NÚpreserve_batch_keep_maskFÚquant_region_timerºrZ   NNÚforward_hook©Úsource©rŒ  r–  r^   r—   ÚtimeÚperf_counterr²   r»  rj   r¬   r£  r¤  Údimr   r¢  Úrecord)r  ÚinputsÚoutputÚkeeprs  ÚtimerÚstartÚ
new_inputsÚxÚxkÚ
new_outputÚy0Úykrp  Ú
inner_hookrÔ   rÉ   s   &&&          €€€€rN   ÚhookÚ/ModuleLooper._masked_hook_wrapper.<locals>.hookO  sw  ø€ ð ×/Ñ/°	×:Ò:Ùà×+Ñ+¨IÓ6ˆDÜ'.¨yÐ:TÐV[Ó'\Ð$ä˜DŸO™OÐ-AÀ4ÓHˆEß+0”D×%Ò%Ô'°dˆEð  ˆJð$Ü˜f¤u¬d m×4Ò4¼¸V»Àq¼ÌUÏ_Ê_Ð]cÐdeÕ]f×MgÒMgØ˜q�	�A÷ 4¸Ò8HÈQÏUÉUËWÐXYÌ\Ü/°Ó8˜Ü% f¬e×4Ò4Ø*,¨´°v¸bµzÓ1BÕ)B™Jà*,¨´°V¸BµZÓ0@Õ)@˜Jð  ˆJð$Ü˜f¤u¬d m×4Ò4¼¸V»Àq¼Ø ��Bç4Ü!ŸOšO¨B×/Ò/Ø Ò,ØŸF™F›H¨œMä/°Ó9˜Ü% f¬e×4Ò4Ø*,¨´°v¸bµzÓ1BÕ)B™Jà*,¨´°V¸BµZÓ0AÕ)A˜Jøç0ÜŸš¨×/Ò/ØÒ(ØŸ
™
›¨Ô)ä!3°FÓ!A�JðÙ! &°jÓAàÒ$¨Ò):Ø—L‘LØ&Ü×)Ò)Ó+¨eÕ3Ø*ð !õ ð *;Ð$øôA ô $à#“
ð$ûô6 ô $Ø#’
ð$ûð
 Ò$¨Ò):Ø—L‘LØ&Ü×)Ò)Ó+¨eÕ3Ø*ð !õ ð *;Ð$úsi   Á2AJ ÃJ ÃAJ Ä'J Å<J+ ÆJ+ ÆAJ+ Ç2J+ ÈJ+ ÈJ+ È1$J+ ÉJ> ÊJ(Ê'J(Ê+J;Ê:J;Ê>:K8rE   )rÉ   rÔ   r‰  rp  rŠ  s   ffff rN   Ú_masked_hook_wrapperÚ!ModuleLooper._masked_hook_wrapperL  s   û€ ÷?	ð ?	ð@ ˆrM   c               ó    € V ^8„  d   QhRRRR/# ro  rE   )rU   s   "rN   rV   r„   ‘  s   € ÷ .ñ .°-ð .ÐZ]ñ .rM   c                ó    a aaa€ VVVV 3R lpV# )zÜ
Pre-forward hook wrapper for MoE expert modules.
This is called BEFORE forward executes (when used with HookedLinear.forward_hook).
Respects hooks_paused state to avoid double-counting during intermediate calculations.
c                ó®  <€ SP                  S4      '       d   R# SP                  S4      p\        SRR4      p\        SP                  RR4      pV'       d   \        P
                  ! 4       MRpTp \        V\        \        34      '       d´   \        V4      ^ 8”  d¤   \        P                  ! V^ ,          4      '       d�   V^ ,          pV'       gp   Vel   VP                  4       ^8¼  dW   \        Wƒ4      p	\        V\        4      '       d   V	3\        VR,          4      ,           pMV	.\        VR,          4      ,           p S! WV4       Ve7   Ve1   VP                  R\        P
                  ! 4       V,
          S
R7       R# R# R#   \         d    Tp LUi ; i  Te5   Te0   TP                  R\        P
                  ! 4       T,
          S
R7       i i i ; i)zAApply the current keep mask before invoking the wrapped pre-hook.Nrs  Frt  ru  Úforward_pre_hookrw  ry  )r  r~  r  r€  rs  r�  r‚  rƒ  r„  r…  rp  r‰  rÔ   rÉ   s   &&&       €€€€rN   Úpre_hookÚ7ModuleLooper._masked_pre_hook_wrapper.<locals>.pre_hook—  sš  ø€ ð ×/Ñ/°	×:Ò:Ùð ×+Ñ+¨IÓ6ˆDÜ'.¨yÐ:TÐV[Ó'\Ð$ä˜DŸO™OÐ-AÀ4ÓHˆEß+0”D×%Ò%Ô'°dˆEð  ˆJð$Ü˜f¤u¬d m×4Ò4¼¸V»Àq¼ÌUÏ_Ê_Ð]cÐdeÕ]f×MgÒMgØ˜q�	�Aß3¸Ò8HÈQÏUÉUËWÐXYÌ\Ü/°Ó8˜Ü% f¬e×4Ò4Ø*,¨´°v¸bµzÓ1BÕ)B™Jà*,¨´°V¸BµZÓ0@Õ)@˜JðÙ˜6¨vÔ6àÒ$¨Ò):Ø—L‘LØ*Ü×)Ò)Ó+¨eÕ3Ø*ð !ö ñ *;Ñ$øô ô $à#’
ð$ûð Ò$¨Ò):Ø—L‘LØ*Ü×)Ò)Ó+¨eÕ3Ø*ð !õ ð *;Ð$ús2   Á2AF ÃF ÃAF Ä'F Å	F ÆFÆFÆ:GrE   )rÉ   rÔ   r‰  rp  r’  s   ffff rN   Ú_masked_pre_hook_wrapperÚ%ModuleLooper._masked_pre_hook_wrapper‘  s   û€ ÷&	ð &	ðP ˆrM   c                ód   € \        V \        R7      pVP                  VVVV P                  VR7      # )z@Capture and cache per-layer calibration inputs for later replay.r†   )ÚlayersÚcalibration_dataÚ	use_cacher˜   Úlayer_names)r8   rž   Úcache_inputsr˜   )rÉ   r—  r˜  r™  rš  Úcapture_stages   &&&&& rN   r›  ÚModuleLooper.cache_inputsÁ  s=   € ô +¨4¼Ô<ˆØ×)Ñ)ØØ-ØØ!×2Ñ2Ø#ð *ó 
ð 	
rM   c                óŒ   € \        4       ;_uu_ 4        V P                  ! RRV/VB uuRRR4       #   + '       g   i     R# ; i)z/Run the quantization loop under the TF32 guard.r²  NrE   )r/   Ú
_loop_impl)rÉ   r²  r  s   &&,rN   ÚloopÚModuleLooper.loopÍ  s0   € ô '×(Õ(Ø—?’?Ñ?¨HÐ?¸Ñ?÷ )×(×(Ó(ús	   “2²A	c                ó¦   a1a2€ Vf"   \        V P                  P                  RR4      pV P                  Ee½   . pV P                  \        P
                  \        P                  39   d)   VP                  RV P                  V P                  34       V P                  \        P                  \        P                  39   d)   VP                  RV P                  V P                  34       R^R^ RRR	R
RR/pV P                  P                  P                  ;'       g    / p/ pV F§  w  rxp	V	e	   V'       g   \        RV R24      h\        V	\!        \"        4      4      '       g0   \%        VP'                  4        R\)        V	4       R\"         R24      hV P                  P                  P+                  VRR7      e   Kš  \-        V4      Wh&   K©  	  V'       d    / VCVCV P                  P                  n        V P                  P                  P.                  '       EdO   V P                  P0                  P2                  P4                  '       d€   \7        V P                  P0                  P0                  R4      '       dP   V P                  P0                  P8                  p
V
 F)  pV P                  P.                  V9   g   K   \%        R4      h	  \;        V P                  P0                  V P                  P.                  R7      p\;        V P                  P0                  V P                  P.                  R7      f$   \        RV P                  P.                   R24      h\        V\!        \"        4      4      '       g   \%        R\)        V4       R\"         24      hR^R^ RRR	R
RR/pV P                  P                  P                  f3   V P                  P.                  V/V P                  P                  n        MsV P                  P                  P+                  V P                  P.                  RR7      f7   WÐP                  P                  P                  V P                  P.                  &   \7        V P                  P0                  P2                  R4      '       d+   V P                  P0                  P2                  P<                  MR
pR
V P                  P0                  P2                  n        \?        V P                  P0                  V P                  PA                  4       4      w  po2\        V P                  RR4      p\C        V PD                  4       EFo  w  ppVPG                  V4      '       Eg!   \        V\H        4      '       gs   \        V\J        4      '       d$   \        V P                  P                  RR4      f:   \        V\J        4      '       du   \        V P                  P                  RR4      eR   V PD                  V^,
          ,          pVPM                  VPN                  4       VPQ                  VPR                  4       MD\        V\T        4      '       d/   VPM                  . 4       VPQ                  \W        . . . . 4      4       EK?  V PY                  VS2VPN                  R
R7      pVPQ                  V4       EKr  	  V PD                   F  pVP[                  4        K  	  V P                  P                  P\                  '       dÞ   \^        Pa                  R4       V P                  Pc                  V P                  P0                  R 7      pV P                  eK   V P                  V P                  3 Uu0 uF  pV'       g   K  VkK  	  ppV Uu. uF  qˆV9  g   K  VNK  	  pp\]        V P                  P0                  VV P                  P                  Pd                  R!7       V PD                   F+  p\        V\f        4      '       g   K  VPi                  4        K-  	  Ve   VPk                  4        \l        ;QJ d&    R" V PD                   4       F  '       g   K   RM	  R
M! R" V PD                   4       4      p\l        ;QJ d&    R# V PD                   4       F  '       g   K   RM	  R
M! R# V PD                   4       4      pV P                  Po                  V P                  P0                  P2                  V P                  P                  VVR$7      pV P                  Pq                  V P                  P0                  P2                  VVR%7      pV P                  P                  Pr                  '       g   \u        V. 4      .p\w        V4      pV P                  \        P
                  \        P                  39   pV P                  \        P                  \        P                  39   pV'       d   ^M^ pV'       g'   V P                  P                  P.                  '       d   ^M^ p\^        Py                  VV,           V,           4      P{                  4       P}                  ^R&7      p V PD                   F  pVVn?        V Vn<        K  	  / p!^R'I@HAo1 . p"V'       d   V"P                  \…        R(R8R(4      4       V"P‡                  V1V23R) l\‰        V4       4       4       V'       d   V"P                  \…        R*VR*4      4       MBV P                  P                  P.                  '       d   V"P                  \…        R+VR+4      4       V PŠ                  P�                  \�        \‘        \!        V"4      4      V P                  \!        V PD                  4      V!V 4      4      V nI        V P                  P                  P.                  '       då   \;        V P                  P0                  V P                  P.                  R7      pV'       d¨   \        V\”        P–                  P˜                  4      '       d~   \š        Pœ                  ! V4      p#V P                  P.                  PŸ                  R4      p$V P                  P0                  p%V$RR8  F  p&\        V%V&4      p%K  	  \¡        V%V$R8,          V#4       \£        V VVVS2VV!V VV\¤        V P                  V P¦                  \^        R,7       V P©                  4        \ª        P¬                  ! 4        V P¯                  4        ^R-IXHYp' V'P­                  4        V P©                  4        / p(\µ        \·        V PD                  4      4      p)\w        V)4      p*\^        Py                  \‰        V*4      4      P{                  4       P}                  R
R.7      P¹                  R/4      P»                  R04      p+V+P½                  4         \C        V)^R17       EFÇ  w  p,p-V P©                  4        \        V-\J        4      '       d   M`\        V-\H        4      '       d   MI\        V-\T        4      '       d   M2\^        Pa                  V-P¿                  4        R2V-P^                   24       V-P¿                  4       p.V-P^                  V(V.&   V.R99   d   V-P^                  V P                  n`        V-P^                   F  p/\^        Pa                  V/4       K  	  V-PÃ                  4        Ve   \Ä        PÆ                  ! 4       MRp0 V-PÈ                  ! R:R3V P                  /VB  Ve3   V0e/   VPË                  R4\Ä        PÆ                  ! 4       V0,
          V.R57       T+P¹                  R6T, R7T* 24      P»                  T-P¿                  4       4      PÍ                  4       P½                  4        EKÊ  	  V+PÏ                  4        Ve   VPk                  4        WàP                  P0                  P2                  n        V(# u upi u upi   Te5   T0e0   TPË                  R4\Ä        PÆ                  ! 4       T0,
          T.R57       i i i ; i  T+PÏ                  4        i ; i);z6Execute the full layer-by-layer quantization workflow.Nr²  Úinputr  ÚbitsÚ
group_sizeÚsymTÚdesc_actFÚmseg333333@zCould not resolve the z% embedding module for requantization.z embedding type z' is not supported; supported types are ri   )ÚdefaultÚ_tied_weights_keyszƒquantization of `lm_head` layer with `tied_weights=True` model state is not supported. Please check model has `tied_weights=False`.)rü  zcould not find layer z in the model, exit...z
This type(zO) of lm_head quantization is currently not supported. SUPPORTS_MODULE_TYPES is r™  rt  ÚgptaqÚfoem)r—  rš  r˜  r™  z"Offloading base modules to disk...)r�   )r�   r  Ú	disk_pathc              3  óN   "  € T F  p\        V\        \        34      x € K  	  R # 5irñ  )r²   r0   r7   ©rò  Úprocs   & rN   rõ  Ú*ModuleLooper._loop_impl.<locals>.<genexpr>K  s#   é € ÐoÑ_nÐW[œj¨´Ô>PÐ/Q×RÐRÓ_nùs   ‚#%c              3  óR   "  € T F  p\        \        VR R4      RR4      x € K  	  R# 5i)Úexecution_configNÚenable_activation_captureFr‹  r¯  s   & rN   rõ  r±  N  s/   é € ð *
á'�ô ”G˜DÐ"4°dÓ;Ð=XÐZ_×`Ð`Û'ùs   ‚%')Úmodel_configrª   Úis_awq_quantizeÚinclude_capture_only)rµ  r¶  r·  )Úleft_steps_offset)Úget_layer_nameÚinput_embeddingsc           	   3  óL   <"  € T F  p\        R VS! SV4      4      x € K  	  R# 5i)r   N)r4   )rò  Úindexr¹  rš  s   & €€rN   rõ  r±  z  s+   øé € ð 6Ù"4˜ô ˜g u©n¸[È%Ó.P×QÐQÛ"4ùs   ƒ!$Úoutput_embeddingsÚlm_head)r—  Úlayer_modulesÚplanning_layer_modulesrš  r²  rT  ÚpbÚlayer_countÚregion_timerÚfinalize_progress_clsr˜   r™   r‡   )ÚSTREAM_DEVICE_POOL)Úshow_left_stepszProcessor finalizationÚ )r‚  z
 summary:
r�   Úprocess_finalizerw  zProcessor finalization Ú/éÿÿÿÿ)Úgptqzgptq v2ÚawqrE   )hr^   r—   rª   r˜   r   ÚINPUTÚBOTHrm   r}   rÆ   ÚOUTPUTr~   rÇ   Údynamicr\   r²   r»  r   ÚNotImplementedErrorÚ
capitalizerX   Údynamic_getÚdictr¾  r�   rÁ   Útie_word_embeddingsr€  rª  r$   r™  r#   Úextract_layers_nodeÚ	enumerater‚   Úverify_calibration_datasetr   r   Úset_calibration_datasetÚcalibration_datasetÚreceive_input_cacheÚinputs_cacher
   r   r›  Úrelease_calibration_datasetr*   rž   r©   Úget_base_modulesÚoffload_to_disk_pathr7   Úprewarm_runtimeÚflushrû  Úsimple_layer_modulesÚfull_layer_modulesÚtrue_sequentialÚsumr¬   rÁ  Úmanualrc  rÂ  Úutils.modelr¹  r4   ÚextendÚranger   r‚  r1   r3   Ú
start_stepr£  r&  ÚLinearr   Úfrom_linearrÉ  rz  r9   r>   r™   rj  r	   ri  rB  Úutils.streamrÅ  rj   ÚreversedÚtitleÚsubtitleÚdrawrù  Ú	quant_logÚ
log_plotlyrz  r{  Úfinalizer}  ÚnextÚclose)3rÉ   r²  r  Úrequested_endpointsÚdefault_configrÐ  Úmissing_defaultsrg   rù  r  Ú	tied_keysr¦  Úlm_head_moduleÚlm_head_quant_configÚforward_pass_use_cacher—  rÃ  Úp_indexrÔ   Úprev_processorÚinput_cacheÚbase_modulesÚendpoint_namesr¶  Úrequires_activation_capturer¿  rÀ  rÂ  Úquant_input_embeddingsÚquant_output_embeddingsÚextra_input_stepÚextra_output_steprÁ  rT  ÚstepsÚhooked_lm_headÚmodule_pathÚparentÚpartrÅ  Ú	total_logÚreversed_processorsÚprocess_finalize_totalÚprocess_finalize_pbr¼  Ú	reverse_pÚprocessor_nameÚ
module_logÚfinalize_startr¹  rš  s3   &&,                                              @@rN   rŸ  ÚModuleLooper._loop_implÓ  s  ù€ ð ÒÜ˜tŸ™×>Ñ>À
ÈDÓQˆHà× Ñ Ó,Ø"$ÐØ×$Ñ$¬×)<Ñ)<¼m×>PÑ>PÐ(QÔQØ#×*Ñ*Ø˜d×8Ñ8¸$×:VÑ:VÐWôð ×$Ñ$¬×)=Ñ)=¼}×?QÑ?QÐ(RÔRØ#×*Ñ*Ø˜t×:Ñ:¸D×<YÑ<YÐZôð
 ˜Ø˜bØ�tØ˜EØ�sðˆNð —o‘o×5Ñ5×=Ñ=×CÐCÀˆGØ!ÐÛ':Ñ#�˜VØ’>¯Ü$Ð'=¸e¸WÐDiÐ%jÓkÐkÜ! &¬%Ô0EÓ*F×GÒGÜ-Ø ×+Ñ+Ó-Ð.Ð.>¼tÀF»|¸nð M/Ü/DÐ.EÀQðHóð ð —?‘?×2Ñ2×>Ñ>¸tÈTÐ>ÓRÔZÜ-1°.Ó-AÐ$Ó*ñ (;÷  Ø:YÐ=MÐ:YÐQXÐ:Y�—‘×/Ñ/Ô7à�?‰?×*Ñ*×2×2Ñ2Ø�‰×$Ñ$×+Ñ+×?×?Ð?ÄGÈDÏOÉO×LaÑLa×LgÑLgÐi}×D~ÒD~Ø ŸO™O×1Ñ1×DÑD�	Û%�DØ—‘×.Ñ.°$Ö6Ü1ð  3xó  yð  yñ &ô (¨¯©×(=Ñ(=À4Ç?Á?×CZÑCZÔ[ˆNÜ˜$Ÿ/™/×/Ñ/°T·_±_×5LÑ5LÔMÒUÜ Ð#8¸¿¹×9PÑ9PÐ8QÐQgÐ!hÓiÐiä˜n¬eÔ4IÓ.J×KÒKÜ)¨J´t¸NÓ7KÐ6Lð MQÜQfÐPgð+ió jð jð %+¨A¨|¸RÀÈÈjÐZ_ÐafÐhkÐ#lÐ Ø�‰×.Ñ.×6Ñ6Ò>Ø;?¿?¹?×;RÑ;RÐThÐ:i�—‘×/Ñ/Õ7Ø—‘×0Ñ0×<Ñ<¸T¿_¹_×=TÑ=TÐ^bÐ<ÓcÒkØSg—‘×/Ñ/×7Ñ7¸¿¹×8OÑ8OÑPäKRÐSW×SbÑSb×ShÑSh×SoÑSoÐq|×K}ÒK} §¡×!6Ñ!6×!=Ñ!=×!GÒ!Gð  DIÐØ16ˆ�‰×Ñ×$Ñ$Ô.Ü6Ø�O‰O×!Ñ!Ø�O‰O×/Ñ/Ó1ó
Ñˆ�ô ˜tŸ™Ð0DÀdÓKˆä"+¨D¯O©O×"<ÑˆG�YØ×7Ñ7¸×@Ó@Ü˜i¬×7Ò7Ü# I¬}×=Ò=Ä'È$Ï/É/×JiÑJiÐkrÐtxÓByò  CFÜ# I¬}×=Ò=Ä'È$Ï/É/×JiÑJiÐkqÐswÓBxò  CEØ%)§_¡_°W¸qµ[Õ%A�NØ×5Ñ5°n×6XÑ6XÔYà×1Ñ1°.×2MÑ2MÕNÜ 	Ô+>×?Ò?à×5Ñ5°bÔ9Ø×1Ñ1´*¸RÀÀRÈÓ2LÔMâà×+Ñ+°6Ø8CØ=F×=ZÑ=ZØ6;ð ,ó =ˆKð ×)Ñ)¨+×6ñ) #=ð. ŸœˆIØ×1Ñ1Ö3ñ )ð �?‰?×*Ñ*×:×:Ð:Ü�H‰HÐ9Ô:ØŸ?™?×;Ñ;À$Ç/Á/×BWÑBWÐ;ÓXˆLØ×$Ñ$Ò0ð "&×!;Ñ!;¸T×=XÑ=XÑ Yó"á Y˜Ü÷ �DÙ Yð ð "ñ
 2>Ó\±¨È^ÑA[§ ±�Ð\ÜØ—o‘o×+Ñ+Ø#ØŸ/™/×9Ñ9×NÑNõð ŸœˆIô ˜)Ô%7×8Ô8Ø×)Ñ)Ö+ñ	 )ð Ò#Ø×ÑÔ ç›#ÑoÐ_c×_nÒ_nÓoŸ#Ÿ#š#ÑoÐ_c×_nÒ_nÓoÓoˆ÷ '*£cñ *
àŸšó*
§c§c¢cñ *
àŸšó*
ó '
Ð#ð Ÿ™×<Ñ<ØŸ™×.Ñ.×5Ñ5Ø ŸO™O×;Ñ;Ø+Ø!<ð	 =ó 
ˆð "&§¡×!CÑ!CØŸ™×.Ñ.×5Ñ5Ø+Ø!<ð "Dó "
Ðð �‰×.Ñ.×>×>Ð>Ü  °Ó3Ð4ˆMä˜&“kˆØ!%×!6Ñ!6¼=×;NÑ;NÔP]×PbÑPbÐ:cÑ!cÐØ"&×"7Ñ"7¼M×<PÑ<PÔR_×RdÑRdÐ;eÑ"eÐß 6™1¸AÐß"9¸T¿_¹_×=\Ñ=\×=d×=dÐ=d™AÐklÐÜ�f‰f�[Ð#3Õ3Ð6GÕGÓHß#™V›Xß ™S°1˜SÓ5ð 	ð ŸœˆIØ$/ˆIÔ!ØˆIŽLñ )ð  "Ðõ
 	1àˆß!Ø�L‰LœÐ"4°bÐ:LÓMÔNØ�‰õ 6Ü"'¨Ô"4ó6ô 	6ç"Ø�L‰LœÐ"5°{ÐDWÓXÕYØ�_‰_×,Ñ,×4×4Ð4Ø�L‰Lœ )¨[¸)ÓDÔEØŸ/™/×/Ñ/´Ü”U˜5“\Ó" D§O¡O´U¸4¿?¹?Ó5KÐMaÐcgó1
ó ˆŒð �?‰?×*Ñ*×2×2Ð2Ü'¨¯©×(=Ñ(=À4Ç?Á?×CZÑCZÔ[ˆNß¤*¨^¼U¿X¹X¿_¹_×"MÒ"MÜ!-×!9Ò!9¸.Ó!I�Ø"Ÿo™o×5Ñ5×;Ñ;¸CÓ@�ØŸ™×.Ñ.�Ø'¨¨Ó,�DÜ$ V¨TÓ2’Fñ -ä˜ ¨B¥°Ô@äØØØ'Ø#9Ø#ØØ!5ØØ#Ø%Ü"6Ø!×2Ñ2Ø—‘Üõ	
ð$ 	×ÑÔä×ÒÔ!à×"Ñ"Ô$å5Ø×ÑÔ!Ø×ÑÔð ˆ	Ü"¤8¨D¯O©OÓ#<Ó=Ðä!$Ð%8Ó!9Ðô —‘”uÐ3Ó4Ó5ß‘F“Hß‘C¨�CÓ.ß‘EÐ2Ó3ß‘H˜R“Lð 	ð 	× Ñ Ô"ð%	(Ü$-Ð.AÈ×$KÐ$KÑ ��yà×%Ñ%Ô'Ü˜i¬×7Ò7ØÜ 	¬=×9Ò9ØÜ 	Ô+>×?Ò?Øä—H‘H 	§¡Ó 0Ð1°¸Y¿]¹]¸OÐLÔMà!*§¡Ó!1�Ø,5¯M©M�	˜.Ñ)Ø!Ð%?Ô?Ø09·±�D—O‘OÔ-à"+§-¤-�JÜ—H‘H˜ZÖ(ñ #0à×$Ñ$Ô&à8DÒ8P¤×!2Ò!2Ô!4ÐVZ�ðØ×&Ò&ÑG¨T¯_©_ÐGÀÒGà#Ò/°NÒ4NØ$×+Ñ+Ø.Ü ×-Ò-Ó/°.Õ@Ø#1ð ,ô ð $×)Ñ)Ø-¨e¨W°AÐ6LÐ5MÐNóç‘(˜9Ÿ>™>Ó+Ó,¯T©T«V·D±D·FñE %LðH  ×%Ñ%Ô'àÒ#Ø×ÑÔ à1G�‰×Ñ×$Ñ$Ô.àÐùòe"ùò
  ]øðv $Ò/°NÒ4NØ$×+Ñ+Ø.Ü ×-Ò-Ó/°.Õ@Ø#1ð ,õ ð 5OÐ/ûð  ×%Ñ%Õ'úsG   Ý:
7Þ	7Þ<Þ"<÷D/A@> üA@ü"BA@> Á@:A@;Á@;A@> Á@>AAc               ó   € V ^8„  d   QhRR/# )rP   rT   r  rE   )rU   s   "rN   rV   r„   ê  s   € ÷ =ñ =ð  Siñ =rM   c
                ó”  € \        V P                  RR4      p
/ p/ pV F£  pRpVP                  \        4      '       d    RpVP	                  \        ^4      ^ ,          pWÒ9   d   W-,          W½&   MEV'       d   \        WR7      w  W½&   pM+V P                  P                  '       d   \        RV R24      hV'       g   KŸ  RWÍ&   K¥  	  . pV EF»  pVV P                  V P                  39   d   TpM%V'       d   V P                  P                  MV RV 2p\        VV,          \        4      '       g­   \        VV,          VVVR	7      pVP                  VR4      '       d   RVP                  R
&   \        V\        4      '       d/   VP                  P!                  RVP"                  V,          /4       VVV&   VVV&   V	e   VP                  P%                  RV	4       M.VP                  VR4      '       d   RVV,          P                  R
&   \        V\&        4      '       d   VP)                  VV,          WŠR7       MVP)                  VV,          4       VP+                  VV,          4      '       g   EKª  VP-                  V4       EK¾  	  V F9  pVP/                  V4       \        VRR4      pVf   K'  VP/                  VR4       K;  	  V# )zFBuild the named-module subset a processor will quantize for one layer.rt  NFT)rÇ  zlayer module item `z5` not found in model, please check your model config.ri   )rù  rë  rV  Úcapture_onlyÚwqÚlayer_module)r²  rÃ  r  )r^   r—   rÔ  r   rÉ  r%   Úlayer_modules_strictr\   r}   r~   r¾  r²   r   rk   rú  r   ÚupdateÚquantized_weightsÚ
setdefaultr   Ú
preprocessÚ
is_skippedrm   Úpop)rÉ   r  ÚfullrS  rV  Úlayers_prefixÚnamesrÔ   r²  r  rÃ  r  Úcapture_only_flagsÚnr  rÍ  Úskipped_modulesrù  Ú
layer_namerç  r(  s   &&&&&&&&&&           rN   Úcreate_named_modulesÚ!ModuleLooper.create_named_modulesê  s|  € ô ˜tŸ™Ð0DÀdÓKˆàˆØ.0ÐÛˆAØ ˆLØ�z‰zÔ+×,Ò,Ø#�Ø—G‘GÔ-¨qÓ1°!Õ4�ØŒyØ �G�’	ßä8¸ÔO‘�‘	™1ð —‘×5×5Ð5Ü Ð#6°q°cÐ9nÐ!oÓpÐpß‰|Ø(,Ð"Ó%ñ ð  ˆÜˆDØ˜×2Ñ2°D×4OÑ4OÐPÔPØ!‘
ç8I˜TŸ_™_×4Ò4ÐR_ÐQ`Ð`aÐbfÐagÐOh�
ô ˜f T�l¬K×8Ò8Ü*¨6°$­<¸dÈjØ7Bô D�à%×)Ñ)¨$°×6Ò6Ø9=�L×&Ñ& ~Ñ6Ü˜i¬×7Ò7Ø ×&Ñ&×-Ñ-Ø˜i×9Ñ9¸*ÕEð/ô ð  ,��t‘Ø)��T‘
ØÒ+Ø ×&Ñ&×1Ñ1°.À,ÔOøØ#×'Ñ'¨¨e×4Ò4Ø59��t•×"Ñ" >Ñ2ä˜)¤]×3Ò3Ø×$Ñ$ V¨D¥\¸HÐ$Õ`à×$Ñ$ V¨D¥\Ô2à×#Ñ# F¨4¥L×1Õ1Ø×&Ñ& t×,ñ= ó@ $ˆDØ�J‰J�tÔÜ˜y¨'°4Ó8ˆHØÔ#Ø—‘˜T 4Ö(ñ	 $ð
 ˆrM   )#r¿   r¥   r¦   r¸   r¶   r»   rŸ   r    r¢   r£   r¤   r¯   r¹   r½   r·   r¼   r«   r°   r®   r­   r¾   r™   r˜   r   r—   rÆ   r}   r�   rÃ   rÂ   rÇ   r~   r‚   rê  rš   rñ  rE   )9rF   rG   rH   rI   rJ   r}   rK   r~   r×   rÄ   r  r*  r/  r3  r:  rB  rI  rL  rE  r_  rd  rj  r^  rt  rÅ   r�  r‡  rŒ  r’  r–  r�  r¨  r³  r¾  rÄ  rÎ  rÖ  r   rä  rþ  r  r  r1  r;  rA  r]  rc  rl  rŒ  r”  r›  r   r£  Úinference_moderŸ  r)  rL   rE   rM   rN   r|   r|   ¥   sS  ‡ ñð ,0Ð˜=Ó/Ø,0Ð˜MÓ0ñJð ÷Jð JõX3
÷j ñ  ÷D?ñ ?õB(õ)õ3õòò
õõ.
õ,$õ
õ6õp)õ4ò+õõ
>õ
õ<õõõ<õ(õ(
õõ.÷+õZ
õ*ðX.ð
 BF÷.ð`@ð
 BF÷@õB3%õjõ0*ðX4
ð
 48ð4
ð 59ð4
ð" $(ð#4
ð$ )-ð%4
ð& )-ð'4
ð( 8<ð)4
ð* .2ð+4
ð, #ð-4
ð. ).ð/4
ð0 "&÷14
ðl-
ð -1ð-
ð  )-ð!-
ð" )-ð#-
ð$ 8<ð%-
ð& .2ð'-
ð( ).ð)-
ð* "&÷+-
ð^0
ð  -1ð!0
ð" )-ð#0
ð$ )-ð%0
ð& 8<ð'0
ð( .2ð)0
ð* "&÷+0
õdCõJ.ô`

ô@ð ×ÒÓóTó ðT÷l=ó =rM   r|   )mÚ__conditional_annotations__rJ   Ú
__future__r   rí   r›   rz  Útypingr   r   r   r   r   r   r£  Útorch.nnr&  rÇ  r	   Úlooper.dequantize_processorr
   Úlooper.eora_processorr   Úlooper.gptq_processorr   Úlooper.input_cacher   Úlooper.loop_processorr   Úlooper.named_moduler   Úmodelsr   Úmodels._constr   Úmodels.baser   Únn_modules.hooked_linearr   Úquantization.configr   r   r   r   Úutils.attn_maskr   Úutils.devicer   Úutils.device_telemetryr   Ú
utils.diskr   Úutils.loggerr   Úutils.looper_helpersr   r   r   r    r!   rç  r"   r#   r$   r%   r&   r'   r(   r)   Úutils.offloadr*   Úutils.pythonr+   r,   Úutils.torchr-   r.   r/   Úawq_processorr0   Ú	extensionr1   r2   r3   r4   Úexecution_stater5   Úforward_executorr6   Úparoquant_processorr7   Ústage_inputs_capturer8   Ústage_layerr9   rž   r;   rK   rœ   rx   Úlogbar.progressr<   r>   rb   rs   r¢  ru   rz   r|   )r,  s   @rN   Ú<module>rL     sþ   øðôõ #ã 	Û Û ß G× Gã Ý å !Ý =Ý 1Ý 1Ý +Ý 1Ý -Ý Ý 1Ý +Ý 3÷ó õ 1Ý %Ý :Ý /Ý '÷õ ÷	÷ 	ó 	õ ,ß <ß >Ñ >Ý 'ß FÓ FÝ 2Ý -Ý 3Ý 4Ý (áƒn€à&*Ð �OÓ *Ø —~’~Ó'Ð çÝ+ô˜:ô õõ$!ôH@�9ô @õôBÐ(ö BrM   