+
    "Ý³j“V  ã                  óì  a € 0 t $ R t^ RIHt ^ RIt^ RIt^ RIHt ^ RIH	t	H
t
 ^ RIt^RIHtHtHtHt ]P"                  ]P$                  ]P&                  0t/ tR]R&   ]P.                  ! 4       t]P.                  ! 4       tR R	 ltR
RR]P"                  /R R lltR R ltRLR R lltR R ltR R ltR R lt R R lt!]! RRRM]RR R]!R R R!R"R#R$7      t"R% t#R& R' lt$R( R) lt%R* R+ lt&R, R- lt'R. R/ lt(R0 R1 lt)R2 R3 lt*R4 R5 lt+R6 R7 lt,R8 R9 lt-R: R; lt.R< R= lt/RNR>R#R?R#/R@ RA lllt0RB RC lt1R
R/RD RE llt2RLRF RG llt3 ! RH RI]Ph                  Pj                  4      t6RLRJ RK llt7. ROOt8R# )Pz>Utility helpers for ParoQuant rotations and extension loading.)ÚannotationsN)ÚPath)ÚOptionalÚTuple)ÚTorchOpsJitExtensionÚdefault_jit_cflagsÚdefault_jit_cuda_cflagsÚdefault_torch_ops_build_rootz)dict[tuple[object, ...], tuple[int, int]]Ú_ROTATION_LAUNCH_CONFIG_CACHEc               ó$   € V ^8„  d   QhRRRRRR/# )é   Ú
group_sizeÚintÚin_featuresÚreturn© )Úformats   "ÚZ/home/luca/screen-checking/.venv/lib/python3.14/site-packages/gptqmodel/utils/paroquant.pyÚ__annotate__r   -   s!   € ÷ ñ  cð ¸ð Àñ ó    c                óØ   € V R	8X  d   TM
\        V 4      pV^ 8:  d   \        RV  RV R24      hW,          ^ 8w  d   \        RV RV R24      hV^,          ^ 8w  d   \        RV R24      hV# )
z.Validate and normalize a ParoQuant group size.zParoQuant: invalid group_size `z` for in_features=Ú.zParoQuant: in_features (z#) must be divisible by group_size (z).zParoQuant: group_size (z) must be even.éÿÿÿÿ)r   Ú
ValueError)r   r   Ú
normalizeds   && r   Ú_normalize_group_sizer   -   s‘   € à *¨bÔ 0‘´c¸*³o€JØ�Q„ÜÐ:¸:¸,ÐFXÐYdÐXeÐefÐgÓhÐhØÕ 1Ô$ÜØ& { mÐ3VÐWaÐVbÐbdÐeó
ð 	
ð �A…~˜ÔÜÐ2°:°,¸oÐNÓOÐOØÐr   ÚdeviceÚdtypec               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   r   r   r   Úkrotr   úOptional[torch.device | str]r   ztorch.dtyper   z/Tuple[torch.Tensor, torch.Tensor, torch.Tensor]r   )r   s   "r   r   r   ;   sN   € ÷ Oñ OàðOð ðOð ð	Oð
 )ðOð ðOð 5ñOr   c                ó¢  € \        W4      pV^ 8:  d   \        RV R24      h. p\        P                  ! V\        P                  R7      pW,          p\        V4       F  p	VP                  V4       K  	  \        P                  ! V^ R7      p
V
P                  ^ 4      P                  V^4      p\        P                  ! W ^,          3VR7      p\        P                  ! ^V 3VR7      pVe7   VP                  VR7      pVP                  VR7      pVP                  VR7      pVP                  4       VP                  4       VP                  4       3# )zFBuild the identity rotation buffers used as the default runtime state.z(ParoQuant: `krot` must be positive, got r   ©r   )Údim)r   )r   r   ÚtorchÚarangeÚint16ÚrangeÚappendÚcatÚ	unsqueezeÚrepeatÚzerosÚonesÚtoÚ
contiguous)r   r   r   r   r   Únormalized_group_sizeÚ	pairs_rowÚlocal_pairsÚ
num_groupsÚ_Úpairs_singleÚpairsÚthetaÚchannel_scaless   $$$$$         r   Úbuild_identity_rotation_buffersr9   ;   s%  € ô 2°*ÓJÐØˆq„yÜÐCÀDÀ6ÈÐKÓLÐLà€IÜ—,’,Ð4¼E¿K¹KÔH€KØÕ5€JÜ�:ÖˆØ×Ñ˜Ö%ñ ä—9’9˜Y¨AÔ.€LØ×"Ñ" 1Ó%×,Ñ,¨T°1Ó5€Eä�KŠK˜¨aÕ/Ð0¸Ô>€EÜ—Z’Z  KÐ 0¸Ô>€NàÒØ—‘ �Ó'ˆØ—‘ �Ó'ˆØ'×*Ñ*°&Ð*Ó9ˆà×ÑÓ˜u×/Ñ/Ó1°>×3LÑ3LÓ3NÐNÐNr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r7   útorch.Tensorr8   úOptional[torch.Tensor]r   Úboolr   )r   s   "r   r   r   [   s#   € ÷ 0ñ 0 ð 0Ð>Tð 0ÐY]ñ 0r   c                ó´   € V f   R# \         P                  ! V 4      P                  4       ^ 8w  d   R# Vf   R# \        \         P                  ! V^8H  4      4      # )z6Check whether a ParoQuant rotation reduces to a no-op.TF)r$   Úcount_nonzeroÚitemr=   Úall)r7   r8   s   &&r   Úis_identity_rotationrB   [   sK   € à‚}ÙÜ×Ò˜5Ó!×&Ñ&Ó(¨AÔ-ÙØÒÙÜ”—	’	˜.¨AÑ-Ó.Ó/Ð/r   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# ©
r   Úxr;   r6   r7   Úscalesr<   r   r   r   r   )r   s   "r   r   r   f   sD   € ÷ +#ñ +#Øð+#àð+#ð ð+#ð #ð	+#ð
 ð+#ð ñ+#r   c                óB  € V P                   pV P                  4       ^8  d   \        R\        V4       R24      hV P	                  RVR,          4      pVP                   R,          p\        WG4      pVeA   VP	                  ^R4      P                  VP                  VP                  R7      pWh,          p	MVP                  4       p	Wt,          p
V^,          pVP                  VP                  \        P                  R7      pVP                  VP                  \        P                  R7      p\        VP                   ^ ,          4       EFC  pV	P                  4       pW,          P	                  W«^4      pW,,          P	                  W«4      p\        V
4       Fï  pVV,          pVVVR^ 3,          ,           pVVVR^3,          ,           p\        P                  ! VV,          4      P                  V	P                  R7      p\        P                  ! VV,          4      P                  V	P                  R7      pV	RV3,          pV	RV3,          pVV,          VV,          ,           VRV3&   V) V,          VV,          ,           VRV3&   Kñ  	  Tp	EKF  	  V	P	                  V4      # )zFPure PyTorch reference implementation of the ParoQuant input rotation.z0ParoQuant rotation expects rank >= 2, got shape r   ©r   r   ºNNNr"   r   )Úshaper#   r   ÚtupleÚreshaper   r.   r   r   Úcloner$   Úint64Úfloat32r'   ÚcosÚsin)rE   r6   r7   rF   r   Ú
orig_shapeÚx2dÚhiddenÚscale_tensorÚoutr3   Ú
half_groupÚrot_idxÚnext_outÚpair_rowÚ	theta_rowÚ	group_idxÚgroup_offsetÚidx_iÚidx_jÚcos_tÚsin_tÚxiÚxjs   &&&&&                   r   Ú"apply_paroquant_rotation_referencerd   f   s'  € ð —‘€JØ‡u�uƒw�„{ÜÐKÌEÐR\ÓL]ÐK^Ð^_Ð`ÓaÐaà
�)‰)�B˜
 2�Ó
'€CØ�Y‰Y�r�]€FÜ& zÓ:€JàÒØ—~‘~ a¨Ó,×/Ñ/°s·z±zÈÏÉÐ/ÓSˆØÕ ‰à�i‰i‹kˆàÕ%€JØ˜q•€JØ�H‰H˜CŸJ™J¬e¯k©kˆHÓ:€EØ�H‰H˜CŸJ™J¬e¯m©mˆHÓ<€Eä˜Ÿ™ Q�×(ˆØ—9‘9“;ˆØ•>×)Ñ)¨*À!ÓDˆØ•N×*Ñ*¨:ÓBˆ	Ü˜zÖ*ˆIØ$ zÕ1ˆLØ  8¨I°q¸!¨OÕ#<Õ<ˆEØ  8¨I°q¸!¨OÕ#<Õ<ˆEÜ—I’I˜i¨	Õ2Ó3×6Ñ6¸S¿Y¹YÐ6ÓGˆEÜ—I’I˜i¨	Õ2Ó3×6Ñ6¸S¿Y¹YÐ6ÓGˆEØ�Q˜�X•ˆBØ�Q˜�X•ˆBØ!# e¥¨b°5­jÕ!8ˆH�Q˜�XÑØ"$  u¥¨r°E­zÕ!9ˆH�Q˜�XÓñ +ð ‹ñ )ð  �;‰;�zÓ"Ð"r   c               ó   € V ^8„  d   QhRR/# ©r   r   z	list[str]r   )r   s   "r   r   r   ”   s   € ÷ 
ñ 
˜9ñ 
r   c                 ó    € \        \        4      P                  4       P                  ^,          R,          R,          p \	        V R,          4      .# )a  Return the native extension sources for the fused CUDA rotation op.

Build this as a plain custom-op library instead of a Python extension.
The Python-module path pulls in pybind11 initialization that segfaults on
this host during ``PyInit_*`` even though the CUDA op itself is fine.
Úgptqmodel_extÚ	paroquantzrotation.cu)r   Ú__file__ÚresolveÚparentsÚstr)Úroots    r   Ú_rotation_sourcesro   ”   sB   € ô ”‹>×!Ñ!Ó#×+Ñ+¨AÕ.°Õ@À;ÕN€DäˆD�=Õ Ó!ðð r   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   rE   r;   r6   r7   rF   r<   r   r   r   r=   r   )r   s   "r   r   r   ¡   sD   € ÷ ñ Øðàðð ðð #ð	ð
 ðð 
ñr   c                óÂ  € V P                   P                  R8H  ;'       dÀ    V P                  \        9   ;'       d¥    VP                   P                  R8H  ;'       d„    VP                   P                  R8H  ;'       dc    VRJ ;'       g"    VP                   P                  R8H  ;'       d6    \	        V4      R9   ;'       d     \	        VP
                  ^ ,          4      R9   # )zCCheck whether the fused CUDA rotation kernel can service this call.ÚcudaN¾   é€   ¾   é   é   )r   Útyper   Ú!_SUPPORTED_ROTATION_KERNEL_DTYPESr   rJ   ©rE   r6   r7   rF   r   s   &&&&&r   Ú_rotation_kernel_readyr{   ¡   sÍ   € ð 	
�‰�‰˜Ñ÷ 	*ð 	*Ø�G‰GÔ8Ñ8÷	*ð 	*à�L‰L×Ñ Ñ'÷	*ð 	*ð �L‰L×Ñ Ñ'÷	*ð 	*ð �tˆ^×;Ð;˜vŸ}™}×1Ñ1°VÑ;÷		*ð 	*ô
 �
‹O˜uÑ$÷	*ð 	*ô �—‘˜A•Ó 6Ñ)ðr   c               ó   € V ^8„  d   QhRR/# )r   r   zset[tuple[int, int]]r   )r   s   "r   r   r   ´   s   € ÷ ñ Ð-Añ r   c                 ó>  € \         P                  P                  4       '       g   \        4       # \	        \         P                  P                  4       4       U u0 uF9  p \        \        \        \         P                  P                  V 4      4      4      kK;  	  up # u up i )z5Return the visible CUDA SM versions for this process.)
r$   rr   Úis_availableÚsetr'   Údevice_countrK   Úmapr   Úget_device_capability)Údevice_indexs    r   Ú#_rotation_visible_cuda_capabilitiesr„   ´   sr   € ô �:‰:×"Ñ"×$Ò$Ü‹uˆô "¤%§*¡*×"9Ñ"9Ó";Ô<óá<ˆLô 	Œc”#”u—z‘z×7Ñ7¸ÓEÓFÖGÙ<ñð ùò s   Á?Bc               ó   € V ^8„  d   QhRR/# rf   r   )r   s   "r   r   r   ¿   s   € ÷ ñ  Yñ r   c                 ó\   € \        4       p \        4       R08X  d   V P                  R4       V # )rw   z--expt-relaxed-constexpr)rw   é    )r   r„   r(   )Úflagss    r   Ú_rotation_extra_cuda_cflagsr‰   ¿   s+   € Ü#Ó%€Eô +Ó,°°Ô8Ø�‰Ð/Ô0Ø€Lr   Úgptqmodel_paroquant_rotationÚgptqmodel_paroquantÚGPTQMODEL_PAROQUANT_BUILD_ROOTc                 ó   € \        R 4      # )ri   )r	   r   r   r   Ú<lambda>rŽ   Ñ   s
   € Ô;¸KÔHr   zParoQuant rotationc                 ó   € \        R R7      # )ÚO2)Ú	opt_level)r   r   r   r   rŽ   rŽ   Ô   s
   € Ô+°dÕ;r   Ú!GPTQMODEL_PAROQUANT_FORCE_REBUILDÚGPTQMODEL_EXT_VERBOSETF)ÚnameÚ	namespaceÚrequired_opsÚsourcesÚbuild_root_envÚdefault_build_rootÚdisplay_nameÚextra_cuda_cflagsÚextra_cflagsÚforce_rebuild_envÚverbose_envÚrequires_cudaÚpython_abi_dependentc                 ó   € ^ RI Hp  V # )r‡   )Ú	extension)Ú	gptqmodelr¢   )Úextension_apis    r   Ú_extension_apir¥   Ü   s   € Ý4àÐr   c               ó   € V ^8„  d   QhRR/# ©r   r   ÚNoner   )r   s   "r   r   r   â   s   € ÷ 0ñ 0°$ñ 0r   c                 óB   € \        4        \        P                  4        R# )zLDelete cached ParoQuant rotation JIT artifacts before the next load attempt.N)Ú'clear_paroquant_rotation_autotune_cacheÚ_PAROQUANT_ROTATION_EXTENSIONÚclear_cacher   r   r   Ú(clear_paroquant_rotation_extension_cacher­   â   s   € ô ,Ô-Ü!×-Ñ-Ö/r   c               ó   € V ^8„  d   QhRR/# r§   r   )r   s   "r   r   r   é   s   € ÷ ñ °ñ r   c                 ó²   € \         ;_uu_ 4        \        P                  4        RRR4       \        R4      p V e
   V ! 4        R# R#   + '       g   i     L*; i)zBDrop the native launch-plan cache used by fused rotation autotune.NÚclear_autotune_cache)Ú"_ROTATION_LAUNCH_CONFIG_CACHE_LOCKr
   ÚclearÚ_rotation_native_op_if_loaded)r¬   s    r   rª   rª   é   s?   € ÷ 
,Ö	+Ü%×+Ñ+Ô-÷ 
,ä/Ð0FÓG€KØÒÙŽñ ÷ 
,×	+ús   �AÁA	c               ó   € V ^8„  d   QhRR/# ©r   r   r=   r   )r   s   "r   r   r   ó   s   € ÷ 6ñ 6 $ñ 6r   c                 ó4   € \        4       P                  R4      # )zCJIT-build and load the optional fused CUDA rotation extension once.ri   )r¥   r~   r   r   r   Ú_load_rotation_extensionr·   ó   s   € ô Ó×(Ñ(¨Ó5Ð5r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r”   rm   Úallowedzset[int]r   zOptional[int]r   )r   s   "r   r   r   ù   s!   € ÷ 	/ñ 	/˜Cð 	/¨(ð 	/°}ñ 	/r   c                óŠ   € \         P                  ! V 4      pVf   R#  \        V4      pY19   d   T# R#   \         d     R# i ; i)z-Parse a bounded integer environment override.N)ÚosÚgetenvr   r   )r”   r¹   ÚrawÚvalues   &&  r   Ú_rotation_env_intr¿   ù   sK   € ä
�)Š)�D‹/€CØ
‚{ÙðÜ�C“ˆð Ô$ˆ5Ð.¨$Ð.øô ô Úðús   ž3 ³AÁAc               ó$   € V ^8„  d   QhRRRRRR/# )r   r”   rm   Údefaultr=   r   r   )r   s   "r   r   r     s&   € ÷ Bñ B˜Sð B¨4ð B°Dñ Br   c                ó|   € \         P                  ! V 4      pVf   V# VP                  4       P                  4       R9  # )z;Parse a boolean environment flag used by rotation autotune.>   Ú0ÚnoÚoffÚfalse)r»   r¼   ÚstripÚlower)r”   rÁ   r½   s   && r   Ú_rotation_env_flagrÉ     s4   € ä
�)Š)�D‹/€CØ
‚{ØˆØ�9‰9‹;×ÑÓÐ&AÑAÐAr   c               ó   € V ^8„  d   QhRR/# rµ   r   )r   s   "r   r   r     s   € ÷ Kñ K Dñ Kr   c                 ó   € \        RR4      # )zJReport whether fused rotation launch autotune is enabled for this process.Ú#GPTQMODEL_PAROQUANT_ROTATE_AUTOTUNET)rÉ   r   r   r   Ú_rotation_autotune_enabledrÍ     s   € äÐCÀTÓJÐJr   c               ó   € V ^8„  d   QhRR/# )r   Úop_namerm   r   )r   s   "r   r   r     s   € ÷ 
ñ 
¨3ñ 
r   c                óÚ   € \         P                  pVe   \        W4      '       g   R# \         P                  P	                  V 4      pVe   V# \        W4      pV\         P                  V &   V# )zDReturn a loaded torch.ops handle without triggering JIT compilation.N)r«   Ú_namespace_cacheÚhasattrÚ	_op_cacheÚgetÚgetattr)rÏ   r•   ÚcachedÚops   &   r   r³   r³     sa   € ä-×>Ñ>€IØÒ¤¨	× ;Ò ;ÙÜ*×4Ñ4×8Ñ8¸ÓA€FØÒØˆÜ	�Ó	$€BØ79Ô!×+Ñ+¨GÑ4Ø€Ir   c               ó   € V ^8„  d   QhRR/# )r   r   útuple[int, int]r   )r   s   "r   r   r     s   € ÷ ñ  Oñ r   c                 ó¸   € \        R0 Rm4      p \        R^ ^04      pV f   Ve$   V f   RM
\        V 4      Vf   R3# \        V4      3# \        4       '       d   R# R# )zIResolve the requested fused launch policy from env overrides or autotune.Ú GPTQMODEL_PAROQUANT_ROTATE_CTA_MÚ"GPTQMODEL_PAROQUANT_ROTATE_ROW_PAD>   é   rw   é   r   )éþÿÿÿrß   )r   r   )r¿   r   rÍ   )Úcta_m_overrideÚrow_pad_overrides     r   Ú_rotation_requested_launchrâ     sx   € ä&Ð'IÊ:ÓV€NÜ(Ð)MÐPQÐSTÈvÓVÐØÒ!Ð%5Ò%Aà Ò(‰B¬c°.Ó.AØ"Ò*ˆBð
ð 	
ä03Ð4DÓ0Eð
ð 	
ô "×#Ò#ØˆØ€Or   c               ó   € V ^8„  d   QhRR/# )r   r   r   r   )r   s   "r   r   r   -  s   € ÷ Jñ J sñ Jr   c                 ó´   € \        R4      p V e   \        V ! 4       4      # \        4       '       g   ^ # \        \        4       P	                  RR4      ! 4       4      # )zNReturn the native fused-rotation autotune cache size for tests and benchmarks.Úautotune_cache_sizeri   )r³   r   r·   r¥   r×   )Ú
cache_sizes    r   Ú_rotation_autotune_cache_sizerç   -  sK   € ä.Ð/DÓE€JØÒÜ‘:“<Ó Ð Ü#×%Ò%ÙÜŒ~Ó×"Ñ" ;Ð0EÔFÓHÓIÐIr   c               ó8   € V ^8„  d   QhRRRRRRRRRRR	RR
RRR/# )r   rE   r;   r6   r7   rF   r<   r   r   Úcta_mÚrow_padr   r   )r   s   "r   r   r   7  sX   € ÷ ñ Øðàðð ðð #ð	ð
 ðð ðð ðð ñr   c               ó„   € \        4       P                  RR4      ! V VVV\        V4      \        V4      \        V4      4      # )z?Execute the fused rotation op with one requested launch policy.ri   Úrotate)r¥   r×   r   )rE   r6   r7   rF   r   ré   rê   s   &&&&&$$r   Ú_run_rotation_oprí   7  sB   € ô Ó×Ñ˜{¨HÔ5Ø	ØØØÜˆJ‹ÜˆE‹
ÜˆG‹óð r   c               ó4   € V ^8„  d   QhRRRRRRRRRRR	RR
R/# )r   rE   r;   r   r   Ú	has_scaler=   r   Úrequested_cta_mÚrequested_row_padr   ztuple[object, ...]r   )r   s   "r   r   r   M  sP   € ÷ ñ Øðð ðð ð	ð
 ðð ðð ðð ñr   c               ól  € \        V P                  R,          4      p\        V P                  4       V,          4      p\        V P                  P                  e   V P                  P                  MR4      V P
                  VV\        V4      \        V4      \        V4      \        V4      \        V4      3	# )z@Build the process-local cache key for one resolved launch shape.r   )r   rJ   Únumelr   Úindexr   r=   )rE   r   rï   r   rð   rñ   rT   Ú
batch_rowss   &$$$$$  r   Ú_rotation_launch_cache_keyrö   M  s‡   € ô �—‘˜•Ó€FÜ�Q—W‘W“Y &Õ(Ó)€Jä˜aŸh™hŸn™nÒ8ˆA�H‰H�NŠN¸bÓAØ	�‰ØØÜˆJ‹ÜˆD‹	ÜˆY‹ÜˆOÓÜÐÓð
ð 
r   c               ó4   € V ^8„  d   QhRRRRRRRRRRR	RR
R/# )r   rE   r;   rF   r<   r   r   r   rð   rñ   r   rÙ   r   )r   s   "r   r   r   f  sP   € ÷ +ñ +Øð+ð #ð+ð ð	+ð
 ð+ð ð+ð ð+ð ñ+r   c               ó:  € VR8w  d   VR8w  d   \        V4      \        V4      3# \        V VVRJVVVR7      p\        ;_uu_ 4        \        P	                  V4      pRRR4       Xe   V# \
        ;_uu_ 4        \        ;_uu_ 4        \        P	                  V4      pRRR4       Ve   VuuRRR4       # \        4       P                  RR4      ! V \        V4      VRJ\        V4      \        V4      \        V4      4      w  r‰\        V4      \        V	4      3p
\        ;_uu_ 4        \        P                  Wj4      pRRR4       VuuRRR4       #   + '       g   i     Lü; i  + '       g   i     LÎ; i  + '       g   i     LB; i  + '       g   i     R# ; i)zDResolve one fused launch config and memoize autotuned shape choices.N)r   rï   r   rð   rñ   ri   Úlaunch_configrß   )	r   rö   r±   r
   rÔ   Ú!_ROTATION_AUTOTUNE_SERIALIZE_LOCKr¥   r×   Ú
setdefault)rE   rF   r   r   rð   rñ   Ú	cache_keyrÖ   ré   rê   Úresolveds   &$$$$$     r   Ú_resolve_rotation_launchrþ   f  sN  € ð ˜"ÔÐ!2°bÔ!8Ü�?Ó#¤SÐ):Ó%;Ð;Ð;ä*Ø	ØØ Ð$ØØ'Ø+ô€I÷ 
,Ö	+Ü.×2Ñ2°9Ó=ˆ÷ 
,àÒØˆç	*Ö	*ß/Ö/Ü2×6Ñ6°yÓAˆF÷ 0àÒØ÷	 
+Ò	*ô (Ó)×,Ñ,¨[¸/ÔJØÜ�‹IØ˜$ÐÜ�
‹OÜ�Ó ÜÐ!Ó"ó
‰ˆô ˜“J¤ G£Ð-ˆß/Ö/Ü2×=Ñ=¸iÓRˆF÷ 0à÷# 
+Ò	*÷ 
,×	+ú÷ 0×/ú÷ 0×/ú÷ 
+×	*Ð	*úsO   ÁEÁ8F	ÂE#	ÂF	Â5A1F	Ä&E6	Ä<	F	ÅE 	Å#E3Å.F	Å6FÆF	Æ	F	Úextension_loadedÚkernel_readyc               ó8   € V ^8„  d   QhRRRRRRRRRRR	R
RR
RR/# )r   rE   r;   r6   r<   r7   rF   r   r   rÿ   r=   r   r   rÙ   r   )r   s   "r   r   r   ”  sX   € ÷ $$ñ $$Øð$$à!ð$$ð "ð$$ð #ð	$$ð
 ð$$ð ð$$ð ð$$ð ñ$$r   c          
     ó  € ??V P                   P                  R8w  d   \        R4      hV'       g   \        4       '       g   \	        R4      h\        4       w  rxVf   ^M\        VP                  ^ ,          4      p	VR8X  g   VR8X  d   \        V V\        V4      V	VVR7      w  r«MM\        4       P                  RR4      ! V \        V	4      VRJ\        V4      \        V4      \        V4      4      w  r«\        V
4      \        V4      3# )	z?Query the native fused-kernel launch shape for one CUDA tensor.rr   z/ParoQuant launch config requires a CUDA tensor.z>ParoQuant launch config requires the fused rotation extension.N©rF   r   r   rð   rñ   ri   rù   rß   )r   rx   r   r·   ÚRuntimeErrorrâ   r   rJ   rþ   r¥   r×   )rE   r6   r7   rF   r   rÿ   r   rð   rñ   r   ré   rê   s   &&&&&$$     r   Ú_rotation_launch_configr  ”  sñ   € ð 	ˆ|Ø‡x�x‡}�}˜ÔÜÐJÓKÐKßÔ$<×$>Ò$>ÜÐ[Ó\Ð\Ü)CÓ)EÑ&€OØ’‰1¤3 u§{¡{°1¥~Ó#6€DØ˜"ÔÐ 1°RÔ 7Ü1ØØÜ˜:“ØØ+Ø/ô
‰ˆˆwô (Ó)×,Ñ,¨[¸/ÔJØÜ�‹IØ˜$ÐÜ�
‹OÜ�Ó ÜÐ!Ó"ó
‰ˆô ˆu‹:”s˜7“|Ð#Ð#r   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# rD   r   )r   s   "r   r   r   »  sD   € ÷ ñ Øðàðð ðð #ð	ð
 ðð ñr   c           
     óª   € \        4       w  rV\        V V\        V4      \        VP                  ^ ,          4      VVR7      w  rx\	        V VVVVVVR7      # )zORun the fused rotation op with explicit overrides or native autotune sentinels.r  )ré   rê   )râ   rþ   r   rJ   rí   )	rE   r6   r7   rF   r   rð   rñ   ré   rê   s	   &&&&&    r   Ú_apply_fused_rotationr  »  sc   € ô *DÓ)EÑ&€OÜ-Ø	ØÜ�z“?Ü�—‘˜Q•Ó Ø'Ø+ô�N€Eô Ø	ØØØØØØôð r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )	r   Úfused_rotationr=   r   r   r   r   r    r   r   )r   s   "r   r   r   ×  s<   € ÷ &ñ &àð&ð ð&ð ð	&ð
 )ð&ð 
ñ&r   c                ó  € V '       g   R# \        V4      R9  d   R# \        V4      R9  d   R# Ve(   \        P                  ! V4      P                  R8w  d   R# Vf'   \        P                  P                  4       '       g   R# \        4       # )zLEagerly build the fused rotation extension before timed quantization starts.Frr   rs   ru   )r   r$   r   rx   rr   r~   r·   )r
  r   r   r   s   $$$$r   Ú$prewarm_paroquant_rotation_extensionr  ×  sj   € ÷ ÙÜ
ˆ:ƒ˜eÔ#ÙÜ
ˆ4ƒy˜ÔÙàÒœeŸlšl¨6Ó2×7Ñ7¸6ÔAÙØ‚~œeŸj™j×5Ñ5×7Ò7Ùä#Ó%Ð%r   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# rD   r   )r   s   "r   r   r   î  óL   € ÷ 
eñ 
eØð
eàð
eð ð
eð #ð	
eð
 ð
eð ñ
er   c           	     ó�   € \        WW#V4      '       d'   \        4       '       d   \        WW#\        V4      4      # \	        WW#VR7      # )zNApply the fused rotation when available, else fall back to the reference path.©rF   r   )r{   r·   r  r   rd   rz   s   &&&&&r   Úapply_paroquant_rotationr  î  s?   € ô ˜a¨°z×BÒBÔG_×GaÒGaÜ$ Q¨u¼cÀ*»oÓNÐNÜ-¨a¸ÐYcÔdÐdr   c                  óJ   € ] tR tRtRt]R	R R ll4       t]R R l4       tRtR# )
Ú_ParoQuantRotateTensorFunciû  zAAutograd wrapper around the fused ParoQuant CUDA rotation kernel.Nc               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# rD   r   )r   s   "r   r   Ú'_ParoQuantRotateTensorFunc.__annotate__ÿ  sF   € ÷ ñ àðð ðð ð	ð
 'ðð ðð 
ñr   c           	     	óÞ  € Vf   R MVP                  4       pVP                  V n        VP                  V n        VR JV n        \        V4      V n        \        VP                  4      V n
        \        VP                  4      V n        \        VR J;'       d    VP                  4      V n        \        WW6\        V4      4      pV P
                  '       d   WW7V3MWW73pV P                  ! V!   V# )N)r/   rJ   rR   r   Ú
orig_dtyperï   r   r   r=   Úrequires_gradÚneeds_x_gradÚneeds_theta_gradÚneeds_scale_gradr  Úsave_for_backward)	ÚctxrE   r6   r7   rF   r   rU   ÚyÚsaveds	   &&&&&&   r   ÚforwardÚ"_ParoQuantRotateTensorFunc.forwardþ  sÊ   € ð  &š~‘t°6×3DÑ3DÓ3FˆØŸ™ˆŒØŸ™ˆŒØ$¨DÐ0ˆŒÜ˜Z›ˆŒÜ §¡Ó0ˆÔÜ# E×$7Ñ$7Ó8ˆÔÜ# L¸Ð$<×$[Ð$[À×A[ÑA[Ó\ˆÔä! !¨EÄÀZÃÓQˆØ69·m·m°m�˜5 \Ñ2È!ÐTYÐI]ˆØ×Ò˜uÒ%Øˆr   c               ó   € V ^8„  d   QhRR/# )r   Úgrad_outr;   r   )r   s   "r   r   r    s   € ÷ D:ñ D: ñ D:r   c                	óŒ	  € V P                   pVR ,          w  r4rVV P                  '       d
   V^,          MRpV P                  pVP                  w  ršW¨,          pV^,          pVP	                  4       V
,          pVP                  WÚ4      pRpV P                  '       Edv   VP                  WÚ4      p\        P                  ! V4      p\        P                  ! W´P                  \        P                  R7      P                  ^4      V,          p\        VP                  ^ ,          ^,
          RR4       EFÝ  pVP                  ^ V^4      pVP                  ^ V^4      P                  4       p\!        VVVRV4      p\!        VVVRV4      pVP                  W¸4      pVR,          V,           P                  R4      pVR,          V,           P                  R4      pVRV3,          P                  WÛV4      pVRV3,          P                  WÛV4      pVRV3,          P                  WÛV4      pVRV3,          P                  WÛV4      pVP                  ^ V^4      P                  W¼4      pVP#                  4       pVP%                  4       pVV,          VV,          ,
          P'                  ^ 4      V,          VV,          VV,          ,           P'                  ^ 4      V,          ,
          P                  R4      P)                  VP*                  4      VV&   EKà  	  Mo\        VP                  ^ ,          ^,
          RR4       FF  pVP                  ^ V^4      pVP                  ^ V^4      P                  4       p\!        VVVRV4      pKH  	  V P                  '       dâ   VP                  R4      pRp V P,                  '       dK   VVP                  ^ 4      ,          P                  V P.                  4      P)                  V P0                  4      p Rp!V P2                  '       d^   VP                  WÚ4      V,          P'                  ^ 4      P)                  VP*                  VP                  R7      p!V!P5                  V4      p!MJV P,                  '       d5   VP                  V P.                  4      P)                  V P0                  4      MRp Rp!V RVV!R3# ):NrÝ   NNrH   rI   )r   r   r   )rI   :r‡   Nr   )rI   :rv   Nr   )Úsaved_tensorsrï   r   rJ   ró   rL   r  r$   Ú
zeros_liker%   r   Úlongr*   r'   ÚnarrowÚnegr  rQ   rP   Úsumr.   r   r  rR   r  r  Ú
reshape_as)"r  r#  r%  rE   r6   r7   r  rU   r   r4   rT   r3   rW   rõ   ÚgradÚ
grad_thetaÚrotatedÚoffsetsrX   rZ   Ú	neg_thetaÚ	pair_viewr^   r_   rb   rc   Úgrad_iÚgrad_jÚ
theta_viewra   r`   Ú
scale_flatÚgrad_xÚ
grad_scales"   &&                                r   ÚbackwardÚ#_ParoQuantRotateTensorFunc.backward  sÿ  € à×)Ñ)ˆØ*¨2Õ.Ñˆ�%Ø+.¯=¯=¨=�} QÖ'¸dˆØ—^‘^ˆ
à—K‘K‰	ˆØÕ)ˆ
Ø 1•_ˆ
Ø—W‘W“Y &Õ(ˆ
Ø×Ñ 
Ó3ˆØˆ
à××ÑØ—i‘i 
Ó3ˆGÜ×)Ò)¨%Ó0ˆJä—’˜Z·±ÄEÇJÁJÔO×YÑYÐZ[Ó\Ð_iÕið ô ! §¡¨Q¥°!Õ!3°R¸×<�Ø Ÿ<™<¨¨7°AÓ6�Ø!ŸL™L¨¨G°QÓ7×;Ñ;Ó=�	Ü/°¸À9ÈdÐT^Ó_�Ü,¨T°8¸YÈÈjÓY�à$×,Ñ,¨ZÓD�	Ø" 7Õ+¨gÕ5×>Ñ>¸rÓB�Ø" 7Õ+¨gÕ5×>Ñ>¸rÓB�à˜Q ˜XÕ&×.Ñ.¨zÀzÓR�Ø˜Q ˜XÕ&×.Ñ.¨zÀzÓR�Ø˜a ˜h�×/Ñ/°
È
ÓS�Ø˜a ˜h�×/Ñ/°
È
ÓS�à"Ÿ\™\¨!¨W°aÓ8×@Ñ@ÀÓX�
Ø"Ÿ™Ó(�Ø"Ÿ™Ó(�ð   "� v°¥{Õ2×7Ñ7¸Ó:¸UÕBØ! B�;¨°"­Õ4×9Ñ9¸!Ó<¸uÕDõE÷ ‘W˜R“[ß‘R˜Ÿ™“_ð ˜7Ô#ò% =ô6 ! §¡¨Q¥°!Õ!3°R¸Ö<�Ø Ÿ<™<¨¨7°AÓ6�Ø!ŸL™L¨¨G°QÓ7×;Ñ;Ó=�	Ü,¨T°8¸YÈÈjÓY’ñ =ð
 �=�=ˆ=Ø%×-Ñ-¨bÓ1ˆJØˆFØ××ÐØ ×!5Ñ!5°aÓ!8Õ8×AÑAÀ#Ç.Á.ÓQ×TÑTÐUX×UcÑUcÓd�ØˆJØ×#×#Ð#ØŸi™i¨
Ó;¸dÕB×GÑGÈÓJ×MÑMØ&×,Ñ,Ø'×.Ñ.ð Nó �
ð (×2Ñ2°<Ó@�
øàHK×HX×HXÐHX�T—\‘\ #§.¡.Ó1×4Ñ4°S·^±^ÔDÐ^bˆFØˆJà�t˜Z¨°TÐ9Ð9r   r   ©Nrt   )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ústaticmethodr   r8  Ú__static_attributes__r   r   r   r  r  û  s-   † ÙKàöó ðð, ôD:ó ôD:r   r  c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# rD   r   )r   s   "r   r   r   ]  r  r   c           	     ó¤   € \        WW#V4      '       d1   \        4       '       d!   \        P                  WW#\	        V4      4      # \        WW#VR7      # )zEApply the fused rotation with custom backward support when available.r  )r{   r·   r  Úapplyr   rd   rz   s   &&&&&r   Ú!apply_paroquant_rotation_autogradrE  ]  sE   € ô ˜a¨°z×BÒBÔG_×GaÒGaÜ)×/Ñ/°¸%ÌÈZËÓYÐYÜ-¨a¸ÐYcÔdÐdr   r:  )rì   rù   r°   rå   )NNNrt   )	r  rE  rd   r  r9   rª   r­   rB   r  )9Ú__conditional_annotations__r?  Ú
__future__r   r»   Ú	threadingÚpathlibr   Útypingr   r   r$   Úcppr   r   r   r	   Úfloat16Úbfloat16rO   ry   r
   Ú__annotations__ÚLockr±   rú   r   r9   rB   rd   ro   r{   r„   r‰   r«   r¥   r­   rª   r·   r¿   rÉ   rÍ   r³   râ   rç   rí   rö   rþ   r  r  r  r  ÚautogradÚFunctionr  rE  Ú__all__)rF  s   @r   Ú<module>rS     s�  øðô Eå "ã 	Û Ý ß "ã ÷ó ð 
‡M�MØ	‡N�NØ	‡M�Mð%Ð !ð LNÐ ÐHÓ Mð &/§^¢^Ó%5Ð "Ø$-§N¢NÓ$4Ð !õðOð
 ,0ðOð Ÿ™÷Oõ@0÷+#õ\
õõ&õñ !5Ø	'Ø#Ø[ØØ3ÙHØ%Ø1Ù;Ø9Ø'ØØô!Ð ò"õ0õõ6õ	/õBõKõ

õõJõõ,õ2+ñ\$$ð #ð$$ð ÷$$ð $$õNð8&ð
 ,0÷&÷.
eô_: §¡×!8Ñ!8ô _:÷D
eò
‚r   