+
    "Ý³j¨Š  ã                  ó”  a € 0 t $ ^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RI	H
t
 ^ RIHtHtHtHtHtHt ^ RIt^ RIt^RIHt ^RIHtHtHtHtHtHtHt ^RIHt ^R	I H!t! ]! 4       t"R
t#Rt$Rt%Rt&R“t'R R lt(R R lt)R R lt*R R lt+R R lt,]+! 4       ;'       g    Rt-R R lt.R R lt/R R lt0R R lt1R  R! lt2R" R# lt3R$ R% lt4R& R' lt5R( R) lt6]! ]#]$R”R* R+R, R-]5]6]4R.R/R0R1R27      t7]! ]%]&R•R3 R4R5 R6]5]6]4R.R/R0R1R27      t8R7 t9R8 R9 lt:R: R; lt;R< R= lt<R–R> R? llt=R–R@ RA llt>RB RC lt?RD RE lt@R–RF RG lltARH RI ltBRJ RK ltCR—RL RM lltDRN RO ltERP RQ ltFR˜RR RS lltGRT RU ltHRV RW ltIRX RY ltJRZ R[ ltKR\ R] ltLR^ R_ ltMR` Ra ltNRb Rc ltORd Re ltPRf Rg ltQRh Ri ltRRj Rk ltSRl Rm ltTRn tU/ tVRo]WRp&   ]P°                  ! 4       tYRq Rr ltZRs t[Rt t\Ru Rv lt]R™Rw Rx llt^RyRRzR0R{R1/R| R} llt_RšR~ R llt`RyRRzR0/R€ R� lltaR›R‚ Rƒ lltbR–R„ R… lltcR–R† R‡ lltdRˆ teR‰ RŠ ltfR‹ RŒ ltgR� RŽ lthR� R� ltiR‘ R’ ltjR# )œé    )ÚannotationsN)ÚPath)Úwhich)ÚCallableÚDictÚListÚOptionalÚTupleÚUnion)Úsetup_logger)ÚTorchOpsJitExtensionÚ cuda_include_paths_with_fallbackÚdefault_jit_cflagsÚdefault_jit_cuda_cflagsÚdefault_torch_ops_build_rootÚ!detected_cuda_wheel_include_pathsÚis_nvcc_compatible)Ú
ScalarType)ÚIS_ROCMÚgptqmodel_marlin_fp16_opsÚgptqmodel_marlin_fp16Úgptqmodel_marlin_bf16_opsÚgptqmodel_marlin_bf16c               ó$   € V ^8„  d   QhRRRRRR/# )é   ÚmajorÚintÚminorÚreturnÚbool© )Úformats   "ÚW/home/luca/screen-checking/.venv/lib/python3.14/site-packages/gptqmodel/utils/marlin.pyÚ__annotate__r$   /   s!   € ÷ 4ñ 4¨ð 4°Cð 4¸Dñ 4ó    c                ó@   € V ^8„  ;'       g    V ^8H  ;'       d    V^8¬  # )é   r!   )r   r   s   &&r#   Ú_marlin_capability_supportedr(   /   s$   € Ø�1‰9×3Ð3˜ !™×2Ð2¨°©
Ð3r%   c               ó    € V ^8„  d   QhRRRR/# )r   Úmin_capabilityúTuple[int, int]r   r    r!   )r"   s   "r#   r$   r$   3   s   € ÷ ñ ¸/ð Èdñ r%   c                óò   a € \         P                  P                  4       pV^ 8„  ;'       dM    \        ;QJ d(    V 3R l\	        V4       4       F  '       d   K   R# 	  R# ! V 3R l\	        V4       4       4      # )r   c              3  óh   <"  € T F'  p\         P                  P                  V4      S8¬  x € K)  	  R # 5i©N)ÚtorchÚcudaÚget_device_capability)Ú.0Úindexr*   s   & €r#   Ú	<genexpr>Ú8_marlin_all_visible_devices_supported.<locals>.<genexpr>6   s,   øé € ð $á(ˆEô 	�
‰
×(Ñ(¨Ó/°>ÖAÛ(ùs   ƒ/2FT)r/   r0   Údevice_countÚallÚrange)r*   r6   s   f r#   Ú%_marlin_all_visible_devices_supportedr9   3   sk   ø€ ä—:‘:×*Ñ*Ó,€LØ˜!Ñ÷ ð §£ô $ä˜<Ô(ó$§¤ð ¢ð  ô $ä˜<Ô(ó$ó !ð r%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Údeviceútorch.devicer*   r+   Úbackend_nameÚstrr   r!   )r"   s   "r#   r$   r$   <   s2   € ÷ ñ Øðð (ðð ð	ð
 ñr%   c          
     óÒ  € \         P                  ! V 4      p \        '       d   \        V R24      hV P                  R8w  d   \        V RV  R24      h \         P
                  P                  V 4      pY18  dH   RP                  R	 T 4       4      pRP                  R
 T 4       4      p\        T RT RT RT  R24      hT#   \         d   p\        T RT  RT 24      ThRp?ii ; i)z?Validate the CUDA device that owns one Marlin module's weights.z is not supported on ROCm.r0   z requires CUDA tensors, got `ú`.z failed to query CUDA device `z` capability: NÚ.c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5ir.   ©r>   ©r2   Úparts   & r#   r4   Ú1marlin_validate_runtime_device.<locals>.<genexpr>Q   s   é € Ð@±¨œ3˜tŸ9˜9³ùó   ‚c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5ir.   rC   rD   s   & r#   r4   rF   R   s   é € Ð=±*¨$œC ŸI˜I³*ùrG   z  requires compute capability >= ú, got z on `)	r/   r;   r   Ú
ValueErrorÚtyper0   r1   Ú	ExceptionÚjoin)r;   r*   r=   Ú
capabilityÚexcÚminimumÚdetecteds   &$$    r#   Úmarlin_validate_runtime_devicerR   <   s
  € ô �\Š\˜&Ó!€FßƒwÜ˜L˜>Ð)CÐDÓEÐEØ‡{�{�fÔÜ˜L˜>Ð)FÀvÀhÈbÐQÓRÐRðÜ—Z‘Z×5Ñ5°fÓ=ˆ
ð Ô"Ø—(‘(Ñ@±Ó@Ó@ˆØ—8‘8Ñ=±*Ó=Ó=ˆÜØˆnÐ<¸W¸Ið FØ�*˜E & ¨ð-ó
ð 	
ð Ðøô ô ÜØˆnÐ:¸6¸(À.ÐQTÐPUÐVó
àð	ûðús   ÁC ÃC&ÃC!Ã!C&c               ó   € V ^8„  d   QhRR/# ©r   r   r>   r!   )r"   s   "r#   r$   r$   Z   s   € ÷ ñ ¨ñ r%   c                 ón   € \         '       d   R # \        P                  P                  4       '       g   R# R# )z'Marlin kernel is not supported on ROCm.zMarlin kernel requires CUDA.Ú )r   r/   r0   Úis_availabler!   r%   r#   Ú_marlin_build_environment_errorrX   Z   s#   € ßƒwÙ8Ü�:‰:×"Ñ"×$Ò$Ù-Ùr%   c               ó   € V ^8„  d   QhRR/# rT   r!   )r"   s   "r#   r$   r$   b   s   € ÷ 
ñ 
 3ñ 
r%   c                 óÞ   € \        4       p V '       d   V #  \        P                  P                  4       w  r\        Y4      '       g
   RT RT R2# R#   \         d   pR T 2u Rp?# Rp?ii ; i)z6Marlin kernel failed to query CUDA device capability: Nz6Marlin kernel requires compute capability >= 7.5, got rA   rV   )rX   r/   r0   r1   rL   r(   )Úbuild_errorr   r   rO   s       r#   Ú_marlin_environment_errorr\   b   su   € Ü1Ó3€KßØÐðNÜ—z‘z×7Ñ7Ó9‰ˆô (¨×5Ò5ØGÈÀwÈaÐPUÈwÐVWÐXÐXÙøô	 ô NØGÈÀuÐMÕMûðNús   – A ÁA,ÁA'Á!A,Á'A,c               ó   € V ^8„  d   QhRR/# ©r   r   r   r!   )r"   s   "r#   r$   r$   t   s   € ÷ Lñ L�dñ Lr%   c                 óz   € \        \        4      P                  4       P                  ^,          R,          R,          # )r   Úgptqmodel_extÚmarlin)r   Ú__file__ÚresolveÚparentsr!   r%   r#   Ú_marlin_rootre   t   s*   € Ü”‹>×!Ñ!Ó#×+Ñ+¨AÕ.°Õ@À8ÕKÐKr%   c               ó   € V ^8„  d   QhRR/# )r   r   z
str | Noner!   )r"   s   "r#   r$   r$   x   s   € ÷ ñ  ñ r%   c                 óþ   € \        \        P                  R R4      p V '       g   R#  \        \	        V 4      P                  R^R7      ^ ,          4      pT^8¼  d   R# T^8X  d   R# R#   \        \        3 d     R# i ; i)r0   NrA   )Úmaxsplitzmarlin-cudazmarlin-cuda12)Úgetattrr/   Úversionr   r>   ÚsplitÚ	TypeErrorrJ   )Úrawr   s     r#   Ú_marlin_cuda_extra_namern   x   sq   € Ü
”%—-‘- ¨Ó
.€CßÙðÜ”C˜“H—N‘N 3°�NÓ3°AÕ6Ó7ˆð �„{ÙØ�„{ÙÙøô ”zÐ"ô Úðús   §,A' Á'A<Á;A<c               ó    € V ^8„  d   QhRRRR/# )r   Ú
error_textr>   r   ú	list[str]r!   )r"   s   "r#   r$   r$   ‡   s   € ÷ ñ ¨Sð °Yñ r%   c                óP   € \          Uu. uF  pV R 2V 9   g   K  VNK  	  up# u upi )z: No such file or directory)Ú_MARLIN_REQUIRED_CUDA_HEADERS)rp   Úheader_names   & r#   Ú_marlin_missing_header_namesru   ‡   s9   € å'DóÙ'D˜Øˆ]Ð5Ð6¸*ÑD÷ 	ˆÑ'Dñð ùò s   ‰#š#c               ó    € V ^8„  d   QhRRRR/# )r   rp   r>   r   r!   )r"   s   "r#   r$   r$   Ž   s   € ÷ ñ ¨Cð °Cñ r%   c                óà   € \        V 4      pV'       g   R # \        4       '       d   R # \        4       pRP                  V4      pVe   RV R2pMRp\	        R4      '       d   RMRpRV R	V R
V 2# )rV   z, zEInstall the wheel-provided CUDA headers with `pip install "gptqmodel[z]"`.zLInstall the CUDA runtime/developer headers that match your Torch CUDA build.Únvccz8A local `nvcc` on PATH is still required for Marlin JIT.z0Marlin JIT also requires a local `nvcc` on PATH.z/Missing CUDA developer headers for Marlin JIT (z). Ú )ru   r   rn   rM   r   )rp   Úmissing_headersÚ
extra_nameÚmissing_headers_textÚinstall_textÚ	nvcc_texts   &     r#   Ú_marlin_header_install_hintr   Ž   s—   € Ü2°:Ó>€OßÙä(×*Ò*Ùä(Ó*€JØŸ9™9 _Ó5ÐØÒð(Ø(2 |°5ð:ñ 	ð [ð 	ô ��=Š=ñ 	Cà?ð ð :Ð:NÐ9OÈsØˆ.˜˜)˜ð	&ðr%   c               ó   € V ^8„  d   QhRR/# r^   r!   )r"   s   "r#   r$   r$   ­   s   € ÷ ñ ¨$ñ r%   c                 ó|  € \        4       p V R ,          p\        P                  ! \        P                  \        V4      R.\        V 4      RRRR7      pVP                  ^ 8X  d   V # \        P                  ! \        P                  \        V4      .\        V 4      RRRR7      pVP                  ^ 8w  dŠ   VP                  ;'       g?    VP                  ;'       g+    VP                  ;'       g    VP                  ;'       g    RP                  4       p\        RV'       d   RV 2,           4      hR,           4      hV # )	zgenerate_kernels.pyz--checkTF)ÚcwdÚcapture_outputÚtextÚcheckrV   zMarlin kernel generation failedz: rA   )re   Ú
subprocessÚrunÚsysÚ
executabler>   Ú
returncodeÚstderrÚstdoutÚstripÚRuntimeError)ÚrootÚ	generatorÚcheck_resultÚresultÚdetailss        r#   Ú _ensure_generated_marlin_kernelsr”   ­   s  € Ü‹>€DØÐ,Õ,€IÜ—>’>Ü	�‰œ˜Y›¨Ð3Ü�‹IØØØô€Lð ×Ñ !Ô#Øˆä�^Š^Ü	�‰œ˜Y›Ð(Ü�‹IØØØô€Fð ×Ñ˜AÔØ—=‘=×eÐe F§M¡M×eÐe°\×5HÑ5H×eÐeÈL×L_ÑL_×eÐeÐce×lÑlÓnˆÜØ-ß!(��G�9ˆ~õ3ó
ð 	
à.1õ3ó
ð 	
ð €Kr%   c               ó    € V ^8„  d   QhRRRR/# )r   Ú	dtype_tagr>   r   rq   r!   )r"   s   "r#   r$   r$   Ê   s   € ÷ ñ ˜sð  yñ r%   c           
     ób  € \        4       p\        VR V  R2,          4      \        VRV  R2,          4      \        VR,          4      \        VR,          4      .pVP                  R \        VP	                  RV  R24      4       4       4       \        V4      ^8:  d   \        R	V  R
V R24      hV# )Úmarlin_torch_z.cppÚgptq_marlin_z.cuzgptq_marlin_repack.cuzawq_marlin_repack.cuc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5ir.   rC   )r2   Úpaths   & r#   r4   Ú"_marlin_sources.<locals>.<genexpr>Ò   s   é € ÐWÑ)V ”3�t—9�9Ó)VùrG   Úkernel_z_*.cuzMarlin z sources are incomplete under `r@   )r”   r>   ÚextendÚsortedÚglobÚlenrŽ   )r–   r�   Úsourcess   &  r#   Ú_marlin_sourcesr£   Ê   s±   € Ü+Ó-€DäˆD�] 9 +¨TÐ2Õ2Ó3ÜˆD�\ ) ¨CÐ0Õ0Ó1ÜˆDÐ*Õ*Ó+ÜˆDÐ)Õ)Ó*ð	€Gð ‡N�NÑW¬°·	±	¸GÀIÀ;ÈeÐ:TÓ0UÔ)VÓWÔWÜ
ˆ7ƒ|�qÔÜ˜W Y KÐ/NÈtÈfÐTVÐWÓXÐXØ€Nr%   c               ó   € V ^8„  d   QhRR/# ©r   r   rq   r!   )r"   s   "r#   r$   r$   Ø   s   € ÷ ñ ˜yñ r%   c                 óH   € \        \        \        4       4      .\        R 7      # ))Úrequired_header_names)r   r>   re   rs   r!   r%   r#   Ú_marlin_include_pathsr¨   Ø   s   € Ü+Ü	Œ\‹^Ó	ÐÜ;ôð r%   c               ó   € V ^8„  d   QhRR/# r¥   r!   )r"   s   "r#   r$   r$   ß   s   € ÷ 0ñ 0˜iñ 0r%   c                 ó   € \        R R7      # )T)Úenable_bf16)r   r!   r%   r#   Ú_marlin_extra_cflagsr¬   ß   s   € Ü¨$Ô/Ð/r%   c               ó   € V ^8„  d   QhRR/# r¥   r!   )r"   s   "r#   r$   r$   ã   s   € ÷ ñ  9ñ r%   c            
     ón   € \        R R R R RR R R7      p \        4       '       d   V P                  ^ R4       V # )TF)r«   Úinclude_lineinfoÚinclude_nvcc_threadsÚinclude_ptxas_optimizationsÚinclude_ptxas_verbosityÚinclude_fatbin_compressionÚinclude_diag_suppressz"-static-global-template-stub=false)r   r   Úinsert)Úflagss    r#   Ú_marlin_extra_cuda_cflagsr·   ã   s@   € Ü#ØØØ!Ø$(Ø %Ø#'Ø"ô€Eô ×ÒØ�‰�QÐ<Ô=Ø€Lr%   c                 ó   € \        R 4      # )Úfp16©r£   r!   r%   r#   Ú<lambda>r»   ÷   ó	   € ”O FÔ+r%   Ú GPTQMODEL_MARLIN_FP16_BUILD_ROOTc                 ó   € \        R 4      # )Úmarlin_fp16©r   r!   r%   r#   r»   r»   ù   ó
   € Ô;¸MÔJr%   zMarlin fp16ÚGPTQMODEL_MARLIN_FORCE_REBUILDÚGPTQMODEL_EXT_VERBOSETF)ÚnameÚ	namespaceÚrequired_opsr¢   Úbuild_root_envÚdefault_build_rootÚdisplay_nameÚextra_cflagsÚextra_cuda_cflagsÚextra_include_pathsÚforce_rebuild_envÚverbose_envÚrequires_cudaÚpython_abi_dependentc                 ó   € \        R 4      # )Úbf16rº   r!   r%   r#   r»   r»   	  r¼   r%   Ú GPTQMODEL_MARLIN_BF16_BUILD_ROOTc                 ó   € \        R 4      # )Úmarlin_bf16rÀ   r!   r%   r#   r»   r»     rÁ   r%   zMarlin bf16c                 ó   € ^ RI Hp  V # )r   )Ú	extension)Ú	gptqmodelr×   )Úextension_apis    r#   Ú_extension_apirÚ     s   € Ý4àÐr%   c               ó    € V ^8„  d   QhRRRR/# )r   ÚdtypeúOptional[torch.dtype]r   útorch.dtyper!   )r"   s   "r#   r$   r$     s   € ÷ Hñ HÐ!6ð H¸;ñ Hr%   c                ól   € V \         P                  8X  d   \         P                  # \         P                  # r.   )r/   Úbfloat16Úfloat16©rÜ   s   &r#   Ú_marlin_runtime_dtyperã     s    € Ø"¤e§n¡nÔ4Œ5�>‰>ÐG¼%¿-¹-ÐGr%   c               ó    € V ^8„  d   QhRRRR/# ©r   rÜ   rÝ   r   r>   r!   )r"   s   "r#   r$   r$   !  s   € ÷ ^ñ ^Ð)>ð ^À3ñ ^r%   c                óF   € \        V 4      \        P                  8X  d   R # R# )rÕ   r¿   )rã   r/   rà   râ   s   &r#   Ú_marlin_kernel_name_for_dtyperç   !  s   € Ü1°%Ó8¼E¿N¹NÔJˆ=Ð]ÐP]Ð]r%   c               ó   € V ^8„  d   QhRR/# )r   r   ÚNoner!   )r"   s   "r#   r$   r$   %  s   € ÷ 3ñ 3 dñ 3r%   c                 óV   € \         P                  4        \        P                  4        R # r.   )Ú _MARLIN_FP16_TORCH_OPS_EXTENSIONÚclear_cacheÚ _MARLIN_BF16_TORCH_OPS_EXTENSIONr!   r%   r#   Úclear_marlin_extension_cacherî   %  s   € Ü$×0Ñ0Ô2Ü$×0Ñ0Ö2r%   c               ó    € V ^8„  d   QhRRRR/# ©r   rÜ   rÝ   r   r    r!   )r"   s   "r#   r$   r$   *  s   € ÷ Oñ OÐ$9ð OÀTñ Or%   c                óZ   € \         e   R# \        4       P                  \        V 4      4      # )NF)Úmarlin_import_exceptionrÚ   rW   rç   râ   s   &r#   Úmarlin_runtime_availableró   *  s%   € ÜÒ*ÙÜÓ×(Ñ(Ô)FÀuÓ)MÓNÐNr%   c               ó    € V ^8„  d   QhRRRR/# rå   r!   )r"   s   "r#   r$   r$   0  s   € ÷ ñ Ð 5ð Àñ r%   c                óè   € \         e   \         # \        V 4      p\        4       pVP                  V4      '       d   R# VP	                  V4      ;'       g    Rp\        V4      pV'       d   V RV 2# V# )NrV   zMarlin runtime unavailable.ry   )rò   rç   rÚ   rW   Úerrorr   )rÜ   Úextension_namerÙ   rp   Úinstall_hints   &    r#   Úmarlin_runtime_errorrù   0  sq   € ÜÒ*Ü&Ð&ä2°5Ó9€NÜ"Ó$€MØ×!Ñ! .×1Ò1ÙØ×$Ñ$ ^Ó4×UÐUÐ8U€JÜ.¨zÓ:€LßØ�˜Q˜|˜nÐ-Ð-ØÐr%   c               ó    € V ^8„  d   QhRRRR/# rð   r!   )r"   s   "r#   r$   r$   ?  s   € ÷ Fñ FÐ$9ð F¸dñ Fr%   c                óZ   € \        V 4      p\        4       P                  VR 7      V,          # ))rÄ   )rç   rÚ   Úload)rÜ   r÷   s   & r#   Úprewarm_marlin_extensionrý   ?  s(   € Ü2°5Ó9€NÜÓ× Ñ  nÐ Ó5°nÕEÐEr%   c               ó    € V ^8„  d   QhRRRR/# )r   rÜ   rÝ   Úop_namer>   r!   )r"   s   "r#   r$   r$   D  s"   € ÷ Nñ Nà ðNð ñNr%   c                óH   € \        4       P                  \        V 4      V4      # r.   )rÚ   Úoprç   ©rÜ   rÿ   s   $$r#   Ú_marlin_resolve_opr  D  s    € ô
 Ó×ÑÔ<¸UÓCÀWÓMÐMr%   c               ó    € V ^8„  d   QhRRRR/# )r   r;   zOptional[torch.device]r   r    r!   )r"   s   "r#   r$   r$   M  s   € ÷ 6ñ 6Ð,Bð 6Èdñ 6r%   c                ó:  € \         '       g%   \        P                  P                  4       '       g   R# V e   \        P                  ! V 4      MRpVe    \        P                  P                  V4      M\        P                  P                  4       w  r#\        W#4      # )zË
Validates if the current device is compatible for Marlin.
ref: https://github.com/IST-DASLab/marlin?tab=readme-ov-file#requirements

Returns:
    bool: indicates if CUDA device is compatible for Marlin
FN)r   r/   r0   rW   r;   r1   r(   )r;   Útargetr   r   s   &   r#   Ú_validate_marlin_device_supportr  M  ss   € ÷ ƒw”e—j‘j×-Ñ-×/Ò/ÙØ%+Ò%7ŒU�\Š\˜&Ô!¸T€Fð Òô 	�
‰
×(Ñ(¨Ô0ä�Z‰Z×-Ñ-Ó/ñ €Eô
 (¨Ó5Ð5r%   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Ú	act_orderr    Úis_row_parallelr   r!   )r"   s   "r#   r$   r$   `  s&   € ÷ Bñ B ð B°tð BÀñ Br%   c                óB   € V '       * ;'       g    T ;'       d    V'       * # r.   r!   )r	  r
  s   &&r#   Úmarlin_is_k_fullr  `  s   € ØŒM×AÐA˜y×@Ð@°Ô-@ÐAr%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   r	  r    Ú
group_sizer   r
  r   r!   )r"   s   "r#   r$   r$   d  s*   € ÷ =ñ =°ð =À3ð =Ø7;ð=Ø@Dñ=r%   c                ó8   € VR8H  pT ;'       g    T;'       d    T# ©é   éÿÿÿÿr!   )r	  r  r
  Úis_channelwises   &&& r#   Ú!marlin_repeat_scales_on_all_ranksr  d  s$   € ð   2Ñ%€NØ×<Ð<˜×;Ð;¨OÐ<r%   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r;   r<   Úmax_blocks_per_smr   r   útorch.Tensorr!   )r"   s   "r#   r$   r$   l  s#   € ÷ ,ñ , lð ,Ø14ð,Ø=Iñ,r%   c                óÈ   € \         P                  P                  V 4      P                  p\	        W!,          ^€4      p\         P
                  ! V\         P                  V RR7      # )é€   F)rÜ   r;   Úrequires_grad)r/   r0   Úget_device_propertiesÚmulti_processor_countÚmaxÚzerosr   )r;   r  ÚsmsÚworkspace_blockss   &&  r#   Úmarlin_make_workspace_newr!  l  sO   € ô �*‰*×
*Ñ
*¨6Ó
2×
HÑ
H€CÜ˜3Õ2°CÓ8ÐÜ�;Š;Ð'Ü"ŸY™YØ$Ø%*ô,ð ,r%   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úvaluer   Úmultipler   r!   )r"   s   "r#   r$   r$   z  s!   € ÷ ;ñ ;�Sð ; Cð ;¨Cñ ;r%   c                ó<   € W,           ^,
          V,          V,          # )z$Round value up to the next multiple.r!   )r#  r$  s   &&r#   Ú	_round_upr&  z  s   € àÕ Õ! hÕ.°(Õ:Ð:r%   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úsize_nr   Úsize_kr   r    r!   )r"   s   "r#   r$   r$   €  s!   € ÷ ñ  3ð °ð ¸ñ r%   c                ó’   € V ^@,          ^ 8H  ;'       d    V^€,          ^ 8H  ;'       g!    V ^€,          ^ 8H  ;'       d    V^@,          ^ 8H  # )é@   r!   )r(  r)  s   &&r#   Úmarlin_is_tile_alignedr,  €  sM   € à���qÑ×.Ð.˜V c�\¨QÑ.÷ð ð 	��˜Ñ×.Ð.˜f r�k¨QÑ.ðr%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   r(  r   r)  r  r   r+   r!   )r"   s   "r#   r$   r$   ˆ  s*   € ÷ ñ ˜Sð ¨#ð Ø!$ðØ.=ñr%   c           	     ó:  € V^ 8”  d   TM^p\        V ^@4      \        V\        P                  ! ^€V4      4      3\        V ^€4      \        V\        P                  ! ^@V4      4      33p\        VR R7      pWPV38w  d    \        P
                  P                  R4       V# )z®Return the smallest N/K pair supported by a Marlin thread tile.

Padded K consumes zero activations; padded N uses zero scales, so neither
region changes the logical output.
c                ó^   € V ^ ,          V ^,          ,          V ^ ,          V ^,          ,           3# )r   r!   )Únks   &r#   r»   Ú"marlin_padded_nk.<locals>.<lambda>•  s!   € °°1µ¸¸1½µ¸rÀ!½uÀrÈ!Åu½}Ñ/Mr%   )ÚkeyzœMarlin is padding a tile-misaligned weight shape. Activations and outputs for this layer are padded and sliced on each forward; performance may be degraded.)r&  ÚmathÚlcmÚminÚlogÚwarnÚonce)r(  r)  r  ÚgroupÚ
candidatesÚ	padded_nks   &&&   r#   Úmarlin_padded_nkr<  ˆ  s“   € ð % qœ.‰J¨a€Eô 
�6˜2Ó	¤	¨&´$·(²(¸3ÀÓ2FÓ GÐHÜ	�6˜3Ó	¤¨6´4·8²8¸BÀÓ3FÓ!GÐHð€Jô �JÑ$MÔN€IØ˜VÐ$Ô$Ü�‰�‰ð+ô	
ð
 Ðr%   c               ó0   € V ^8„  d   QhRRRRRRRRRRRR/# )	r   Úqweightr  r(  r   r)  Úpadded_nÚpadded_kr   r!   )r"   s   "r#   r$   r$   Ÿ  s9   € ÷ 
ñ 
 ð 
°cð 
À3ð 
Ø!$ð
Ø03ð
Ø8Dñ
r%   c                óÈ   € W43W38X  d   V # W P                  ^ 4      ,          p\        P                  P                  P	                  V ^ W1,
          ^ WB,
          V,          34      # )z=Zero-pad a GPTQ-layout packed weight before Marlin repacking.)Úsizer/   ÚnnÚ
functionalÚpad)r>  r(  r)  r?  r@  Úpack_factors   &&&&& r#   Úmarlin_pad_qweightrG  Ÿ  s^   € ð 	Ð Ð/Ô/ØˆàŸL™L¨›OÕ+€KÜ�8‰8×Ñ×"Ñ"ØØ	
ˆHÕ˜q 8Õ#4¸Õ"DÐEóð r%   c               ó4   € V ^8„  d   QhRRRRRRRRRRRRR	R/# )
r   r>  r  r(  r   r)  r?  r@  Únum_bitsr   r!   )r"   s   "r#   r$   r$   ¬  sA   € ÷ ñ  Lð ¸#ð Àsð Ø%(ðØ47ðà%(ðà-9ñr%   c                ó>  € ^ V,          pW!V,          3p\        V P                  4      V8w  d&   \        RV R\        V P                  4       R24      hW43W38X  d   V # \        P                  P
                  P                  V ^ W1,
          V,          ^ WB,
          34      # )z=Zero-pad an AWQ-layout packed weight before Marlin repacking.zAWQ qweight shape must be rI   rA   ©ÚtupleÚshaperJ   r/   rC  rD  rE  )r>  r(  r)  r?  r@  rI  rF  Úexpected_shapes   &&&&&&  r#   Úmarlin_pad_awq_qweightrO  ¬  s¢   € ð ˜•.€KØ¨Õ3Ð4€NÜˆW�]‰]Ó˜~Ô-ÜØ(¨Ð(8¸¼uÀWÇ]Á]Ó?SÐ>TÐTUÐVó
ð 	
ð 	Ð Ð/Ô/ØˆÜ�8‰8×Ñ×"Ñ"ØØ	
ˆXÕ ;Õ.°°8Õ3DÐEóð r%   c               ó8   € V ^8„  d   QhRRRRRRRRRRRRR	RR
R/# )r   Úqzerosr  r(  r   r)  r?  r@  r  rI  r   r!   )r"   s   "r#   r$   r$   ¾  sI   € ÷ ñ  ,ð ¸ð ÀSð Ø$'ðØ36ðØDGðà$'ðà,8ñr%   c                ó‚  € ^ V,          pV^ 8”  d	   W%,          M^pV^ 8”  d	   WE,          M^p	W�V,          3p
\        V P                  4      V
8w  d&   \        RV
 R\        V P                  4       R24      hW43W38X  d   V # \        P                  P
                  P                  V ^ W1,
          V,          ^ W˜,
          34      # )zBZero-pad AWQ packed zero-points to the padded group and N extents.zAWQ qzeros shape must be rI   rA   rK  )rQ  r(  r)  r?  r@  r  rI  rF  ÚgroupsÚpadded_groupsrN  s   &&&&&&&    r#   Úmarlin_pad_awq_qzerosrU  ¾  sÂ   € ð ˜•.€KØ%/°!¤^ˆVÖ!¸€FØ.8¸1¬n�HÖ*À!€MØ¨Õ3Ð4€NÜˆV�\‰\Ó˜nÔ,ÜØ'¨Ð'7°v¼eÀFÇLÁLÓ>QÐ=RÐRSÐTó
ð 	
ð 	Ð Ð/Ô/ØˆÜ�8‰8×Ñ×"Ñ"ØØ	
ˆXÕ ;Õ.°°=Õ3IÐJóð r%   c               ó4   € V ^8„  d   QhRRRRRRRRRRRRR	R/# )
r   Úscalesr  r(  r   r)  r?  r@  r  r   r!   )r"   s   "r#   r$   r$   Ò  sA   € ÷ ñ ˜lð °Cð Àð Ø #ðØ/2ðà"%ðà*6ñr%   c                óò   € W43W38X  d   V # V^ 8”  d   WE,          V P                  ^ 4      ,
          M^ pV^ 8  d   \        R4      h\        P                  P                  P                  V ^ W1,
          ^ V34      # )z;Zero-pad scale rows and columns to the padded Marlin shape.z2Padded Marlin K cannot contain fewer scale groups.)rB  rJ   r/   rC  rD  rE  )rW  r(  r)  r?  r@  r  Úpad_rowss   &&&&&& r#   Úmarlin_pad_scalesrZ  Ò  su   € ð 	Ð Ð/Ô/Øˆà:DÀq¼.ˆxÕ%¨¯©°A«Ö6Èa€HØ�!„|ÜÐMÓNÐNÜ�8‰8×Ñ×"Ñ"Ø��HÕ% q¨(Ð3óð r%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Úxr  rB  r   Úpaddedr   r!   )r"   s   "r#   r$   r$   á  s(   € ÷ :ñ :�lð :¨#ð :°sð :¸|ñ :r%   c                óv   € W!8X  d   V # \         P                  P                  P                  V ^ W!,
          34      # )zBZero-pad the last tensor dimension when a Marlin tile requires it.)r/   rC  rD  rE  )r\  rB  r]  s   &&&r#   Úmarlin_pad_dimr_  á  s0   € à„~ØˆÜ�8‰8×Ñ×"Ñ" 1 q¨&­-Ð&8Ó9Ð9r%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Úoutputr  r(  r   r?  r   r!   )r"   s   "r#   r$   r$   è  s*   € ÷ -ñ - ð -°cð -Ø"%ð-Ø*6ñ-r%   c                óH   € W!8X  d   V # V RRV13,          P                  4       # )z>Slice a padded Marlin result back to its logical output width..N)Ú
contiguous)ra  r(  r?  s   &&&r#   Úmarlin_unpad_outputrd  è  s+   € ð ÔØˆØ�#�w˜�w�,Õ×*Ñ*Ó,Ð,r%   c               ó    € V ^8„  d   QhRRRR/# )r   Údstr  Úsrcr!   )r"   s   "r#   r$   r$   ð  s   € ÷ 
ñ 
˜|ð 
°,ñ 
r%   c                ó|  € V P                   VP                   8X  g   Q R 4       h\        P                  ! 4       ;_uu_ 4        V P                  VP                  VP                  4       4       V P                  4       VP                  4       8w  d   V P                  V4       ?RRR4       R#   + '       g   i     R# ; i)z Tensors must have the same dtypeN)rÜ   r/   Úno_gradÚas_strided_rM  ÚstrideÚdata_ptrÚcopy_)rf  rg  s   &&r#   Úupdate_tensor_inplacern  ð  sq   € Ø�9‰9˜Ÿ	™	Ô!ÐEÐ#EÓEÐ!ä	�Š��à�‰˜Ÿ	™	 3§:¡:£<Ô0ð �<‰<‹>˜SŸ\™\›^Ô+Ø�I‰I�cŒNØ÷ 
��Šús   Á A B*Â*B;	c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   Úmodútorch.nn.ModulerÄ   r>   Únewz'Union[torch.Tensor, torch.nn.Parameter]r   ré   r!   )r"   s   "r#   r$   r$   ÿ  s0   € ÷ Mñ M˜?ð M°#ð MØBðMØGKñMr%   c                ó  € \        W4      p\        V4      \        V4      J dh   VP                  VP                  8X  dM   VP                  4       P	                  4       VP                  4       P	                  4       8X  d   \        W24       R# \        V\        P                  P                  4      '       g"   \        P                  P                  VR R7      pV P                  V\        P                  P                  VR R7      4       R# )F©r  N)ri   rK   rÜ   Úuntyped_storageÚnbytesrn  Ú
isinstancer/   rC  Ú	ParameterÚregister_parameter)rp  rÄ   rr  Úolds   &&& r#   Úreplace_parameterr{  ÿ  s»   € ä
�#Ó
€CÜˆCƒy”D˜“IÓ #§)¡)¨s¯y©yÔ"8Ø×ÑÓ!×(Ñ(Ó*¨c×.AÑ.AÓ.C×.JÑ.JÓ.LÔLô 	˜cÖ'ô ˜#œuŸx™x×1Ñ1×2Ò2Ü—(‘(×$Ñ$ S¸Ð$Ó>ˆCØ×Ñ˜tÜ$Ÿx™x×1Ñ1°#ÀUÐ1ÓKö	Mr%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   Úlayerrq  rÄ   zOptional[str]Úfnr   r   ré   r!   )r"   s   "r#   r$   r$     s0   € ÷ 	Eñ 	E˜Oð 	E°=ð 	EØ!ð	EØ&*ñ	Er%   c           	     óÂ   € Ve[   \        WR 4      eK   \        W4      pV! V4      p\        W\        P                  P	                  VP
                  RR7      4       R # R # R # )NFrt  )ri   r{  r/   rC  rx  Údata)r}  rÄ   r~  Ú	old_paramÚ	new_params   &&&  r#   Ú_transform_paramrƒ    sY   € àÒœG E°Ó6ÒBÜ˜EÓ(ˆ	Ù�y“Mˆ	ô 	ØÜ�H‰H×Ñ˜yŸ~™~¸UÐÓCö	Eñ CÑr%   c               ó    € V ^8„  d   QhRRRR/# )r   Úg_idxr  r   ú!Tuple[torch.Tensor, torch.Tensor]r!   )r"   s   "r#   r$   r$     s   € ÷ 9ñ 9Øð9Ø Añ9r%   c                ó|   € \         P                  ! V 4      P                  \         P                  4      pW,          V3# r.   )r/   ÚargsortÚtor   )r…  Úg_idx_sort_indicess   & r#   Úmarlin_sort_g_idxr‹    s/   € äŸš uÓ-×0Ñ0´·±Ó;ÐØÕ$Ð&8Ð8Ð8r%   c               ó    € V ^8„  d   QhRRRR/# )r   r;   r<   r   r  r!   )r"   s   "r#   r$   r$   $  s   € ÷ 3ñ 3 Lð 3°\ñ 3r%   c                óŽ   € \         P                  P                  \         P                  ! ^ \         P                  V R7      RR7      # )r   )rÜ   r;   Frt  )r/   rC  rx  Úemptyr   )r;   s   &r#   Úmarlin_make_empty_g_idxr�  $  s4   € Ü�8‰8×ÑœeŸkšk¨!´5·9±9ÀVÔLØ,1ð ó 3ð 3r%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   r}  rq  rÄ   r>   Únew_tr  r   ré   r!   )r"   s   "r#   r$   r$   +  s*   € ÷ ñ ˜/ð °ð Ø&ðØ+/ñr%   c                ó„   € \        W4      P                  VP                  4       \        W4      P                  V4       ?R # r.   )ri   Úresize_rM  rm  )r}  rÄ   r‘  s   &&&r#   Úreplace_tensorr”  +  s1   € ô ˆEÓ× Ñ  §¡Ô-ÜˆEÓ×Ñ˜uÔ%Úr%   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   Úsr  r)  r   r(  r  r   r!   )r"   s   "r#   r$   r$   4  s1   € ÷ 	ñ 	˜\ð 	°3ð 	Àð 	Ø&)ð	Ø.:ñ	r%   c                óB  € \        V P                  4      w  rEW18  d5   VR8w  d.   V P                  R\        V4      34      P	                  ^V4      p M,V P                  R\        V4      34      P	                  ^V4      p V P                  RV34      P                  4       p V # r  )Ú_get_scale_perm_tensorsr;   Úreshaper¡   Úindex_selectrc  )r–  r)  r(  r  Ú
scale_permÚscale_perm_singles   &&&&  r#   Úmarlin_permute_scalesr�  4  s‹   € ä$;¸A¿H¹HÓ$EÑ!€JØÔ˜z¨RÔ/Ø�I‰I�rœ3˜z›?Ð+Ó,×9Ñ9¸!¸ZÓH‰à�I‰I�rœ3Ð0Ó1Ð2Ó3×@Ñ@ÀÐDUÓVˆØ	�	‰	�2�v�,Ó×*Ñ*Ó,€Aà€Hr%   c            	     ó8  € . p \        ^4       F:  pT P                  \        ^4       Uu. uF  q!^V,          ,           NK  	  up4       K<  	  . p\        ^4       F2  pTP                  R Uu. uF  p^V,          V,           NK  	  up4       K4  	  W3# u upi u upi )é   )r   r  rŸ  é	   é   é   é   é   )r8   rž   )r›  ÚiÚjrœ  s       r#   Úget_scale_permsr§  @  s“   € Ø€JÜ�1ŽXˆØ×Ñ¬e°A¬hÓ7©h¨˜q 1�uŸ9˜9©hÑ7Ö8ñ à#%ÐÜ�1ŽXˆØ× Ñ Ù <Ó=Ñ <˜1ˆQ��U�Q�YˆYÑ <Ñ=ö	?ñ ð Ð(Ð(ùò 8ùò >s   ©B
Á+B
z5Dict[torch.device, Tuple[torch.Tensor, torch.Tensor]]Ú_SCALE_PERM_TENSOR_CACHEc               ó    € V ^8„  d   QhRRRR/# )r   r;   r<   r   r†  r!   )r"   s   "r#   r$   r$   Q  s   € ÷ ñ  Lð Ð5Vñ r%   c                óŒ  € \         P                  V 4      pVe   V# \        ;_uu_ 4        \         P                  V 4      pVfe   \        4       w  r#\        P
                  ! W \        P                  R7      p\        P
                  ! W0\        P                  R7      pWE3pV\         V &   RRR4       V#   + '       g   i     T# ; i)zAReturn (scale_perm, scale_perm_single) index tensors on `device`.N)r;   rÜ   )r¨  ÚgetÚ_SCALE_PERM_TENSOR_LOCKr§  r/   ÚtensorÚlong)r;   Úcacher›  rœ  Úperm_tÚperm_single_ts   &     r#   r˜  r˜  Q  sž   € ä$×(Ñ(¨Ó0€EØÒØˆç	 Ö	 Ü(×,Ñ,¨VÓ4ˆØŠ=Ü,;Ó,=Ñ)ˆJÜ—\’\ *Ä5Ç:Á:ÔNˆFÜ!ŸLšLÐ):ÔQV×Q[ÑQ[Ô\ˆMØÐ+ˆEØ/4Ô$ VÑ,÷ 
!ð €L÷ 
!Ö	 ð €Lús   ªA>B2Â2C	c                 óR   € \         P                  P                  4       '       d   R # R # r.   )r/   ÚcompilerÚis_dynamo_compilingr!   r%   r#   Ú maybe_warn_marlin_atomic_add_envrµ  a  s   € Ü‡~�~×)Ñ)×+Ò+Ùñ ,r%   c                ó  € \         P                  P                  4       '       d   R # \         P                  P	                  V 4      pV^ ,          ^	8  d/   V\         P
                  8X  d   \        P                  R4       R # R # R # )Nz‡You are running Marlin kernel with bf16 on GPUs before SM90. You can consider change to fp16 to achieve better performance if possible.)r/   r³  r´  r0   r1   rà   r6  Ú	info_once)r;   rÜ   Údevice_capabilitys   && r#   Úmaybe_warn_marlin_atomic_addr¹  j  s^   € Ü‡~�~×)Ñ)×+Ò+ÙÜŸ
™
×8Ñ8¸Ó@ÐØ˜Õ˜aÔ E¬U¯^©^Ô$;Ü�‰ðö	ñ %<Ñr%   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   Úmr   ÚnÚkr;   r<   rÜ   rÞ   r   r    r!   )r"   s   "r#   r$   r$   u  s8   € ÷ ñ  Cð ¨Cð °Cð Àð Ø(3ðØ8<ñr%   c                óæ   € VR 8¼  g   VR 8  g   VP                   R8w  d   R# \        P                  P                  V4      pV^ ,          ^	8  d#   V\        P                  8X  d   \        W44       R# R# )i   r0   FT)rK   r/   r0   r1   rà   r¹  )r»  r¼  r½  r;   rÜ   r¸  s   &&&&& r#   Úshould_use_atomic_add_reducer¿  u  s\   € ð 	ˆD„y�A˜”H §¡¨vÔ 5Ùô Ÿ
™
×8Ñ8¸Ó@ÐØ˜Õ˜aÔ E¬U¯^©^Ô$;Ü$ VÔ3Ùár%   c               óT   € V ^8„  d   QhRRRRRRRRRRRRRRR	R
RRRRRRRRRRRRRR/# )r   Úinputr  ÚweightÚweight_scaleÚ	weight_zpr…  rŠ  Ú	workspaceÚwtyper   Úoutput_size_per_partitionr   Úinput_size_per_partitionÚ	is_k_fullr    ÚbiasúOptional[torch.Tensor]Úuse_fp32_reduceÚuse_atomicsr   r!   )r"   s   "r#   r$   r$   …  sž   € ÷ -%ñ -%Øð-%àð-%ð #ð-%ð  ð	-%ð
 ð-%ð )ð-%ð  ð-%ð ð-%ð $'ð-%ð #&ð-%ð ð-%ð %ð-%ð ð-%ð ð-%ð  ñ!-%r%   c                óŒ  € V P                  RV P                  R,          4      pV P                  RR V3,           pT;'       dC    \        VP                  ^ 4      VVP                  ^4      V P                  V P
                  R7      p\        VRVVVRVVVVVVP                  ^ ,          VV	V
VVRR7      pVP                  V4      # )r  N©r»  r¼  r½  r;   rÜ   F)Úsize_mr(  r)  rÉ  Úuse_atomic_addrÌ  Úis_zp_floatr  ©r™  rM  r¿  rB  r;   rÜ   Úgptq_marlin_gemm)rÁ  rÂ  rÃ  rÄ  r…  rŠ  rÅ  rÆ  rÇ  rÈ  rÉ  rÊ  rÌ  rÍ  Ú
reshaped_xÚ	out_shapera  s   &&&&&&&&&&&&&&   r#   Úapply_gptq_marlin_linearr×  …  sÕ   € ð" —‘˜r 5§;¡;¨r¥?Ó3€JØ—‘˜C˜RÐ Ð$=Ð#?Õ?€Ià÷ Eð EÔ">ÀÇÁÐQRÓASØ4MØ4>·O±OÀAÓ4FØ9>¿¹Ø8=¿¹ô	#E€Kô ˜jØ"Ø$Ø"Ø*Ø"Ø'Ø#Ø0Ø'Ø#Ø%/×%5Ñ%5°aÕ%8Ø%>Ø%=Ø(1Ø-8Ø.=Ø*/ô#1€Fð& �>‰>˜)Ó$Ð$r%   rÊ  rÌ  rÍ  c                óX   € V ^8„  d   QhRRRRRRRRRRRRR	RR
RRRRRRRRRRRRRRRRR/# )r   Útile_paddingr+   rÁ  r  rÂ  rÃ  rÄ  r…  rŠ  rÅ  rÆ  r   rÇ  r   rÈ  rÉ  r    rÊ  rË  rÌ  rÍ  r   r!   )r"   s   "r#   r$   r$   µ  s¼   € ÷ %Lñ %Là%ð%Lð ð%Lð ð	%Lð
 #ð%Lð  ð%Lð ð%Lð )ð%Lð  ð%Lð ð%Lð $'ð%Lð #&ð%Lð ð%Lð %ð%Lð ð%Lð  ð!%Lð" ñ#%Lr%   c                ón   € V w  pp\        WV4      p\        VVVVVVVVVVVVVVR7      p\        VWŸ4      # )zBPad one tile-misaligned GEMM around the unchanged Marlin hot path.)rÁ  rÂ  rÃ  rÄ  r…  rŠ  rÅ  rÆ  rÇ  rÈ  rÉ  rÊ  rÌ  rÍ  )r_  r×  rd  )rÙ  rÁ  rÂ  rÃ  rÄ  r…  rŠ  rÅ  rÆ  rÇ  rÈ  rÉ  rÊ  rÌ  rÍ  r?  r@  Úpadded_inputra  s   $$$$$$$$$$$$$$$    r#   Úapply_gptq_marlin_linear_paddedrÜ  µ  s^   € ð& &Ñ€HˆhÜ! %À8ÓL€LÜ%ØØØ!ØØØ-ØØØ"*Ø!)ØØØ'Øô€Fô  ˜vÐ'@ÓKÐKr%   c               óL   € V ^8„  d   QhRRRRRRRRRRRRRRR	R
RRRRRRRRRR/# )r   rÁ  r  rÂ  rÃ  rÄ  r…  rŠ  rÅ  Ú
quant_typer   rÇ  r   rÈ  rÊ  rË  rÌ  r    r   r!   )r"   s   "r#   r$   r$   Ý  sŠ   € ÷ (%ñ (%Øð(%àð(%ð #ð(%ð  ð	(%ð
 ð(%ð )ð(%ð  ð(%ð ð(%ð $'ð(%ð #&ð(%ð %ð(%ð ð(%ð *6ñ(%r%   c                óv  € V P                  RV P                  R,          4      pV P                  RR V3,           p\        VP                  ^ 4      VVP                  ^4      V P                  V P
                  R7      p\        VRVV
VRVVVVVVP                  ^ ,          VV	VVRR7      pVP                  V4      # )r  NrÏ  F)rÐ  r(  r)  rÑ  rÌ  rÒ  r  rÓ  )rÁ  rÂ  rÃ  rÄ  r…  rŠ  rÅ  rÞ  rÇ  rÈ  rÊ  rÌ  rÕ  rÖ  rÑ  ra  s   &&&&&&&&&&&&    r#   Úapply_awq_marlin_linearrà  Ý  sÂ   € ð —‘˜r 5§;¡;¨r¥?Ó3€JØ—‘˜C˜RÐ Ð$=Ð#?Õ?€Iä1°J·O±OÀAÓ4FØ4MØ4>·O±OÀAÓ4FØ9>¿¹Ø8=¿¹ô	E€Nô ˜jØ"Ø$Ø"Ø*Ø"Ø'Ø#Ø0Ø'Ø(Ø%/×%5Ñ%5°aÕ%8Ø%>Ø%=Ø-;Ø.=Ø*/ô!1€Fð$ �>‰>˜)Ó$Ð$r%   c               óP   € V ^8„  d   QhRRRRRRRRRRRRR	RR
RRRRRRRRRRRRR/# )r   rÙ  r+   rÁ  r  rÂ  rÃ  rÄ  r…  rŠ  rÅ  rÞ  r   rÇ  r   rÈ  rÊ  rË  rÌ  r    r   r!   )r"   s   "r#   r$   r$     s¦   € ÷ !Lñ !Là%ð!Lð ð!Lð ð	!Lð
 #ð!Lð  ð!Lð ð!Lð )ð!Lð  ð!Lð ð!Lð $'ð!Lð #&ð!Lð %ð!Lð ð!Lð ñ!Lr%   c                óh   € V w  rÞ\        WV4      p\        VVVVVVVVVVVVR7      p\        VW�4      # )z7Pad one AWQ GEMM around the unchanged Marlin call path.)rÁ  rÂ  rÃ  rÄ  r…  rŠ  rÅ  rÞ  rÇ  rÈ  rÊ  rÌ  )r_  rà  rd  )rÙ  rÁ  rÂ  rÃ  rÄ  r…  rŠ  rÅ  rÞ  rÇ  rÈ  rÊ  rÌ  r?  r@  rÛ  ra  s   $$$$$$$$$$$$$    r#   Úapply_awq_marlin_linear_paddedrã    sV   € ð" &Ñ€HÜ! %À8ÓL€LÜ$ØØØ!ØØØ-ØØØ"*Ø!)ØØ'ô€Fô ˜vÐ'@ÓKÐKr%   c          &     ód   € V ^8„  d   QhRRRRRRRRRRRRR	RR
RRRRRRRRRRRRRRRRRRRRRRR/# )r   Úar  ÚcrË  Ú
b_q_weightÚb_biasÚb_scalesÚglobal_scaleÚb_zerosr…  ÚpermrÅ  Úb_q_typer   rÐ  r   r(  r)  rÉ  r    rÑ  rÌ  rÒ  r   r!   )r"   s   "r#   r$   r$   ,  sÄ   € ÷ .ñ .˜ð .Ø.ð.à!-ð.ð 4ð.ð  ,ð	.ð
 $:ð.ð 5ð.ð 3ð.ð 2ð.ð !-ð.ð  *ð.ð !ð.ð !ð.ð !ð.ð !%ð.ð &*ð.ð  '+ð!.ð" #'ð#.ð" 4@ñ#.r%   c                óÐ   € \        V P                  4      \        P                  8X  d   R pMRp\	        V P                  VR7      pV! V VVVVVVVVV	V
P
                  VVVVVVV4      # )Úgptq_marlin_gemm_bf16Úgptq_marlin_gemm_fp16r  )rã   rÜ   r/   rà   r  Úid)rå  ræ  rç  rè  ré  rê  rë  r…  rì  rÅ  rí  rÐ  r(  r)  rÉ  rÑ  rÌ  rÒ  rÿ   r  s   &&&&&&&&&&&&&&&&&&  r#   rÔ  rÔ  ,  s€   € ô$ ˜QŸW™WÓ%¬¯©Ô7Ø)‰à)ˆä	Ø�g‰gØô
€Bñ Ø	Ø	ØØØØØØØØØ�‰ØØØØØØØó%ð r%   c               ó4   € V ^8„  d   QhRRRRRRRRRRRR	R
R/# )r   rç  r  rì  r)  r   r(  rI  rÜ   rÝ   r   r!   )r"   s   "r#   r$   r$   ^  sH   € ÷ :ñ : <ð :°|ð :Ø"ð:Ø,/ð:à!$ð:ð 4ð:ð @Lñ:r%   c                ó0   € \        VR R7      pV! WW#V4      # )Úgptq_marlin_repackr  ©r  )rç  rì  r)  r(  rI  rÜ   r  s   &&&&&& r#   rô  rô  ^  s%   € ô 
ØØ$ô
€Bñ ˆj °Ó9Ð9r%   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# )
r   rç  r  r)  r   r(  rI  rÜ   rÝ   r   r!   )r"   s   "r#   r$   r$   i  sB   € ÷ 	4ñ 	4 ,ð 	4Ø!ð	4à!ð	4ð !$ð	4ð 3ð		4ð ?Kñ		4r%   c                ó.   € \        VR R7      pV! WW#4      # )Úawq_marlin_repackr  rõ  )rç  r)  r(  rI  rÜ   r  s   &&&&& r#   rø  rø  i  s#   € ô
 
ØØ#ô
€Bñ ˆj &Ó3Ð3r%   c                óD   € ^ V ,          ^ 8X  g   Q RV  24       h^ V ,          # )é    zUnsupported num_bits = r!   )rI  s   &r#   Úget_pack_factorrû  u  s+   € Ø��=˜AÔÐCÐ!8¸¸
ÐCÓCÐØ��>Ðr%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Úq_wr  rI  r   r)  r(  r!   )r"   s   "r#   r$   r$   z  s0   € ÷ ñ Øðàðð ðð ñ	r%   c                óV  € V P                   W#38X  g   Q h\        V4      pW4,          ^ 8X  g   Q hV P                  pV P                  4       P	                  4       P                  \        P                  4      p \        P                  ! W#V,          3\        P                  R7      p\        V4       F$  pW`RVRV13,          W,          ,          ,          pK&  	  \        P                  ! VP                  \        P                  4      4      P                  V4      pVP                  4       pV# )r   râ   ºNNNN)rM  rû  r;   ÚcpuÚnumpyÚastypeÚuint32r  r8   r/   Ú
from_numpyÚint32r‰  rc  )rý  rI  r)  r(  rF  Úorig_deviceÚq_resr¥  s   &&&&    r#   Ú	pack_colsr  z  så   € ð �9‰9˜Ð(Ô(Ð(Ð(ä! (Ó+€KØÕ 1Ô$Ð$Ð$à—*‘*€Kà
�'‰'‹)�/‰/Ó
×
"Ñ
"¤5§<¡<Ó
0€Cä�KŠK˜¨;Õ!6Ð7¼u¿|¹|ÔL€Eä�;ÖˆØ�Q˜˜˜;˜Ð&Õ'¨8­<Õ7Õ7Šñ  ô ×Ò˜UŸ\™\¬%¯+©+Ó6Ó7×:Ñ:¸;ÓG€EØ×ÑÓ€Eà€Lr%   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Ú
packed_q_wr  rI  r   r)  r(  r!   )r"   s   "r#   r$   r$   ”  s0   € ÷ ñ Ø ðàðð ðð ñ	r%   c                ó¦  € \        V4      pW4,          ^ 8X  g   Q hV P                  W#V,          38X  g#   Q RP                  V P                  W#V4      4       hV P                  pV P	                  4       P                  4       P                  \
        P                  4      p\
        P                  ! W#3\
        P                  R7      p^V,          ^,
          p\        V4       F  p	Wh,          p
Wa,          pW§RV	RV13&   K  	  \        P                  ! VP                  \
        P                  4      4      P                  V4      pVP                  4       pV# )r   z?packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}râ   rÿ  N)rû  rM  r"   r;   r   r  r  r  r  r8   r/   r  r  r‰  rc  )r
  rI  r)  r(  rF  r  Úpacked_q_w_cpur  Úmaskr¥  Úvalss   &&&&       r#   Úunpack_colsr  ”  s)  € ô " (Ó+€KØÕ 1Ô$Ð$Ð$Ø×ÑØ˜+Õ%ð ô ð 7àH×OÑOØ×Ñ˜&¨+ó7ó7ð ð
 ×#Ñ#€Kà—^‘^Ó%×+Ñ+Ó-×4Ñ4´U·\±\ÓB€NÜ�KŠK˜Ð(´·±Ô=€Eà��M˜QÕ€DÜ�;ÖˆØÕ$ˆØÕ#ˆØ#'ˆa���K�ÐÓ ñ  ô
 ×Ò˜UŸ\™\¬%¯+©+Ó6Ó7×:Ñ:¸;ÓG€EØ×ÑÓ€Eà€Lr%   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   Úzpr  r)  r   r(  rI  r   r!   )r"   s   "r#   r$   r$   ²  s1   € ÷ ñ ˜<ð °ð ¸cð Ø!$ðØ)5ñr%   c                óÜ  € \        4       w  rEV P                  R\        V4      34      RV3,          p V^8X  d   \        P                  ! . RO4      pM:V^8X  d   \        P                  ! . RO4      pM\        RP                  V4      4      hV P                  R\        V4      34      RV3,          P                  4       p V P                  RV34      P                  4       p \        WW4      p V # )r  rÿ  únum_bits must be 4 or 8, got {}r  ©r   r   é   é   r  é   é   r'   ©r   r   r  r  )
r§  r™  r¡   r  ÚarrayrL   r"   Úravelrc  r  )r  r)  r(  rI  r›  Ú_Ú
interleaves   &&&&   r#   Úmarlin_zero_pointsr  ²  sÇ   € ô $Ó%�M€JØ	�‰�Rœ˜Z›Ð)Ó	*¨1¨j¨=Õ	9€Bð �1„}Ü—[’[Ò!9Ó:‰
Ø	�QŒÜ—[’[¢Ó.‰
äÐ9×@Ñ@ÀÓJÓKÐKà	�‰�Rœ˜Z›Ð)Ó	*¨1¨j¨=Õ	9×	?Ñ	?Ó	A€BØ	�‰�R˜�LÓ	!×	,Ñ	,Ó	.€BÜ	�2 Ó	0€Bà€Ir%   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   Úq_zp_packedr  r)  r   r(  rI  r   r!   )r"   s   "r#   r$   r$   È  s2   € ÷ ñ ¨<ð Àð Ø&)ðØ58ðØ=Iñr%   c                óâ  € \        WW4      pV^8X  d.   \        P                  ! \        P                  ! . RO4      4      pMNV^8X  d.   \        P                  ! \        P                  ! . RO4      4      pM\	        RP                  V4      4      hVP                  R\        V4      34      RV3,          P                  4       pVP                  RV34      P                  4       p\        WAW#4      pV# )r  r  rÿ  r  r  r  )r  r  rˆ  r  rL   r"   r™  r¡   r  rc  r  )r   r)  r(  rI  Úq_zpÚundo_interleaveÚ	marlin_zps   &&&&   r#   Úawq_to_marlin_zero_pointsr%  È  s¾   € ô �{¨fÓ=€Dð �1„}ÜŸ-š-¬¯ªÒ4LÓ(MÓN‰Ø	�QŒÜŸ-š-¬¯ª²LÓ(AÓB‰äÐ9×@Ñ@ÀÓJÓKÐKà�<‰<˜œS Ó1Ð2Ó3°A°Ð4FÕG×MÑMÓO€DØ�<‰<˜˜V˜Ó%×0Ñ0Ó2€Dä" 4°ÓB€IØÐr%   c               ó    € V ^8„  d   QhRRRR/# )r   r–  r  r   r!   )r"   s   "r#   r$   r$   ß  s   € ÷ 1ñ 1˜<ð 1¨Lñ 1r%   c                óÜ   € V P                   p\        V P                  4      w  r#V P                  R\	        V4      34      P                  ^V4      p V P                  ! V!  P                  4       # r  )rM  r˜  r;   r™  r¡   rš  rc  )r–  Úorigin_shaper  rœ  s   &   r#   Úmarlin_permute_biasr)  ß  s[   € Ø—7‘7€LÜ2°1·8±8Ó<Ñ€AØ	�	‰	�2”sÐ,Ó-Ð.Ó/×<Ñ<¸QÐ@QÓR€AØ�9Š9�lÑ#×.Ñ.Ó0Ð0r%   )zcuda_runtime_api.hz
cusparse.hzcublas_v2.hz
cublasLt.hzcusolverDn.h)rð  rô  rø  )rï  rô  rø  r.   )r  )r  )NTF)NT)TFFF)kÚ__conditional_annotations__Ú
__future__r   r3  r†   rˆ   Ú	threadingÚpathlibr   Úshutilr   Útypingr   r   r   r	   r
   r   r  r/   Úutils.loggerr   Úcppr   r   r   r   r   r   r   Úmarlin_scalar_typer   Úrocmr   r6  Ú_MARLIN_FP16_OPS_NAMEÚ_MARLIN_FP16_NAMESPACEÚ_MARLIN_BF16_OPS_NAMEÚ_MARLIN_BF16_NAMESPACErs   r(   r9   rR   rX   r\   rò   re   rn   ru   r   r”   r£   r¨   r¬   r·   rë   rí   rÚ   rã   rç   rî   ró   rù   rý   r  r  r  r  r!  r&  r,  r<  rG  rO  rU  rZ  r_  rd  rn  r{  rƒ  r‹  r�  r”  r�  r§  r¨  Ú__annotations__ÚLockr¬  r˜  rµ  r¹  r¿  r×  rÜ  rà  rã  rÔ  rô  rø  rû  r  r  r  r%  r)  )r*  s   @r#   Ú<module>r:     s€  øð÷
 #Ð "ã Û Û 
Û Ý Ý ß ?× ?ã Û å '÷÷ ñ õ +Ý ñ ƒn€à3Ð Ø0Ð Ø3Ð Ø0Ð ð!Ð õ4õõõ<õ
ñ :Ó;×CÐC¸tÐ õLõõõõ>õ:õõ0õñ  $8Ø	Ø$ØUÙ+Ø5ÙJØØ%Ø/Ø-Ø6Ø'ØØô$Ð  ñ$ $8Ø	Ø$ØUÙ+Ø5ÙJØØ%Ø/Ø-Ø6Ø'ØØô$Ð  ò$õHõ^õ3÷
O÷õFõ
N÷6õ&Bõ=÷,õ;õ÷õ.
õõ$õ(õ:õ-õ
õMõ&	Eõ9õ3õõ	ò)ð SUÐ ÐOÓ TØ#Ÿ.š.Ó*Ð õò òõ÷ -%ð`%Lð (,ð%Lð !%ð%Lð  "÷!%L÷P(%ðV!Lð (,ð!Lð !%÷!L÷H.÷d:÷	4òõ
õ4õ<õ,÷.1r%   