+ "ݳj8t^RIt^RIHt^RIt^RIHt]P P RR4t]PR4]PR4]PR4]PR 4]PR ]PPP.R 7]PR 4]PR 4]PR4]PR4]PR4]PR4Rt Rt ]PR4tRJRRllt] !R4RJRRll4tRKRRllt] !R4RKRRll4tRR lt] !R!4R"R#l4tR$R%lt] !R&4R'R(l4tR)R*lt] !R+4R,R-l4t]P!4R.4tR/t] !R04R1R2l4tR3R4lt] !R54R6R7l4tR8R9lt] !R:4R;R<l4t] !R=4R>R?l4tR@RAlt] !RB4RCRDl4tRERFlt] !RG4RHRIl4tR#)LN)OptionalTensortorchaoFRAGMENTzrowwise_scaled_linear_sparse_cutlass_f8f8(Tensor input, Tensor input_scale, Tensor weight, Tensor weight_meta, Tensor weight_scale, Tensor? bias=None, ScalarType? out_dtype=None) -> TensorzLto_sparse_semi_structured_cutlass_sm9x_f8(Tensor weight) -> (Tensor, Tensor)z\swizzle_mm(Tensor mat1, Tensor mat2, bool mat1_is_swizzled, bool mat2_is_swizzled) -> Tensorzswizzle_scaled_mm(Tensor mat1, Tensor mat2, bool mat1_is_swizzled, bool mat2_is_swizzled, Tensor scale_a, Tensor scale_b, Tensor? bias=None, Tensor? scale_result=None, ScalarType? out_dtype=None) -> TensorzImx_fp8_bf16(Tensor a, Tensor b, Tensor a_scale, Tensor b_scale) -> Tensor)tagsa6qscaled_dot_product(Tensor query, Tensor key, Tensor value, Tensor? attn_mask=None, float dropout_p=0.0, bool is_causal=False, float? scale=None, float q_scale=1.0, int q_zp=0, float k_scale=1.0, int k_zp=0, float v_scale=1.0, int v_zp=0, float a_scale=1.0, int a_zp=0, float o_scale=1.0, int o_zp=0) -> Tensorzida8w4_linear_prepack_cpu(Tensor weight, Tensor scales, Tensor qzeros) -> (Tensor, Tensor, Tensor, Tensor)zda8w4_linear_cpu(Tensor input, Tensor input_scales, Tensor input_qzeros, Tensor weight, Tensor weight_scales, Tensor weight_qzeros, Tensor compensation, Tensor? bias, ScalarType output_dtype) -> Tensorz_scaled_embedding_bag(Tensor qweight, Tensor indices, Tensor offsets, Tensor weight_scale, float o_scale, int mode, bool include_last_offset, ScalarType output_dtype) -> TensorzKfloat8_linear_prepack_cpu(Tensor weight, Tensor scales) -> (Tensor, Tensor)zfloat8_linear_cpu(Tensor input, Tensor input_scales, Tensor weight, Tensor weight_scales, Tensor? bias, ScalarType output_dtype) -> TensorcaV3RlpV#)cP<\PPS 4!V4#N)torchlibrary register_fakefuncnames&L/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torchao/ops.py decorator%register_custom_op..decorator4s}}**dV5d;;rrsf rregister_custom_opr3s< rcaV3RlpV#)cT<\PPS RR7!V4#)) mutates_argsr)r r custom_oprs&rr*register_custom_op_impl..decorator;s$}}&&$r&B4HHrrrsf rregister_custom_op_implr:sI rc\PP\PP44pVP^ ,VP ,pV#) )r cudaget_device_propertiescurrent_devicemajorminor) device_propscompute_capabilitys rcached_compute_capabilityr(AsE::33EJJ4M4M4OPL%++b0<3E3EE rc$V^8dQhR\R\R\R\\,R\R\R\\,R\R \R \R \R \R \R\R\R\R\R\/#querykeyvalue attn_mask dropout_p is_causalscaleq_scaleq_zpk_scalek_zpv_scalev_zpa_scalea_zpo_scaleo_zpreturnrrfloatboolint)formats"r __annotate__rCHs<< < < < <  <  < E?<< << << << < !<" #<$ %<rc\PPPP VVVVVVVVVV V V V V VVV4#)a Quantized SDPA with quantized inputs and outputs. Arguments query: input query tensor, key: input key tensor, value: input value tensor, attn_mask: attention mask tensor, dropout_p: dropout probability, is_causal: causal flag, scale: scaling factor applied prior to softmax, q_scale: scale for query from linear quantization, q_zp: zero point for query from linear quantization, k_scale: scale for key from linear quantization, k_zp: zero point of key from linear quantization, v_scale: zero point for value from linear quantization, v_zp: zero point of value from linear quantization, a_scale: scale for attention from softmax quantization, a_zp: zero point for attention from softmax quantization, o_scale: scale for output from linear quantization, o_zp: zero point for output from linear quantization, Returns output of quantized SDPA )r opsrqscaled_dot_productdefaultr,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<s&&&&&&&&&&&&&&&&&rrFrFHs[T 99   0 0 8 8         # rztorchao::qscaled_dot_productc$V^8dQhR\R\R\R\\,R\R\R\\,R\R \R \R \R \R \R\R\R\R\R\/#r*r>)rBs"rrCrCs>> > > > >  >  > E?>> >> >> >> > !>" #>$ %>rcbVP^^4P4P^^4#)) transpose contiguousrHs&&&&&&&&&&&&&&&&&r_rNs+( ??1a + + - 7 71 ==rcV^8dQhR\R\R\R\R\R\\,R\\P,R\/# r+input input_scaleweight weight_meta weight_scalebias out_dtyper=rrr dtype)rBs"rrCrCsf      6   $ rc n\PPPP WW#WEV4#)a CUTLASS-based row-wise scaled F8F8 linear operator, for sparsified weight case. Args: input: quantized input tensor, in row-major layout. input_scale: scale factors for input tensor, has to be tensor of the same shape as the input tensor, minus the last dimension. weight: sparsified quantized weight matrix, in row-major layout. weight_meta: sparsify metadata for weight tensor. weight_scale: scale factors for weight tensor, one value per row of weight matrix (thus also tensor of the same shape as the weight tensor, minus the last dimension). bias: an optional vector of size equal to number of rows of weight tensor, or None. out_dtype: optional data type for output tensor. Returns: output: result tensor, in row-major layout. )r rEr)rowwise_scaled_linear_sparse_cutlass_f8f8rG)rQrRrSrTrUrVrWs&&&&&&&rr[r[s0. 99   F F N N FY rz2torchao::rowwise_scaled_linear_sparse_cutlass_f8f8cV^8dQhR\R\R\R\R\R\\,R\\P,R\/#rPrX)rBs"rrCrCspYY YY Y Y  Y 6  Y $Y YrcVeTM VPpVPp\P!.VPRROVP^,N5WxR7#)NrYdevice)rYr_r emptyshape) rQrRrSrTrUrVrWrYr_s &&&&&&& rrNrNsN#.IK4E4EE \\F ;;;Sb);6<<?;5 XXrc<V^8dQhR\R\\3/#r+rSr=r)rBs"rrCrCs% W W  W f Wrch\PPPP V4#)a CUTLASS-based conversion from sparsified input tensor to corresponding compressed tensor, along with corresponding metadata tensor. Args: weight: input tensor, in row-major layout. Returns: weight_compressed: compressed weight tensor, with sparsity eliminated, in row-major layout. weight_meta: metadata tensor, describing the sparsity structure of the input tensor, also in row-major layout. )r rEr)to_sparse_semi_structured_cutlass_sm9x_f8rGrSs&rrfrfs% 99   F F N Nv VVrz2torchao::to_sparse_semi_structured_cutlass_sm9x_f8c<V^8dQhR\R\\3/#rdr)rBs"rrCrCs!     f rcVPV^,V^,^,4VPV^,\V^,^,^4\PR73#)rrY) new_emptymaxr charrgs&rrNrNsV F1IN3Cq Q$;5::N rc TV^8dQhR\R\R\R\R\/#r+mat1mat2mat1_is_swizzledmat2_is_swizzledr=rr@)rBs"rrCrCs4 26JN rcj\PPPP WW#4#)zG Similar to torch.mm but Tensor inputs can be SwizzleTensor instances. )r rEr swizzle_mmrGrprqrrrss&&&&rrvrvs- 99   ' ' / / $ rztorchao::swizzle_mmc TV^8dQhR\R\R\R\R\/#rort)rBs"rrCrCs488 88268JN8 8rcjVPVP^,VP^,4#rrkrbrws&&&&rrNrNs% >>$**Q-A 77rcV^8dQhR\R\R\R\R\R\R\\,R\\,R \\P,R \/ # r+rprqrrrsscale_ascale_brV scale_resultrWr=rr@rr rY)rBs"rrCrCs        6 6" $ rc x\PPPP VVVVVVVVV4 #)zH Similar to torch.mm but Tensor inputs can be SwizzleTensor instances. )r rErswizzle_scaled_mmrG rprqrrrsr~rrVrrWs &&&&&&&&&rrrsB 99   . . 6 6     rztorchao::swizzle_scaled_mmcV^8dQhR\R\R\R\R\R\R\\,R\\,R \\P,R \/ #r}r)rBs"rrCrCs 8 8  8  8 8 8  8  8 6  86" 8 $ 8  8rc jVPVP^,VP^,4#rzr{rs &&&&&&&&&rrNrNs% >>$**Q-A 77rc\\R4'd!\P\P3#\P3#)zGTODO: when e8m0 is hardened and major release lets remove uint8 supportfloat8_e8m0fnu)hasattrr uint8rrrr _get_dtypesr%s3u&'' U1122 KK>rcaa\4p\P!SPV9V3Rl4\P!SPV9V3Rl4R#)c"<RSP 2#)z4A_scale tensor must be uint8 or float8_e8m0fnu, got rj)A_scalesr%_check_scale_dtypes..2Fw}}oVrc"<RSP 2#)z4B_scale tensor must be uint8 or float8_e8m0fnu, got rj)B_scalesrrr6rrN)rr _checkrY)rrallowed_dtypessff r_check_scale_dtypesr-sB ]N LL 'V LL 'Vrztorchao::mx_fp8_bf16cHV^8dQhR\R\R\R\/#)r+ABrrr)rBs"rrCrC;s.VVV6VFVVVrc\P!VP^4VP^43\PVPR7#)zMeta impl for mx_fp8_bf16r^)r rasizebfloat16r_)rrrrs&&&&rmeta_mx_fp8_bf16r:s4 ;;q 166!9-U^^AHH UUrcHV^8dQhR\R\R\R\/#r+rSscalesqzerosr=r)rBs"rrCrC@s6VV V V V Vrcj\PPPP WV4#)z Prepack weights for DA8W4 linear operator on CPU. Args: weight: weight tensor. scales: scales for weight tensor. qzeros: zero points for weight tensor. Returns: packed weight, scales, and zero points. )r rErda8w4_linear_prepack_cpurGrSrrs&&&rrr@s' 99   5 5 = =ff UUrz!torchao::da8w4_linear_prepack_cpucHV^8dQhR\R\R\R\/#rr)rBs"rrCrCRs(22f2f2f22rc2WV\P!43#r )r rrs&&&rrNrNQs 65<<> 11rcV^8dQhR\R\R\R\R\R\R\R\\,R \P/ #) r+rQ input_scales input_qzerosrS weight_scales weight_qzeros compensationrVrWrX)rBs"rrCrCVsk$$ $$$  $  $  $$ 6 ${{$rc x\PPPP VVVVVVVVV4 #)a DA8W4 linear operator on CPU. Args: input: input tensor. input_scales: scales for input tensor. input_qzeros: zero points for input tensor. weight: weight tensor. weight_scales: scales for weight tensor. weight_qzeros: zero points for weight tensor. compensation: compensation tensor for weight. bias: optional bias tensor. out_dtype: output data type. Returns: output tensor in out_dtype. )r rErda8w4_linear_cpurG) rQrrrSrrrrVrWs &&&&&&&&&rrrVsB4 99   - - 5 5    rztorchao::da8w4_linear_cpucV^8dQhR\R\R\R\R\R\R\R\\,R \PR \/ #) r+rQrrrSrrrrVrWr=rX)rBs"rrCrC~s B B  B B B  B  B  B B 6  B{{ B  Brc VP4^8XgQhVP^4VP^4,^,p VP!.VPRROV N5RV/#)NrYr`dimrrkrb) rQrrrSrrrrVrWNs &&&&&&&&& rrNrN}s^ ::<1   AQ'!+A ?? AEKK, Aa Ay AArztorchao::_scaled_embedding_bagcV^8dQhR\R\R\R\R\R\R\R\P R \/ #) r+qweightindicesoffsetsw_scalesr;modeinclude_last_offsetrWr=)rr?rAr@r rY)rBs"rrCrCsq L L  L  L L L  L  L L{{ L  LrcVR8XgQhVP^,^, pVPWP^,VR7#)Trj)rbrk) rrrrr;rrrW batch_sizes &&&&&&&& rrNrNsD $ && &q!A%J   Zq)9  KKrc<V^8dQhR\R\R\/#r+rSrr=r)rBs"rrCrCs+ O O  O  O  Orch\PPPP W4#)z Prepack weights for float8 linear operator on CPU. Args: weight: weight tensor. scales: scales for weight tensor. Returns: packed weight, packed scales )r rErfloat8_linear_prepack_cpurGrSrs&&rrrs% 99   6 6 > >v NNrz"torchao::float8_linear_prepack_cpuc<V^8dQhR\R\R\/#rr)rBs"rrCrCs!ffrcW3#r rrs&&rrNrNs >rc V^8dQhR\R\R\R\R\\,R\P/#)r+rQrrSrrVrWrX)rBs"rrCrCsM    6   {{ rcr\PPPP VVVVVV4#)a float8 linear operator on CPU. Args: input: input tensor. input_scales: scales for input tensor. weight: weight tensor. weight_scales: scales for weight tensor. bias: optional bias tensor. out_dtype: output data type. Returns: output tensor in out_dtype. )r rErfloat8_linear_cpurG)rQrrSrrVrWs&&&&&&rrrs9( 99   . . 6 6    rztorchao::float8_linear_cpucV^8dQhR\R\R\R\R\\,R\PR\/#)r+rQrrSrrVrWr=rX)rBs"rrCrCs` B B  B B  B B 6  B {{ B  BrcVP4R9gQhVP4^8Xd(VP^4VP^4,MVP^4pVP!.VPRROVN5RV/#)r+NrY)r+rr`r)rQrrSrrVrWrs&&&&&& rrNrNsp ::<6 !! !+1::<1+< AQ'&++a.A ?? AEKK, Aa Ay AAr)NgFN?rrrrrrrrr)NN) functoolstypingrr rr Librarylibdefine_CTagneeds_fixed_stride_orderrr lru_cacher(rFrNr[rfrvrrrrrrrrrrrrs   mmIz2 C R b T  O ((,, / / 0  } o P w Q Q   <~23>4>,8HIYJY" WHI J )*8+8 801 82 8  *+V,V V"78292$N/0 B1 B 45 L6 L O89:<01 B2 Br