+
    œ°³j,u  ã                   óâ  € ^ RI t ^ RIt^ RIt^ RIHt ^ RIHt ^RIHt ]P                  ! ]
4      t ! R R]P                  4      t ! R R]P                  4      t ! R	 R
]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tRR ltRR ltRR lt ! R R]4      t ! R R]4      t ! R R]4      tRR lt ! R R]4      tR# )é    N)Únn)ÚFunction)Úloggingc                   óD   a a€ ] tR t^t oRtRV 3R lltRR ltRtVtV ;t	# )ÚQuantEmbeddingaº  
Quantized version of `torch.nn.Embedding`. Adds quantization-specific arguments on top of `torch.nn.Embedding`.

Args:
    weight_bit (`int`, *optional*, defaults to `8`):
        Bitwidth for the quantized weight.
    momentum (`float`, *optional*, defaults to `0.95`):
        Momentum for updating the activation quantization range.
    quant_mode (`bool`, *optional*, defaults to `False`):
        Whether or not the layer is quantized.
c                ó  <€ \         SV `  4        Wn        W n        W0n        W@n        WPn        W`n        Wpn        \        P                  ! \        P                  ! W.4      4      V n        \        P                  ! \        P                  ! ^4      4      V n        \        P                  ! \        P                   ! V P                  4      4      V n        W�n        W n        W°n        RV n        \,        P.                  V n        R# )é   FN)ÚsuperÚ__init__Únum_ÚdimÚpadding_idxÚmax_normÚ	norm_typeÚscale_grad_by_freqÚsparser   Ú	ParameterÚtorchÚzerosÚweightÚBufferÚweight_scaling_factorÚ
zeros_likeÚweight_integerÚ
weight_bitÚmomentumÚ
quant_modeÚpercentile_modeÚSymmetricQuantFunctionÚapplyÚweight_function)ÚselfÚnum_embeddingsÚembedding_dimr   r   r   r   r   Ú_weightr   r   r   Ú	__class__s   &&&&&&&&&&&&€Úh/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/ibert/quant_modules.pyr   ÚQuantEmbedding.__init__+   s¯   ø€ ô 	‰ÑÔØ"Œ	Ø ŒØ&ÔØ ŒØ"ŒØ"4ÔØŒä—l’l¤5§;¢;°Ð/NÓ#OÓPˆŒÜ%'§Y¢Y¬u¯{ª{¸1«~Ó%>ˆÔ"Ü Ÿiši¬×(8Ò(8¸¿¹Ó(EÓFˆÔà$ŒØ ŒØ$ŒØ$ˆÔÜ5×;Ñ;ˆÖó    c           	     ól  € V P                   '       gd   \        P                  P                  VV P                  V P
                  V P                  V P                  V P                  V P                  4      R 3# V P                  pVP                  P                  4       pVP                  4       P                  ^4      pVP                  4       P                  ^4      p\        V P                   WgR4      V n        V P%                  V P                  V P                   V P&                  V P"                  4      V n        \        P                  P                  VV P(                  V P
                  V P                  V P                  V P                  V P                  4      pW€P"                  ,          V P"                  3# )NF)r   r   Ú
functionalÚ	embeddingr   r   r   r   r   r   ÚdataÚdetachÚminÚexpandÚmaxÚ$symmetric_linear_quantization_paramsr   r   r!   r   r   )	r"   ÚxÚ	positionsÚincremental_stateÚwÚw_transformÚw_minÚw_maxÚemb_ints	   &&&&     r'   ÚforwardÚQuantEmbedding.forwardL   sQ  € Ø��ˆä—‘×'Ñ'ØØ—K‘KØ×$Ñ$Ø—M‘MØ—N‘NØ×+Ñ+Ø—K‘Kóð ðð ð �K‰KˆØ—f‘f—m‘m“oˆØ—‘Ó!×(Ñ(¨Ó+ˆØ—‘Ó!×(Ñ(¨Ó+ˆä%IÈ$Ï/É/Ð[`ÐinÓ%oˆÔ"Ø"×2Ñ2Ø�K‰K˜Ÿ™¨$×*>Ñ*>À×@ZÑ@Zó
ˆÔô —-‘-×)Ñ)ØØ×ÑØ×ÑØ�M‰MØ�N‰NØ×#Ñ#Ø�K‰Kó
ˆð ×3Ñ3Õ3°T×5OÑ5OÐOÐOr)   )r   r   r   r   r   r   r   r   r   r   r   r   r!   r   r   )	NNç       @FFNé   çffffffî?F©NN)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r;   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r&   Ú__classdict__s   @@r'   r   r      s   ù‡ € ñ
÷<÷B"Pô "Pr)   r   c                   óJ   a a€ ] tR t^qt oRtRV 3R lltR tRR ltRtVt	V ;t
# )ÚQuantActa<  
Quantizes the given activation.

Args:
    activation_bit (`int`):
        Bitwidth for the quantized activation.
    act_range_momentum (`float`, *optional*, defaults to `0.95`):
        Momentum for updating the activation quantization range.
    per_channel (`bool`, *optional*, defaults to `False`):
        Whether to or not use channel-wise quantization.
    channel_len (`int`, *optional*):
        Specify the channel length when set the *per_channel* True.
    quant_mode (`bool`, *optional*, defaults to `False`):
        Whether or not the layer is quantized.
c                óD  <€ \         SV `  4        Wn        W n        WPn        W0n        R V n        \        P                  V n	        V P
                  '       gÂ   \        P                  ! \        P                  ! ^4      4      V n        \        P                  ! \        P                  ! ^4      4      V n        \        P                  ! \        P                  ! ^4      4      V n        V ;P                  R,          un        V ;P                  R,          un        R# \#        R4      h)Fgñhãˆµøä>ú;per-channel mode is not currently supported for activation.N)r
   r   Úactivation_bitÚact_range_momentumr   Úper_channelÚ
percentiler   r    Úact_functionr   r   r   r   Úx_minÚx_maxÚact_scaling_factorÚNotImplementedError)r"   rO   rP   rQ   Úchannel_lenr   r&   s   &&&&&&€r'   r   ÚQuantAct.__init__‚   s·   ø€ Ü‰ÑÔà,ÔØ"4ÔØ$ŒØ&ÔØˆŒÜ2×8Ñ8ˆÔà××ÐÜŸš¤5§;¢;¨q£>Ó2ˆDŒJÜŸš¤5§;¢;¨q£>Ó2ˆDŒJÜ&(§i¢i´·²¸A³Ó&?ˆDÔ#Ø�JŠJ˜$Õ�JØ�JŠJ˜$Õ�Jä%Ð&cÓdÐdr)   c           
     óØ   € V P                   P                   R V P                   RV P                   RV P                  P                  4       R RV P                  P                  4       R R2
# )z(activation_bit=z, quant_mode: z, Act_min: z.2fz, Act_max: Ú))r&   rA   rO   r   rT   ÚitemrU   )r"   s   &r'   Ú__repr__ÚQuantAct.__repr__•   si   € à�~‰~×&Ñ&Ð'Ð'7¸×8KÑ8KÐ7Lð MØŸ?™?Ð+¨;°t·z±z·±Ó7HÈÐ6Mð NØŸ
™
Ÿ™Ó)¨#Ð.¨að1ð	
r)   c                ó0  € Vf   TMW1,           pV P                   '       Ed   V P                  '       d   Q R4       hV P                  '       d   Q R4       hVP                  P	                  4       pVP                  P                  4       p	V	P                  4       P                  4       ^ 8X  d$   VP                  4       P                  4       ^ 8X  g   Q R4       hV P                  P	                  4       R8”  dQ   V P                  P                  4       R8  d2   V P                  V,           V n        V P                  V	,           V n	        MàV P                  R8X  dN   \        P                  ! V P                  V4      V n        \        P
                  ! V P                  V	4      V n	        M‚V P                  V P                  ,          V^V P                  ,
          ,          ,           V n        V P                  V P                  ,          V	^V P                  ,
          ,          ,           V n	        V P                  '       g   VR 3# Vf   V P                  MTpVf   V P                  MTp	\        V P                  W‰V P                  R7      V n        Vf3   V P!                  WP                  V P                  V P                  4      p
M.\"        P%                  VVV P                  V P                  VV4      p
V P                  P'                  R4      pW«,          V P                  3# )Nz:percentile mode is not currently supported for activation.rN   z5NaN detected when computing min/max of the activationg¢&ú|”ç>)rQ   g¢&ú|”ç¾éÿÿÿÿ)ÚtrainingrR   rQ   r-   r/   r1   ÚisnanÚsumrT   rU   rP   r   r   r2   rO   rV   rS   ÚFixedPointMulr    Úview)r"   r3   Úpre_act_scaling_factorÚidentityÚidentity_scaling_factorÚspecified_minÚspecified_maxÚx_actrT   rU   Úquant_act_intÚcorrect_output_scales   &&&&&&&     r'   r;   ÚQuantAct.forwardœ   s[  € ð Ò%‘¨8­<ˆà�=�=‰=Ø——�ÐdÐ(dÓdÐ&Ø×'×'Ð'ÐfÐ)fÓfÐ'Ø—J‘J—N‘NÓ$ˆEØ—J‘J—N‘NÓ$ˆEà—;‘;“=×$Ñ$Ó&¨!Ô+°·±³×0AÑ0AÓ0CÀqÔ0Hð ØGóÐHð
 �z‰z�~‰~Ó 'Ô)¨d¯j©j¯n©nÓ.>ÀÔ.GØ!ŸZ™Z¨%Õ/�”
Ø!ŸZ™Z¨%Õ/�•
ð ×(Ñ(¨BÔ.Ü"ŸYšY t§z¡z°5Ó9�”
Ü"ŸYšY t§z¡z°5Ó9�•
à!ŸZ™Z¨$×*AÑ*AÕAÀEÈQÐQU×QhÑQhÕMhÕDiÕi�”
Ø!ŸZ™Z¨$×*AÑ*AÕAÀEÈQÐQU×QhÑQhÕMhÕDiÕi�”
à��ˆØ˜$�;Ðà+Ò3�—
’
¸ˆØ+Ò3�—
’
¸ˆä"FØ×Ñ ¸4×;KÑ;Kô#
ˆÔð "Ò)à ×-Ñ-¨a×1DÑ1DÀdÇoÁoÐW[×WnÑWnÓo‰Mä)×/Ñ/ØØ&Ø×#Ñ#Ø×'Ñ'ØØ'óˆMð  $×6Ñ6×;Ñ;¸BÓ?ÐàÕ3°T×5LÑ5LÐLÐLr)   )	rS   rP   rV   rO   rQ   rR   r   rU   rT   )r?   FNF)NNNNN©rA   rB   rC   rD   rE   r   r]   r;   rF   rG   rH   rI   s   @@r'   rL   rL   q   s!   ù‡ € ñ÷ eò&
÷<Mô <Mr)   rL   c                   óP   a a€ ] tR t^Ût oRtRV 3R lltV 3R ltRR ltRtVt	V ;t
# )ÚQuantLineara  
Quantized version of `torch.nn.Linear`. Adds quantization-specific arguments on top of `torch.nn.Linear`.

Args:
    weight_bit (`int`, *optional*, defaults to `8`):
        Bitwidth for the quantized weight.
    bias_bit (`int`, *optional*, defaults to `32`):
        Bitwidth for the quantized bias.
    per_channel (`bool`, *optional*, defaults to `False`):
        Whether or not to use channel-wise quantization.
    quant_mode (`bool`, *optional*, defaults to `False`):
        Whether or not the layer is quantized.
c                óÔ  <€ \         SV `  4        Wn        W n        \        P
                  ! \        P                  ! W!.4      4      V n        \        P                  ! \        P                  ! V P                  4      4      V n        \        P                  ! \        P                  ! V P                  4      4      V n        V'       di   \        P
                  ! \        P                  ! V4      4      V n        \        P                  ! \        P                  ! V P                  4      4      V n        W@n        Wpn        W`n        WPn        Wpn        R V n        \(        P*                  V n        R# )FN)r
   r   Úin_featuresÚout_featuresr   r   r   r   r   r   r   r   Úfc_scaling_factorÚbiasÚbias_integerr   r   rQ   Úbias_bitr   r   r    r!   )	r"   rs   rt   rv   r   rx   rQ   r   r&   s	   &&&&&&&&€r'   r   ÚQuantLinear.__init__ê   sä   ø€ ô 	‰ÑÔØ&ÔØ(Ôä—l’l¤5§;¢;°Ð/JÓ#KÓLˆŒÜ Ÿiši¬×(8Ò(8¸¿¹Ó(EÓFˆÔÜ!#§¢¬5¯;ª;°t×7HÑ7HÓ+IÓ!JˆÔßÜŸš¤U§[¢[°Ó%>Ó?ˆDŒIÜ "§	¢	¬%×*:Ò*:¸4¿9¹9Ó*EÓ FˆDÔà$ŒØ$ŒØ&ÔØ ŒØ$ŒØ$ˆÔÜ5×;Ñ;ˆÖr)   c                ód   <€ \         SV `  4       pR V RV P                   RV P                   R2pV# )Ú(z weight_bit=z, quant_mode=r[   )r
   r]   r   r   )r"   Úsr&   s   & €r'   r]   ÚQuantLinear.__repr__   s9   ø€ Ü‰GÑÓˆØ�ˆs�,˜tŸ™Ð/¨}¸T¿_¹_Ð<MÈQÐOˆØˆr)   c                ó@  € V P                   '       g8   \        P                  P                  WP                  V P
                  R 7      R3# Ve   VP                  R8X  g   Q R4       hV P                  pVP                  P                  4       pV P                  '       d8   \        P                  ! V^RR7      w  rV\        P                  ! V^RR7      w  rvM>VP                  4       P                  ^4      pVP                  4       P                  ^4      p\        V P                  WWV P                  4      V n        V P#                  V P                  V P                  V P$                  V P                   4      V n        V P                   V,          pV P
                  e.   V P#                  V P
                  V P(                  RV4      V n        VP-                  ^R4      pW,          p	\        P                  P                  W�P&                  V P*                  R 7      V,          V3# ))r   rv   Nz«Input activation to the QuantLinear layer should be globally (non-channel-wise) quantized. Please add a QuantAct layer with `per_channel = True` before this QuantAct layer)r   ÚoutF©r	   r`   )r   r   r+   Úlinearr   rv   Úshaper-   r.   rQ   r   r/   r1   r0   r2   r   ru   r!   r   r   rx   rw   re   )
r"   r3   Úprev_act_scaling_factorr6   r7   r8   Ú_r9   Úbias_scaling_factorÚx_ints
   &&&       r'   r;   ÚQuantLinear.forward  s»  € Ø��ˆÜ—=‘=×'Ñ'¨·+±+ÀDÇIÁIÐ'ÓNÐPTÐTÐTð 'Ò2Ð7N×7TÑ7TÐX\Ô7\ð 	
ð_ó	
Ð\ð
 �K‰KˆØ—f‘f—m‘m“oˆØ××ÐÜ—y’y °!¸Ô>‰HˆEÜ—y’y °!¸Ô>‰HˆE�1à—O‘OÓ%×,Ñ,¨QÓ/ˆEØ—O‘OÓ%×,Ñ,¨QÓ/ˆEä!EÀdÇoÁoÐW\Ðei×euÑeuÓ!vˆÔØ"×2Ñ2Ø�K‰K˜Ÿ™¨$×*>Ñ*>À×@VÑ@Vó
ˆÔð #×4Ñ4Ð7NÕNÐà�9‰9Ò Ø $× 4Ñ 4°T·Y±YÀÇÁÈuÐViÓ jˆDÔà"9×">Ñ">¸qÀ"Ó"EÐØÕ+ˆô �M‰M× Ñ  ×/BÑ/BÈ×IZÑIZÐ Ó[Ð^qÕqØð
ð 	
r)   )rv   rx   rw   ru   rs   rt   rQ   r   r   r   r   r!   r   )Tr>   é    FF©Nro   rI   s   @@r'   rq   rq   Û   s   ù‡ € ñ÷<õ,÷
#
ô #
r)   rq   c                   óJ   a a€ ] tR tRt oRtRV 3R lltR tRR ltRtVt	V ;t
# )	ÚIntGELUi+  aa  
Quantized version of `torch.nn.GELU`. Adds quantization-specific arguments on top of `torch.nn.GELU`.

Args:
    quant_mode (`bool`, *optional*, defaults to `False`):
        Whether or not the layer is quantized.
    force_dequant (`str`, *optional*, defaults to `"none"`):
        Force dequantize the layer if either "gelu" or "nonlinear" is given.
c                ó\  <€ \         SV `  4        Wn        VR9   d   \        P	                  R4       RV n        V P                  '       g   \
        P                  ! 4       V n        RV n        ^V n	        . ROV n
        V P                  ^;;,          V P                  ^ ,          ,          uu&   R# )Ú	nonlinearzForce dequantize geluFgà-� ö?N)r�   Úgelu)g]mÅþ²{Ò¿gçû©ñÒMü¿r	   )r
   r   r   ÚloggerÚinfor   ÚGELUÚactivation_fnÚkÚconstÚcoeff)r"   r   Úforce_dequantr&   s   &&&€r'   r   ÚIntGELU.__init__6  sv   ø€ Ü‰ÑÔØ$ŒàÐ1Ô1Ü�K‰KÐ/Ô0Ø#ˆDŒOà��ˆÜ!#§¢£ˆDÔàˆŒØˆŒ
Ú)ˆŒ
Ø�
‰
�1�˜Ÿ™ A�Õ&�r)   c                ó\  € \         P                  ! V P                  ^,          V,          4      p\         P                  ! V P                  ^,          V^,          ,          4      p\         P                  ! V4      p\         P                  ! \         P
                  ! V4      V) 4      pWVV,           ^,          V,           ,          pV^,          V P                  ^ ,          ,          p\        P                  V^V P                  ,          ,          4      pV^V P                  ,          ,          pWr3# r€   )	r   Úfloorr•   Úsignr/   ÚabsÚ	floor_ster    r”   )r"   r†   Úscaling_factorÚb_intÚc_intrš   Úabs_intÚy_ints   &&&     r'   Úint_erfÚIntGELU.int_erfF  sË   € Ü—’˜DŸJ™J q�M¨NÕ:Ó;ˆÜ—’˜DŸJ™J q�M¨N¸AÕ,=Õ=Ó>ˆÜ�zŠz˜%Ó ˆä—)’)œEŸIšI eÓ,¨u¨fÓ5ˆØ 5�¨QÕ.°Õ6Õ7ˆØ'¨Õ*¨T¯Z©Z¸­]Õ:ˆô —‘ ¨¨4¯:©:­Õ 5Ó6ˆØ'¨!¨T¯Z©Z­-Õ7ˆàÐ$Ð$r)   c                ó  € V P                   '       g   V P                  V4      R 3# W,          pV P                  W2V P                  ,          4      w  rERV,          pW4V,           ,          pW%,          ^,          pW2,          V3# )Nç      ð?)r   r’   r¢   r“   )r"   r3   r�   r†   Úsigmoid_intÚsigmoid_scaling_factorÚ	shift_ints   &&&    r'   r;   ÚIntGELU.forwardU  sy   € Ø��ˆØ×%Ñ% aÓ(¨$Ð.Ð.àÕ"ˆØ.2¯l©l¸5ÐSW×SYÑSYÕBYÓ.ZÑ+ˆàÐ1Õ1ˆ	à yÕ0Õ1ˆØ'Õ@À1ÕDˆàÕ% ~Ð5Ð5r)   )r’   r•   r”   r“   r   )TÚnoner‰   )rA   rB   rC   rD   rE   r   r¢   r;   rF   rG   rH   rI   s   @@r'   r‹   r‹   +  s   ù‡ € ñ÷'ò %÷6ô 6r)   r‹   c                   óL   a a€ ] tR tRt oRtRV 3R lltR tR tR tRt	Vt
V ;t# )	Ú
IntSoftmaxid  a´  
Quantized version of `torch.nn.Softmax`. Adds quantization-specific arguments on top of `torch.nn.Softmax`.

Args:
    output_bit (`int`):
        Bitwidth for the layer output activation.
    quant_mode (`bool`, *optional*, defaults to `False`):
        Whether or not the layer is quantized.
    force_dequant (`str`, *optional*, defaults to `"none"`):
        Force dequantize the layer if either "softmax" or "nonlinear" is given.
c                ó¶  <€ \         SV `  4        Wn        ^ V n        W n        VR9   d   \
        P                  R4       RV n        \        ^V P                  R7      V n        RV n	        ^V n
        . ROV n        V P                  ^;;,          V P                  ^ ,          ,          uu&   V P                  ^;;,          V P                  ^ ,          ,          uu&   R# )rˆ   zForce dequantize softmaxF©r   N)r�   Úsoftmaxgvqà-æ¿)gN„ª$ôëÖ?g¾Ã'|:ï?r¥   )r
   r   Ú
output_bitÚmax_bitr   r�   r�   rL   ÚactÚx0r”   Úcoef)r"   r°   r   r–   r&   s   &&&&€r'   r   ÚIntSoftmax.__init__q  s—   ø€ Ü‰ÑÔØ$ŒØˆŒØ$ŒàÐ4Ô4Ü�K‰KÐ2Ô3Ø#ˆDŒOä˜B¨4¯?©?Ô;ˆŒØˆŒØˆŒ
Ú1ˆŒ	Ø�	‰	�!�˜Ÿ	™	 !�Õ$‹Ø�	‰	�!�˜Ÿ	™	 !�Õ$�r)   c                ó°  € \         P                  ! 4       ;_uu_ 4        \         P                  ! V P                  ^,          V,          4      p\         P                  ! V P                  ^,          V^,          ,          4      pRRR4       VX,           V,          X,           pV P                  ^ ,          V^,          ,          pWR3#   + '       g   i     LK; i)r	   N)r   Úno_gradr™   r´   )r"   r†   r�   rž   rŸ   Úzs   &&&   r'   Úint_polynomialÚIntSoftmax.int_polynomial‚  sŠ   € Ü�]Š]�_�_Ü—K’K §	¡	¨!¥¨~Õ =Ó>ˆEÜ—K’K §	¡	¨!¥¨~¸qÕ/@Õ @ÓAˆE÷ ð �U�]˜eÕ# eÕ+ˆØŸ™ 1�¨¸Õ(9Õ9ˆØÐ Ð ÷ �_ús   žA$CÃC	c                óX  € \         P                  ! 4       ;_uu_ 4        \         P                  ! V P                  V,          4      pR R R 4       \         P                  ! WP
                  X,          4      p\        P                  W,          4      pWV,          ,
          pV P                  WR4      w  rg\         P                  ! \        P                  V^V P
                  V,
          ,          ,          4      ^ R7      pV^V P
                  ,          ,          pWb3#   + '       g   i     LÛ; i)N©r/   )
r   r·   r™   r³   r1   r”   rœ   r    r¹   Úclamp)r"   r†   r�   Úx0_intÚqÚrÚexp_intÚexp_scaling_factors   &&&     r'   Úint_expÚIntSoftmax.int_expŠ  s¾   € Ü�]Š]�_�_Ü—[’[ §¡¨>Õ!9Ó:ˆF÷ ä—	’	˜%§¡¨fÕ!4Ó5ˆä�O‰O˜E�NÓ+ˆØ˜Q•JÕˆØ&*×&9Ñ&9¸!Ó&LÑ#ˆÜ—+’+œiŸo™o¨g¸¸d¿j¹jÈ1½nÕ8MÕ.MÓNÐTUÔVˆØ+¨a°·±­mÕ;ˆØÐ&Ð&÷ �_ús   ž(DÄD)	c                ób  € V P                   '       g$   \        P                  P                  VRR7      R3# W,          pVP	                  RRR7      w  rEW4,
          pV P                  W24      w  rgV P                  Wg4      w  r‡W‡,          pVP                  RRR7      p	\        P                  ^V P                  ,          V	,          4      p
\        P                  Wj,          ^V P                  V P                  ,
          ,          ,          4      p^^V P                  ,          ,          pWb,          V3# )r	   ©r   NT)r   Úkeepdimr`   )r   r   r+   r¯   r1   rÃ   r²   rc   rœ   r    r±   r°   )r"   r3   r�   r†   Ú	x_int_maxr„   rÁ   rÂ   ÚexpÚexp_int_sumÚfactors   &&&        r'   r;   ÚIntSoftmax.forward–  sð   € Ø��ˆÜ—=‘=×(Ñ(¨°Ð(Ó3°TÐ9Ð9àÕ"ˆà—y‘y R°�yÓ6‰ˆ	ØÕ!ˆØ&*§l¡l°5Ó&IÑ#ˆð #'§(¡(¨7Ó"GÑˆØÕ*ˆà—k‘k b°$�kÓ7ˆÜ—‘  D§L¡L¥°;Õ!>Ó?ˆÜ—/‘/ 'Õ"2°Q¸4¿<¹<È$Ï/É/Õ;YÕ5ZÕ"ZÓ[ˆØ˜Q §¡Õ/Õ/ˆØÕ'¨Ð7Ð7r)   )r²   r´   r”   r±   r°   r   r³   )Frª   )rA   rB   rC   rD   rE   r   r¹   rÃ   r;   rF   rG   rH   rI   s   @@r'   r¬   r¬   d  s#   ù‡ € ñ
÷%ò"!ò
'÷8ò 8r)   r¬   c                   óP   a a€ ] tR tRt oRtRV 3R lltR tR tR	R ltRt	Vt
V ;t# )
ÚIntLayerNormi«  a×  
Quantized version of `torch.nn.LayerNorm`. Adds quantization-specific arguments on top of `torch.nn.LayerNorm`.

Args:
    output_bit (`int`, *optional*, defaults to `8`):
        Bitwidth for the layer output activation.
    quant_mode (`bool`, *optional*, defaults to `False`):
        Whether or not the layer is quantized.
    force_dequant (`str`, *optional*, defaults to `"none"`):
        Force dequantize the layer if either "layernorm" or "nonlinear" is given.
c                ó  <€ \         SV `  4        Wn        W n        \        P
                  ! \        P                  ! V4      4      V n        \        P
                  ! \        P                  ! V4      4      V n	        W@n
        VR9   d   \        P                  R4       RV n
        \        P                  ! \        P                  ! ^4      4      V n        W0n        ^ V n        RV n        \%        V P                  V P                  R7      V n        R# )r�   zForce dequantize layernormFNr®   )r�   Ú	layernorm)r
   r   Únormalized_shapeÚepsr   r   r   r   r   rv   r   r�   r�   r   Úshiftr°   r±   Údim_sqrtrL   Ú
activation)r"   rÑ   rÒ   r°   r   r–   r&   s   &&&&&&€r'   r   ÚIntLayerNorm.__init__¸  s°   ø€ Ü‰ÑÔØ 0ÔØŒä—l’l¤5§;¢;Ð/?Ó#@ÓAˆŒÜ—L’L¤§¢Ð-=Ó!>Ó?ˆŒ	à$ŒØÐ6Ô6Ü�K‰KÐ4Ô5Ø#ˆDŒOä—Y’YœuŸ{š{¨1›~Ó.ˆŒ
Ø$ŒØˆŒØˆŒÜ" 4§?¡?¸t¿¹ÔOˆŽr)   c           
     óL  € \         P                  ! 4       ;_uu_ 4        V^,          p\         P                  ! V^RR7      p\         P                  ! \         P                  ! V^V P
                  ,          ,          4      4      P                  4       P                  4       pV P                  p\         P                  ! V P                  V4      V n        \        P                  R\        V4       R\        V P                  4       24       RRR4       R#   + '       g   i     R# ; i)é   T©ÚaxisrÇ   zDynamic shift adjustment: z -> N)r   r·   rc   Úlog2Úsqrtr±   Úceilr1   rÓ   r�   r�   Úint)r"   r¡   Úy_sq_intÚvar_intrÓ   Ú	shift_olds   &&    r'   Ú	set_shiftÚIntLayerNorm.set_shiftË  s±   € Ü�]Š]�_�_Ø˜a•xˆHÜ—i’i ¨q¸$Ô?ˆGÜ—Z’Z¤§
¢
¨7°Q¸¿¹µ_Õ+DÓ EÓF×KÑKÓM×RÑRÓTˆEØŸ
™
ˆIÜŸš 4§:¡:¨uÓ5ˆDŒJÜ�K‰KÐ4´S¸³^Ð4DÀDÌÈTÏZÉZËÐHYÐZÔ[÷ �_�_Š_ús   žC*DÄD#	c                óÆ   € V P                  V4       \        P                  V^V P                  ,          ,          4      pV^,          p\        P
                  ! V^RR7      pV# )z™
This fallback function is called when overflow is detected during training time, and adjusts the `self.shift`
to avoid overflow in the subsequent runs.
TrÙ   )râ   rœ   r    rÓ   r   rc   )r"   r¡   Úy_int_shiftedrß   rà   s   &&   r'   Úoverflow_fallbackÚIntLayerNorm.overflow_fallbackÔ  sL   € ð
 	�‰�uÔÜ!Ÿ™¨°°4·:±:µÕ(=Ó>ˆØ  !Õ#ˆÜ—)’)˜H¨1°dÔ;ˆØˆr)   c                ó|  € V P                   '       g‘   VP                  ^RR7      pW,
          p\        P                  ! V^,          ^RR7      pV\        P                  ! V P                  V,           4      ,          pWP
                  ,          V P                  ,           pVR3# V P                  fl   \        P                  ! VP                  ^,          \        P                  R7      p\        P                  ! V4      P                  VP                  4      V n        W,          p\        P                  VP                  ^RR7      4      pWx,
          p	\        P                  V	^V P                   ,          ,          4      p
V
^,          p\        P"                  ! V^RR7      pV P$                  '       dl   VP'                  4       ^V P(                  ,          8¼  dF   V P+                  V	4      pVP'                  4       ^V P(                  ,          R,           8  g   Q R4       h\        P                  \        P                  ! V4      4      ^V P                   ,          ,          p\        P                  RV,          4      p\        P                  Wž,          ^,          4      p	V P                  R,          pV P                  P,                  P/                  4       V P
                  P,                  P/                  4       ,          p\        P                  Wò,          4      pV	V,           p	W P
                  ,          pW’,          pW3# )	rØ   TrÙ   N)Údtypegš™™™™™¹?zfError detected in overflow handling: `var_int` exceeds `self.max_bit` (the maximum possible bit width)l        i   @)r   Úmeanr   rÜ   rÒ   r   rv   rÔ   Útensorr‚   ÚfloatÚtoÚdeviceÚ	round_ster    rœ   rÓ   rc   ra   r1   r±   ræ   r-   r.   )r"   r3   r�   rê   ÚyÚvarÚnr†   Úmean_intr¡   rå   rß   rà   Ústd_intrË   rv   Úbias_ints   &&&              r'   r;   ÚIntLayerNorm.forwardß  s>  € Ø��ˆØ—6‘6˜q¨$�6Ó/ˆDØ•ˆAÜ—*’*˜Q �T¨°4Ô8ˆCØ”E—J’J˜tŸx™x¨#�~Ó.Õ.ˆAØ—K‘K• $§)¡)Õ+ˆAØ�d�7ˆNð �=‰=Ò Ü—’˜QŸW™W Q�Z¬u¯{©{Ô;ˆAÜ!ŸJšJ q›M×,Ñ,¨Q¯X©XÓ6ˆDŒMð Õ"ˆÜ—?‘? 5§:¡:°1¸d :Ó#CÓDˆØÕ ˆÜ!Ÿ™¨°°4·:±:µÕ(=Ó>ˆØ  !Õ#ˆÜ—)’)˜H¨1°dÔ;ˆð �=�=ˆ=à�{‰{‹}  4§<¡<¥Ô/Ø×0Ñ0°Ó7�Ø—{‘{“} q¨$¯,©,¥¸Õ'<Ô<ð ðXóÐ<ô —/‘/¤%§*¢*¨WÓ"5Ó6¸¸D¿J¹J½ÕFˆÜ—‘ ¨¥Ó1ˆÜ—‘ ¥°Õ 2Ó3ˆØŸ™¨Õ.ˆð �y‰y�~‰~×$Ñ$Ó&¨$¯+©+×*:Ñ*:×*AÑ*AÓ*CÕDˆÜ—?‘? 4Õ#8Ó9ˆà˜Õ ˆØ'¯+©+Õ5ˆØÕ"ˆàÐ Ð r)   )
rÕ   rv   rÔ   rÒ   r±   rÑ   r°   r   rÓ   r   )r>   Frª   r‰   )rA   rB   rC   rD   rE   r   râ   ræ   r;   rF   rG   rH   rI   s   @@r'   rÎ   rÎ   «  s%   ù‡ € ñ
÷Pò&\ò	÷.!ô .!r)   rÎ   c                óš  € V P                   ^ ,          p\        V^VR,          ,
          ,          4      p\        WB,          R,          4      p\        P                  ! WR7      P                  pV^ 8X  d   V^ ,          pM$\        P                  ! V ) VR7      P                  ) pV'       g!   VP                  4       pVP                  4       pW‡3# )a’  
Calculate the percentile max and min values in a given tensor

Args:
    input (`torch.Tensor`):
        The target tensor to calculate percentile max and min.
    lower_percentile (`float`):
        If 0.1, means we return the value of the smallest 0.1% value in the tensor as percentile min.
    upper_percentile (`float`):
        If 99.9, means we return the value of the largest 0.1% value in the tensor as percentile max.
    output_tensor (`bool`, *optional*, defaults to `False`):
        If True, this function returns tensors, otherwise it returns values.

Returns:
    `Tuple(torch.Tensor, torch.Tensor)`: Percentile min and max value of *input*
g{®Gáz„?)r“   )r‚   Úroundr   ÚkthvalueÚvaluesr\   )	ÚinputÚlower_percentileÚupper_percentileÚoutput_tensorÚinput_lengthÚlower_indexÚupper_indexÚupper_boundÚlower_bounds	   &&&&     r'   Úget_percentile_min_maxr    s§   € ð" —;‘;˜q•>€Lä˜¨Ð,<¸tÕ,CÕ(CÕDÓE€KÜ˜Õ7¸$Õ>Ó?€Kä—.’. Ô6×=Ñ=€Kà˜1ÔØ! A•o‰ô —~’~ u f°Ô<×CÑCÐCˆçØ!×&Ñ&Ó(ˆØ!×&Ñ&Ó(ˆØÐ#Ð#r)   c                ó  € \        V P                  4      ^8X  d*   VP                  R^^^4      pVP                  R^^^4      pMa\        V P                  4      ^8X  d&   VP                  R^4      pVP                  R^4      pM"VP                  R4      pVP                  R4      pV'       d8   V P                  RV,          4      P	                  V4      P                  4        V # \        P                  ! RV,          V ,          V,           4      # )a  
Quantize single-precision input tensor to integers with the given scaling factor and zeropoint.

Args:
    input (`torch.Tensor`):
        Single-precision input tensor to be quantized.
    scale (`torch.Tensor`):
        Scaling factor for quantization.
    zero_point (`torch.Tensor`):
        Shift for quantization.
    inplace (`bool`, *optional*, defaults to `False`):
        Whether to compute inplace or not.

Returns:
    `torch.Tensor`: Linearly quantized value of *input* according to *scale* and *zero_point*.
r¥   r`   )Úlenr‚   re   Úmul_Úadd_Úround_r   rø   )rû   ÚscaleÚ
zero_pointÚinplaces   &&&&r'   Úlinear_quantizer  4  sÒ   € ô$ ˆ5�;‰;Ó˜1ÔØ—
‘
˜2˜q ! QÓ'ˆØ—_‘_ R¨¨A¨qÓ1‰
ä	ˆU�[‰[Ó	˜QÔ	Ø—
‘
˜2˜qÓ!ˆØ—_‘_ R¨Ó+‰
à—
‘
˜2“ˆØ—_‘_ RÓ(ˆ
çØ�
‰
�3˜•;Ó×$Ñ$ ZÓ0×7Ñ7Ô9ØˆÜ�;Š;�s˜U•{ UÕ*¨ZÕ7Ó8Ð8r)   c           	     ó"  € \         P                  ! 4       ;_uu_ 4        ^V ^,
          ,          ^,
          pV'       do   \         P                  ! \         P                  ! VP	                  4       VP	                  4       .^R7      ^R7      w  rV\         P
                  ! VRR7      V,          pMG\        VP	                  4       VP	                  4       4      p\         P
                  ! VRR7      V,          pRRR4       V#   + '       g   i     X# ; i)aÿ  
Compute the scaling factor with the given quantization range for symmetric quantization.

Args:
    saturation_min (`torch.Tensor`):
        Lower bound for quantization range.
    saturation_max (`torch.Tensor`):
        Upper bound for quantization range.
    per_channel (`bool`, *optional*, defaults to `False`):
        Whether to or not use channel-wise quantization.

Returns:
    `torch.Tensor`: Scaling factor that linearly quantizes the given range between *saturation_min* and
    *saturation_max*.
rÆ   g:Œ0âŽyE>r¼   N)r   r·   r1   Ústackr›   r½   )Únum_bitsÚsaturation_minÚsaturation_maxrQ   rò   r
  r„   s   &&&&   r'   r2   r2   W  sÀ   € ô$ 
�Š��Ø�(˜Q•,Õ !Õ#ˆçÜ—y’y¤§¢¨n×.@Ñ.@Ó.BÀN×DVÑDVÓDXÐ-YÐ_`Ô!aÐghÔi‰HˆEÜ—K’K ¨4Ô0°1Õ4‰Eô ˜×*Ñ*Ó,¨n×.@Ñ.@Ó.BÓCˆEÜ—K’K ¨4Ô0°1Õ4ˆE÷ 
ð €L÷ 
Žð €Lús   žCC=Ã=D	c                   óD   a € ] tR tRt o Rt]R 4       t]R 4       tRtV t	R# )r   iw  zo
Class to quantize the given floating-point values using symmetric quantization with given range and bitwidth.
c                óÞ   € \         P                  ! RVP                  R7      p^V^,
          ,          ^,
          p\        WVRR7      p\         P                  ! Wv) V^,
          4      pW@n        V# )aÎ  
Args:
    x (`torch.Tensor`):
        Floating point tensor to be quantized.
    k (`int`):
        Quantization bitwidth.
    percentile_mode (`bool`):
        Whether or not to use percentile calibration.
    scale (`torch.Tensor`):
        Pre-calculated scaling factor for *x*. Note that the current implementation of SymmetricQuantFunction
        requires pre-calculated scaling factor.

Returns:
    `torch.Tensor`: Symmetric-quantized value of *input*.
g        )rî   F)r  )r   rë   rî   r  r½   r
  )Úctxr3   r“   r   r
  r  rò   Únew_quant_xs   &&&&&   r'   r;   ÚSymmetricQuantFunction.forward|  sY   € ô" —\’\ #¨e¯l©lÔ;ˆ
à�!�a•%�L˜1ÕˆÜ% a°
ÀEÔJˆÜ—k’k +¨r°1°qµ5Ó9ˆàŒ	ØÐr)   c                ó,  € V P                   p\        VP                  4      ^8X  d   VP                  R^^^4      pM>\        VP                  4      ^8X  d   VP                  R^4      pMVP                  R4      pVP	                  4       V,          RRRR3# )é   Nr`   )r
  r  r‚   re   Úclone)r  Úgrad_outputr
  s   && r'   ÚbackwardÚSymmetricQuantFunction.backward–  s�   € à—	‘	ˆÜˆ{× Ñ Ó! QÔ&Ø—J‘J˜r 1 a¨Ó+‰Eä�×"Ñ"Ó# qÔ(Ø—J‘J˜r 1Ó%‰Eà—J‘J˜r“NˆEà× Ñ Ó" UÕ*¨D°$¸¸dÐBÐBr)   © N©
rA   rB   rC   rD   rE   Ústaticmethodr;   r  rF   rG   ©rJ   s   @r'   r   r   w  s4   ø‡ € ñð ñó ðð2 ñ
Có ö
Cr)   r   c                   óD   a € ] tR tRt o Rt]R 4       t]R 4       tRtV t	R# )rœ   i¤  z3
Straight-through Estimator(STE) for torch.floor()
c                ó.   € \         P                  ! V4      # r‰   )r   r™   ©r  r3   s   &&r'   r;   Úfloor_ste.forward©  ó   € ä�{Š{˜1‹~Ðr)   c                ó"   € VP                  4       # r‰   ©r  ©r  r  s   &&r'   r  Úfloor_ste.backward­  ó   € à× Ñ Ó"Ð"r)   r  Nr  r!  s   @r'   rœ   rœ   ¤  ó2   ø‡ € ñð ñó ðð ñ#ó ö#r)   rœ   c                   óD   a € ] tR tRt o Rt]R 4       t]R 4       tRtV t	R# )rï   i²  z3
Straight-through Estimator(STE) for torch.round()
c                ó.   € \         P                  ! V4      # r‰   )r   rø   r$  s   &&r'   r;   Úround_ste.forward·  r&  r)   c                ó"   € VP                  4       # r‰   r(  r)  s   &&r'   r  Úround_ste.backward»  r+  r)   r  Nr  r!  s   @r'   rï   rï   ²  r,  r)   rï   c                óì  € V P                  4       pV P                  R4      p \        P                  ! V P	                  4       P                  4       4      w  r4. pV Ft  p\        \        P                  ! V^V,          ,          4      P                  \        P                  ! ^4      \        P                  R7      4      pVP                  V4       Kv  	  \        P                  ! V4      p\        V4      V,
          p\        P                  ! V4      P!                  V P"                  4      P                  V4      \        P                  ! V4      P!                  V P"                  4      P                  V4      3# )zà
Decompose the scaling factor into mantissa and twos exponent.

Args:
    scaling_factor (`torch.Tensor`):
        Target scaling factor to decompose.

Returns:
    ``Tuple(torch.Tensor, torch.Tensor)`: mantisa and exponent
)Úroundingr`   )Úsizere   ÚnpÚfrexpÚcpuÚnumpyrÞ   ÚdecimalÚDecimalÚquantizeÚROUND_HALF_UPÚappendÚarrayrì   r   Ú
from_numpyrí   rî   )Úinputsr±   Úshape_of_inputÚoutput_mÚoutput_eÚtmp_mÚmÚint_m_shifteds   &&      r'   Úbatch_frexprG  À  s  € ð —[‘[“]€Nð �[‰[˜‹_€FäŸš &§*¡*£,×"4Ñ"4Ó"6Ó7Ñ€HØ€EÛˆÜÜ�OŠO˜A  G¥Õ,Ó-×6Ñ6´w·²ÀqÓ7IÔT[×TiÑTiÐ6Ójó
ˆð 	�‰�]Ö#ñ	 ô
 �xŠx˜‹€Hä�W‹~ Õ(€Hô 	×Ò˜Ó"×%Ñ% f§m¡mÓ4×9Ñ9¸.ÓIÜ×Ò˜Ó"×%Ñ% f§m¡mÓ4×9Ñ9¸.ÓIðð r)   c                   óH   a € ] tR tRt o Rt]RR l4       t]R 4       tRtV t	R# )rd   iâ  a	  
Function to perform fixed-point arithmetic that can match integer arithmetic on hardware.

Args:
    pre_act (`torch.Tensor`):
        Input tensor.
    pre_act_scaling_factor (`torch.Tensor`):
        Scaling factor of the input tensor *pre_act*.
    bit_num (`int`):
        Quantization bitwidth.
    z_scaling_factor (`torch.Tensor`):
        Scaling factor of the output tensor.
    identity (`torch.Tensor`, *optional*):
        Identity tensor, if exists.
    identity_scaling_factor (`torch.Tensor`, *optional*):
        Scaling factor of the identity tensor *identity*, if exists.

Returns:
    `torch.Tensor`: Output tensor(*pre_act* if *identity* is not given, otherwise the addition of *pre_act* and
    *identity*), whose scale is rescaled to *z_scaling_factor*.
Nc                ó®  € \        VP                  4      ^8X  d   R pMR pWPn        ^V^,
          ,          ^,
          p\        P                  ! 4       ;_uu_ 4        V! V4      pVe	   V! V4      pW@n        \        P                  ! W,          4      p	VP                  \        P                  4      p
VP                  \        P                  4      P                  \        P                  4      pW«,          pV! V4      p\        V4      w  rÞV	P                  \        P                  4      VP                  \        P                  4      ,          p\        P                  ! VRV,          ,          4      pVEe   \        P                  ! WV,          4      pVP                  \        P                  4      p
VP                  \        P                  4      P                  \        P                  4      pW«,          pV! V4      p\        V4      w  ppVP                  \        P                  4      VP                  \        P                  4      ,          p\        P                  ! VRV,          ,          4      pVV,           p\        P                  ! VP                  \        P                  4      V) ^,
          V4      uuRRR4       #   + '       g   i     R# ; i)é   c                 ó   € V # r‰   r  ©r3   s   &r'   Ú<lambda>Ú'FixedPointMul.forward.<locals>.<lambda>  s   € ¡r)   c                 ó(   € V P                  ^^R4      # )r	   r`   )re   rL  s   &r'   rM  rN    s   €  §¡ q¨!¨RÔ 0r)   Nr=   )r  r‚   rg   r   r·   Úz_scaling_factorrø   ÚtypeÚdoublerì   rG  r½   )r  Úpre_actrf   Úbit_numrP  rg   rh   Úreshaperò   Úz_intÚ_AÚ_BÚ	new_scalerE  ÚeÚoutputÚwx_intÚm1Úe1Úoutput1s   &&&&&&&             r'   r;   ÚFixedPointMul.forwardù  së  € ô Ð%×+Ñ+Ó,°Ô1Ù!‰Gá0ˆGØŒà�'˜A•+Õ Õ"ˆä�]Š]�_�_Ù%,Ð-CÓ%DÐ"ØÒ#Ù*1Ð2IÓ*JÐ'à#3Ô ä—K’K Õ @ÓAˆEØ'×,Ñ,¬U¯\©\Ó:ˆBØ"×'Ñ'¬¯©Ó4×:Ñ:¼5¿<¹<ÓHˆBØ�ˆIÙ 	Ó*ˆIä˜yÓ)‰DˆAà—Z‘Z¤§¡Ó-°·±´u·|±|Ó0DÕDˆFÜ—[’[ ¨3°­6Õ!2Ó3ˆFàÓ#äŸš XÕ%GÓH�à,×1Ñ1´%·,±,Ó?�Ø&×+Ñ+¬E¯K©KÓ8×>Ñ>¼u¿|¹|ÓL�Ø�G�	Ù# IÓ.�	ä$ YÓ/‘��BØ Ÿ+™+¤e§l¡lÓ3°b·g±g¼e¿l¹lÓ6KÕK�ÜŸ+š+ g°°bµÕ&9Ó:�à  6Õ)�ä—;’;˜vŸ{™{¬5¯;©;Ó7¸!¸¸a½ÀÓC÷C �_�_‹_ús   ÁIKËK	c                ó´   € R pV P                   e"   VP                  4       V P                  ,          pVP                  4       V P                  ,          R R R R VR 3# r‰   )rg   r  rP  )r  r  Úidentity_grads   && r'   r  ÚFixedPointMul.backward.  sU   € àˆØ�<‰<Ò#Ø'×-Ñ-Ó/°#×2FÑ2FÕFˆMØ× Ñ Ó" S×%9Ñ%9Õ9¸4ÀÀtÈTÐS`ÐbfÐfÐfr)   r  r@   r  r!  s   @r'   rd   rd   â  s7   ø‡ € ñð, ó2Dó ð2Dðh ñgó ögr)   rd   )F)é   )r9  r8  r5  r   r   Útorch.autogradr   Úutilsr   Ú
get_loggerrA   r�   ÚModuler   rL   rq   r‹   r¬   rÎ   r  r  r2   r   rœ   rï   rG  rd   r  r)   r'   Ú<module>ri     sî   ðó" ã Û Ý Ý #å ð 
×	Ò	˜HÓ	%€ôPP�R—Y‘Yô PPôfgMˆr�y‰yô gMôTM
�"—)‘)ô M
ô`66ˆb�i‰iô 66ôrD8�—‘ô D8ôNb!�2—9‘9ô b!ôJ!$ôH 9ôFô@*C˜Xô *CôZ#�ô #ô#�ô #ôôDQg�Hö Qgr)   